AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend/copilot): dream facts cite their sources, and a forget reaches what was derived from them (#15020)
Nicholas Tindledae22d41d078
diff --git a/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes.py b/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes.pyindex 523f781278..699cbafe17 100644--- a/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes.py+++ b/autogpt_platform/backend/backend/api/features/admin/memory_admin_routes.py@@ -1031,7 +1031,8 @@ ADMIN_CANCEL_REASON = "cancelled by an admin"class DreamPassCancelRequest(BaseModel):- """Why an admin cancels a dream pass, kept as the pass's error."""+ """Why an admin cancels a dream pass, for the log; the pass records+ ``cancelled: by_admin``."""# Stripped before the length check, so a blank reason is refused.model_config = ConfigDict(str_strip_whitespace=True)diff --git a/autogpt_platform/backend/backend/api/features/memory/models.py b/autogpt_platform/backend/backend/api/features/memory/models.pyindex be62eb679c..fd7b288fde 100644--- a/autogpt_platform/backend/backend/api/features/memory/models.py+++ b/autogpt_platform/backend/backend/api/features/memory/models.py@@ -30,8 +30,16 @@ class MemoryFactListResponse(BaseModel):class ForgetFactResponse(BaseModel):+ """A forgotten fact. ``derived_forgotten`` counts the facts the dream had+ derived from it, retracted with it (``graphiti/recall_cascade.py``).+ ``resumed`` is True when the fact had already been erased by an earlier+ hard forget, and this one went on retracting and erasing what the dream+ derived from it (``graphiti/recall_forget.py``)."""+uuid: strforgotten: bool+ derived_forgotten: int = 0+ resumed: bool = Falseclass EraseMemoryResponse(BaseModel):diff --git a/autogpt_platform/backend/backend/api/features/memory/routes.py b/autogpt_platform/backend/backend/api/features/memory/routes.pyindex 486bc618ae..45e8a38e07 100644--- a/autogpt_platform/backend/backend/api/features/memory/routes.py+++ b/autogpt_platform/backend/backend/api/features/memory/routes.py@@ -9,10 +9,12 @@ so these routes can never read or delete another user's memory by construction.Facts are listed and counted with the recall policy's live-fact test(``graphiti/recall.py``), and a single-fact forget is the chat forget tool'ssoft ``retract`` (``graphiti/recall_forget.py``): the edge is marked-``retracted`` and stops being served but stays for audit, and every episode-it came from stops being recalled. A forget whose clean-up failed answers 500,-not success, and one that found memory busy answers 409; both retry. The-scope erase hard-deletes every node and edge in the scope's graph.+``retracted`` and stops being served but stays for audit, every episode it+came from stops being recalled, and so does every fact the dream derived+from it (counted in the response's ``derived_forgotten``). A forget whose+clean-up failed, or that left derived facts for a second forget, answers+500, not success, and one that found memory busy answers 409; both retry.+The scope erase hard-deletes every node and edge in the scope's graph."""import logging@@ -228,8 +230,13 @@ async def _forget_fact_impl(status_code=500,detail="Forgot this memory but could not finish cleaning up; try again",)- if result.deleted:- return ForgetFactResponse(uuid=fact_uuid, forgotten=True)+ if result.deleted or result.resumed:+ return ForgetFactResponse(+ uuid=fact_uuid,+ forgotten=True,+ derived_forgotten=len(result.derived),+ resumed=bool(result.resumed),+ )if MemoryForgetFailureCode.QUERY_ERROR in codes:raise HTTPException(status_code=500, detail="Could not forget this memory")raise HTTPException(status_code=404, detail="Memory not found")diff --git a/autogpt_platform/backend/backend/api/features/memory/routes_test.py b/autogpt_platform/backend/backend/api/features/memory/routes_test.pyindex 3bad82f918..e585b68bb6 100644--- a/autogpt_platform/backend/backend/api/features/memory/routes_test.py+++ b/autogpt_platform/backend/backend/api/features/memory/routes_test.py@@ -182,9 +182,47 @@ class TestForgetFact:with patch(f"{_MOCK_MODULE}.retract", retract):resp = client.delete("/memory/facts/edge-1")assert resp.status_code == 200- assert resp.json() == {"uuid": "edge-1", "forgotten": True}+ assert resp.json() == {+ "uuid": "edge-1",+ "forgotten": True,+ "derived_forgotten": 0,+ "resumed": False,+ }retract.assert_awaited_once_with(MemoryScope.for_user(test_user_id), ["edge-1"])+ def test_counts_the_facts_derived_from_it(self) -> None:+ """The dream's facts derived from the forgotten one were retracted+ with it (``graphiti/recall_cascade.py``); the page is told how many."""+ result = ForgetResult(deleted=["edge-1"], derived=["d1", "d2", "d3"])+ with patch(f"{_MOCK_MODULE}.retract", AsyncMock(return_value=result)):+ resp = client.delete("/memory/facts/edge-1")+ assert resp.status_code == 200+ assert resp.json()["derived_forgotten"] == 3++ def test_derived_facts_left_for_another_forget_is_500(self) -> None:+ """The forget stopped at its bound with derived facts still live: not+ reported as done, and forgetting again continues."""+ failure = MemoryForgetFailure.derived_left("edge-1")+ result = ForgetResult(deleted=["edge-1"], derived=["d1"], failures=[failure])+ with patch(f"{_MOCK_MODULE}.retract", AsyncMock(return_value=result)):+ resp = client.delete("/memory/facts/edge-1")+ assert resp.status_code == 500+ assert "try again" in resp.json()["detail"]++ def test_a_fact_already_erased_whose_cascade_resumed_is_forgotten(self) -> None:+ """An earlier hard forget purged it before its cascade finished; this+ forget went on with the cascade (``graphiti/recall_forget.py``)."""+ result = ForgetResult(resumed=["edge-1"], derived=["d1", "d2"])+ with patch(f"{_MOCK_MODULE}.retract", AsyncMock(return_value=result)):+ resp = client.delete("/memory/facts/edge-1")+ assert resp.status_code == 200+ assert resp.json() == {+ "uuid": "edge-1",+ "forgotten": True,+ "derived_forgotten": 2,+ "resumed": True,+ }+def test_no_match_is_404(self) -> None:result = ForgetResult(failures=[MemoryForgetFailure.no_match("edge-unknown")])with patch(f"{_MOCK_MODULE}.retract", AsyncMock(return_value=result)):diff --git a/autogpt_platform/backend/backend/copilot/db.py b/autogpt_platform/backend/backend/copilot/db.pyindex 8f27debef8..652003d36f 100644--- a/autogpt_platform/backend/backend/copilot/db.py+++ b/autogpt_platform/backend/backend/copilot/db.py@@ -1267,6 +1267,148 @@ async def update_message_content_by_sequence(return False+# A dream's narrative, the message its session opens with, or *erased* when+# a hard forget has marked the pass (``DreamPass.erasedAt``), in one statement+# that holds the pass's row FOR SHARE: a forget's mark that lands first is+# read here, and one that comes after waits for the insert, which its blank+# of the pass's narratives (``blank_dream_narratives``) then finds. It goes+# only into the pass's own dream session (the pass's owner's, of kind dream,+# naming the pass), which takes a copy of the pass's read set+# (``DREAM_READ_UUIDS``), so a forget finds the narrative once the pass's row+# has aged out; the read set of a pass from before read sets were recorded+# holds ``*``, which the copy keeps. With no row for the pass, or a session+# that is not its own, nothing is written.+_DREAM_NARRATIVE_SQL = """+WITH pass AS (+ SELECT "id", "userId", "erasedAt", "readUuids"+ FROM {schema_prefix}"DreamPass" WHERE "id" = $3 FOR SHARE+),+session AS (+ UPDATE {schema_prefix}"ChatSession" AS s SET+ "metadata" = COALESCE(s."metadata", '{{}}'::jsonb) || jsonb_build_object(+ $7::text, to_jsonb(COALESCE(pass."readUuids", ARRAY[]::text[]))),+ "updatedAt" = CURRENT_TIMESTAMP AT TIME ZONE 'UTC'+ FROM pass+ WHERE s."id" = $2 AND s."userId" = pass."userId"+ AND s."metadata"->>'kind' = 'dream'+ AND s."metadata"->>'dream_pass_id' = pass."id"+ RETURNING s."id", pass."erasedAt"+)+INSERT INTO {schema_prefix}"ChatMessage"+ ("id", "sessionId", "role", "sequence", "content", "metadata")+SELECT $1, session."id", 'assistant', 0,+ CASE WHEN session."erasedAt" IS NOT NULL THEN $5 ELSE $4 END,+ $6::jsonb+FROM session+"""++# The key of a dream session's metadata holding its pass's read set, which+# ``ChatSessionMetadata`` leaves out: only the forget's SQL reads it, and+# every write to session metadata after its creation merges keys in SQL.+DREAM_READ_UUIDS = "dream_read_uuids"++# The narratives' blank's own deadline in the database, just under the+# forget's for it (``dream/forget_copies.NARRATIVE_TIMEOUT_SECONDS``).+NARRATIVE_BLANK_STATEMENT_TIMEOUT_MS = 9_000++# The narrative of each of the user's dream sessions a hard forget reached+# ($1): the session of a pass it marked or named ($2), one whose pass read+# what it reached ($3, matched against the session's copy of the read set by+# a hash join, which outlives the pass's row), or one of the forget's scope+# (the account's when $6 is null, else that expert's) that may have read+# anything: its session holds no read set (written before sessions kept+# one) or one holding ``*`` (its pass's, from before read sets were+# recorded: ``data/dream_pass_models.UNKNOWN_READS``). A narrative already+# blank is not written again.+_BLANK_NARRATIVES_SQL = """+WITH reached AS (+ SELECT DISTINCT uuid FROM unnest($3::text[]) AS reached(uuid)+),+dream AS (+ SELECT s."id", s."metadata", s."expertId"+ FROM {schema_prefix}"ChatSession" AS s+ WHERE s."userId" = $1 AND s."metadata"->>'kind' = 'dream'+),+hit AS (+ SELECT "id" FROM dream+ WHERE "metadata"->>'dream_pass_id' IN (SELECT unnest($2::text[]))+ UNION+ SELECT dream."id"+ FROM dream,+ jsonb_array_elements_text(CASE+ WHEN jsonb_typeof(dream."metadata"->$5::text) = 'array'+ THEN dream."metadata"->$5::text ELSE '[]'::jsonb END) AS read(uuid)+ WHERE read.uuid IN (SELECT uuid FROM reached)+ UNION+ SELECT "id" FROM dream+ WHERE "expertId" IS NOT DISTINCT FROM $6::text+ AND (jsonb_typeof("metadata"->$5::text) IS DISTINCT FROM 'array'+ OR "metadata"->$5::text @> '["*"]'::jsonb)+)+UPDATE {schema_prefix}"ChatMessage" AS m SET "content" = $4+WHERE m."sessionId" IN (SELECT "id" FROM hit)+ AND m."sequence" = 0 AND m."role" = 'assistant'+ AND m."content" IS DISTINCT FROM $4+"""+++async def add_dream_narrative(+ session_id: str, pass_id: str, content: str, erased: str+) -> bool:+ """Open pass *pass_id*'s dream session *session_id* with its narrative+ *content*, or *erased* once a hard forget has marked the pass+ (``_DREAM_NARRATIVE_SQL``, ``dream/apply._write_dream_summary_message``).+ ``False``, logged, when nothing was written: the pass has no row, or the+ session is not its own."""+ written = await db.execute_raw_with_schema(+ _DREAM_NARRATIVE_SQL,+ str(uuid.uuid4()),+ session_id,+ pass_id,+ sanitize_string(content),+ sanitize_string(erased),+ dumps({"dream_pass_id": pass_id}),+ DREAM_READ_UUIDS,+ )+ if not written:+ logger.warning(+ f"Dream pass {pass_id}: no record of it, or session {session_id} is "+ "not its dream session; its narrative is not written"+ )+ return written > 0+++async def blank_dream_narratives(+ user_id: str,+ expert_id: str | None,+ pass_ids: list[str],+ reached: list[str],+ placeholder: str,+) -> int:+ """Replace with *placeholder*, for a hard forget in *user_id*'s scope of+ *expert_id* (the account's when ``None``) (``dream/forget_copies.py``),+ the narrative of each of the user's dream sessions for *pass_ids*, whose+ pass read one of the *reached* uuids, or of the scope with no record of+ what its pass read (the assistant message a dream opens its session+ with, ``dream/apply._write_dream_summary_message``); how many changed.+ The rest of the session is the user's chat with it, which no forget+ reaches. Raises when the statement runs past+ ``NARRATIVE_BLANK_STATEMENT_TIMEOUT_MS``."""+ if not pass_ids and not reached:+ return 0+ async with db.bounded_transaction(NARRATIVE_BLANK_STATEMENT_TIMEOUT_MS) as tx:+ return await db.execute_raw_with_schema(+ _BLANK_NARRATIVES_SQL,+ user_id,+ pass_ids,+ reached,+ sanitize_string(placeholder),+ DREAM_READ_UUIDS,+ expert_id,+ client=tx,+ )++async def update_chat_message_stamps(session_id: str,sequence: int,diff --git a/autogpt_platform/backend/backend/copilot/db_dream_narrative_test.py b/autogpt_platform/backend/backend/copilot/db_dream_narrative_test.pynew file mode 100644index 0000000000..e00ba96f31--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/db_dream_narrative_test.py@@ -0,0 +1,454 @@+"""The narrative a dream opens its session with, against a real database.+``blank_dream_narratives``: a hard forget blanks it (``dream/forget_copies.py``)+for the user's passes it names, whose pass read a uuid it reached (the+session's copy of the read set, which outlives the pass's row), or, in the+forget's scope only, whose session has no record of what its pass read (no+read set, or one holding ``UNKNOWN_READS``), and nothing else: not the+user's chat with the narrative, another pass's, another scope's, another+user's or an ordinary chat. ``add_dream_narrative``: written only into the pass's+own session and only while its row is there, blank once the pass is marked,+in one statement holding the pass's row, so a mark landing on either side of+the write leaves no narrative behind (the review's cancelled-apply race)."""++import asyncio+import logging+import time+import uuid+from collections.abc import AsyncIterator, Awaitable, Callable+from datetime import datetime, timezone++import pytest+from prisma.enums import DreamPassRoute, DreamPassTrigger+from prisma.models import ChatMessage, ChatSession, DreamPass, Expert, User++from backend.data import db+from backend.data.dream_pass_models import UNKNOWN_READS+from backend.util.json import SafeJson++from . import db as chat+from .db import (+ DREAM_READ_UUIDS,+ NARRATIVE_BLANK_STATEMENT_TIMEOUT_MS,+ add_dream_narrative,+ blank_dream_narratives,+ clear_session_pending_question,+ get_chat_session_metadata,+ set_session_pending_question,+)++pytestmark = pytest.mark.asyncio(loop_scope="session")++_NARRATIVE = "Last night I noticed Alice runs the Atlas launch."+++async def make_user() -> AsyncIterator[Callable[[], Awaitable[str]]]:+ created: list[str] = []++ async def make() -> str:+ user_id = str(uuid.uuid4())+ await User.prisma().create(+ data={+ "id": user_id,+ "email": f"dream-narrative-{user_id}@example.com",+ "topUpConfig": SafeJson({}),+ "timezone": "UTC",+ }+ )+ created.append(user_id)+ return user_id++ yield make+ await User.prisma().delete_many(where={"id": {"in": created}})+++async def _session(+ user_id: str, metadata: dict[str, object], expert_id: str | None = None+) -> str:+ """A session of the user's scope of *expert_id* (the account's when+ ``None``) that opens with ``_NARRATIVE``, then the user's reply and the+ assistant's answer, which repeats it."""+ session = await ChatSession.prisma().create(+ data={+ "userId": user_id,+ "metadata": SafeJson(metadata),+ **({"expertId": expert_id} if expert_id else {}),+ }+ )+ for sequence, (role, content) in enumerate(+ [+ ("assistant", _NARRATIVE),+ ("user", "Does she?"),+ ("assistant", _NARRATIVE),+ ]+ ):+ await ChatMessage.prisma().create(+ data={+ "sessionId": session.id,+ "role": role,+ "content": content,+ "sequence": sequence,+ }+ )+ return session.id+++async def _contents(session_id: str) -> list[str | None]:+ messages = await ChatMessage.prisma().find_many(+ where={"sessionId": session_id}, order={"sequence": "asc"}+ )+ return [message.content for message in messages]+++async def test_only_the_named_passes_narratives_are_blanked(make_user):+ owner, stranger = await make_user(), await make_user()+ read = {DREAM_READ_UUIDS: ["u9"]}+ named = await _session(owner, {"kind": "dream", "dream_pass_id": "p1", **read})+ unnamed = await _session(owner, {"kind": "dream", "dream_pass_id": "p2", **read})+ ordinary = await _session(owner, {"kind": "normal", "dream_pass_id": "p1"})+ theirs = await _session(stranger, {"kind": "dream", "dream_pass_id": "p1"})++ blanked = await blank_dream_narratives(+ owner, None, ["p1", "p-gone"], [], "[forgotten]"+ )++ assert blanked == 1+ assert await _contents(named) == ["[forgotten]", "Does she?", _NARRATIVE]+ for session_id in (unnamed, ordinary, theirs):+ assert await _contents(session_id) == [_NARRATIVE, "Does she?", _NARRATIVE]+++async def test_no_passes_and_nothing_reached_writes_nothing(make_user):+ owner = await make_user()+ legacy = await _session(owner, {"kind": "dream", "dream_pass_id": "p1"})++ assert await blank_dream_narratives(owner, None, [], [], "[forgotten]") == 0+ assert await _contents(legacy) == [_NARRATIVE, "Does she?", _NARRATIVE]+++async def test_a_narrative_whose_pass_read_a_reached_uuid_is_blanked_once(make_user):+ """Found by the session's copy of the read set, its pass's row gone or+ not; a session whose pass read none of it, or another user's, is left.+ A narrative already blank is not written again."""+ owner, stranger = await make_user(), await make_user()+ read = await _session(+ owner, {"kind": "dream", "dream_pass_id": "p1", DREAM_READ_UUIDS: ["u1", "u2"]}+ )+ unread = await _session(+ owner, {"kind": "dream", "dream_pass_id": "p2", DREAM_READ_UUIDS: ["u3"]}+ )+ theirs = await _session(+ stranger, {"kind": "dream", "dream_pass_id": "p3", DREAM_READ_UUIDS: ["u2"]}+ )++ assert await blank_dream_narratives(owner, None, [], ["u2"], "[forgotten]") == 1+ assert await blank_dream_narratives(owner, None, [], ["u2"], "[forgotten]") == 0++ assert await _contents(read) == ["[forgotten]", "Does she?", _NARRATIVE]+ for session_id in (unread, theirs):+ assert await _contents(session_id) == [_NARRATIVE, "Does she?", _NARRATIVE]+++async def _expert(owner: str) -> str:+ expert = await Expert.prisma().create(+ data={+ "ownerUserId": owner,+ "name": "Scout",+ "role": "Research",+ "identity": "Finds things out.",+ }+ )+ return expert.id++[email protected]("forget_in", ["account", "expert"])+async def test_a_narrative_of_unknown_provenance_goes_with_any_forget_in_its_scope(+ make_user, forget_in: str+):+ """A session written before sessions kept their pass's read set, or+ whose read set holds ``UNKNOWN_READS`` (its pass's, from before read+ sets were recorded), may rest on anything its pass read: any hard forget+ in its scope blanks it, whatever it reached. One whose read set is known+ and holds none of it is left, and so is every session of another scope,+ of another user, or that is not a dream."""+ owner, stranger = await make_user(), await make_user()+ expert = await _expert(owner)+ scope, other = (None, expert) if forget_in == "account" else (expert, None)++ def dream(read: object = None) -> dict[str, object]:+ known = {} if read is None else {DREAM_READ_UUIDS: read}+ return {"kind": "dream", "dream_pass_id": str(uuid.uuid4()), **known}++ no_read_set = await _session(owner, dream(), scope)+ marked = await _session(owner, dream([UNKNOWN_READS, "u1"]), scope)+ unreadable = await _session(owner, dream("u1"), scope)+ known = await _session(owner, dream(["u3"]), scope)+ other_scopes = await _session(owner, dream(), other)+ other_scopes_marked = await _session(owner, dream([UNKNOWN_READS]), other)+ theirs = await _session(stranger, dream())+ ordinary = await _session(owner, {"kind": "normal"}, scope)++ blanked = await blank_dream_narratives(owner, scope, [], ["u2"], "[forgotten]")++ assert blanked == 3+ for session_id in (no_read_set, marked, unreadable):+ assert await _contents(session_id) == ["[forgotten]", "Does she?", _NARRATIVE]+ for session_id in (known, other_scopes, other_scopes_marked, theirs, ordinary):+ assert await _contents(session_id) == [_NARRATIVE, "Does she?", _NARRATIVE]+ assert await blank_dream_narratives(owner, scope, [], ["u2"], "[forgotten]") == 0+++async def test_a_pass_from_before_read_sets_marks_its_narrative_as_unknown(make_user):+ """A pass whose writer did not state its read set (one in flight across+ the deploy) reads ``UNKNOWN_READS``; its narrative's session takes it,+ and an unrelated forget in its scope blanks the narrative once the+ pass's row is gone."""+ owner = await make_user()+ pass_id = str(uuid.uuid4())+ await DreamPass.prisma().create(+ data={+ "id": pass_id,+ "userId": owner,+ "scopeKey": owner,+ "route": DreamPassRoute.SYNC,+ "trigger": DreamPassTrigger.CRON,+ }+ )+ session = await ChatSession.prisma().create(+ data={+ "userId": owner,+ "metadata": SafeJson({"kind": "dream", "dream_pass_id": pass_id}),+ }+ )+ assert await add_dream_narrative(session.id, pass_id, _NARRATIVE, "[forgotten]")+ stored = await ChatSession.prisma().find_unique(where={"id": session.id})+ assert stored is not None+ assert dict(stored.metadata)[DREAM_READ_UUIDS] == [UNKNOWN_READS]+ await DreamPass.prisma().delete(where={"id": pass_id})++ blanked = await blank_dream_narratives(+ owner, None, [], ["unrelated"], "[forgotten]"+ )++ assert blanked == 1+ assert await _contents(session.id) == ["[forgotten]"]+++async def _dream_session(+ user_id: str, *, erased: bool, reads: tuple[str, ...] = ("fact-1", "episode-1")+) -> tuple[str, str]:+ """A pass of the user's that read *reads*, marked by a hard forget when+ ``erased``, and its empty dream session: the pass id and the session+ id."""+ pass_id = str(uuid.uuid4())+ await DreamPass.prisma().create(+ data={+ "id": pass_id,+ "userId": user_id,+ "scopeKey": user_id,+ "route": DreamPassRoute.SYNC,+ "trigger": DreamPassTrigger.CRON,+ "readUuids": list(reads),+ **({"erasedAt": datetime_now()} if erased else {}),+ }+ )+ session = await ChatSession.prisma().create(+ data={+ "userId": user_id,+ "metadata": SafeJson({"kind": "dream", "dream_pass_id": pass_id}),+ }+ )+ return pass_id, session.id+++def datetime_now() -> datetime:+ return datetime.now(timezone.utc)++[email protected]("erased", [False, True])+async def test_a_narrative_is_written_blank_once_its_pass_is_marked(+ make_user, erased: bool+):+ owner = await make_user()+ pass_id, session_id = await _dream_session(owner, erased=erased)++ assert await add_dream_narrative(session_id, pass_id, _NARRATIVE, "[forgotten]")++ [message] = await ChatMessage.prisma().find_many(where={"sessionId": session_id})+ assert (message.role, message.sequence) == ("assistant", 0)+ assert message.content == ("[forgotten]" if erased else _NARRATIVE)+ session = await ChatSession.prisma().find_unique(where={"id": session_id})+ assert session is not None+ assert dict(session.metadata)[DREAM_READ_UUIDS] == ["fact-1", "episode-1"]+++async def test_a_narrative_with_no_pass_row_is_not_written(make_user, caplog):+ """A pass whose row is not there (its insert failed, so it should never+ have run) has nothing a forget could find it by."""+ owner = await make_user()+ session = await ChatSession.prisma().create(+ data={+ "userId": owner,+ "metadata": SafeJson({"kind": "dream", "dream_pass_id": "p-no-row"}),+ }+ )++ with caplog.at_level(logging.WARNING, logger=chat.logger.name):+ written = await add_dream_narrative(+ session.id, "p-no-row", _NARRATIVE, "[forgotten]"+ )++ assert written is False+ assert await _contents(session.id) == []+ assert "its narrative is not written" in caplog.text+++ "owner_is, metadata",+ [+ ("stranger", {"kind": "dream"}),+ ("owner", {"kind": "normal"}),+ ("owner", {"kind": "dream", "dream_pass_id": "another-pass"}),+ ],+ ids=["another_users", "not_a_dream", "another_passes"],+)+async def test_a_narrative_goes_only_into_its_passes_own_session(+ make_user, owner_is: str, metadata: dict[str, str]+):+ owner, stranger = await make_user(), await make_user()+ pass_id, _ = await _dream_session(owner, erased=False)+ session = await ChatSession.prisma().create(+ data={+ "userId": stranger if owner_is == "stranger" else owner,+ "metadata": SafeJson({"dream_pass_id": pass_id, **metadata}),+ }+ )++ assert not await add_dream_narrative(session.id, pass_id, _NARRATIVE, "[forgotten]")++ assert await _contents(session.id) == []+ stored = await ChatSession.prisma().find_unique(where={"id": session.id})+ assert stored is not None and DREAM_READ_UUIDS not in dict(stored.metadata)+++async def test_the_read_set_outlives_the_sessions_reads_and_merges(make_user):+ """``ChatSessionMetadata`` leaves the key out (only the forget's SQL+ reads it), and every write to a session's metadata after its creation+ merges keys, so the copy stays for the forget."""+ owner = await make_user()+ pass_id, session_id = await _dream_session(owner, erased=False)+ assert await add_dream_narrative(session_id, pass_id, _NARRATIVE, "[forgotten]")++ await set_session_pending_question(+ session_id, owner, "Does she?", datetime.now(timezone.utc)+ )+ await clear_session_pending_question(session_id, owner)+ loaded = await get_chat_session_metadata(session_id)++ assert loaded is not None and loaded.metadata.kind == "dream"+ blanked = await blank_dream_narratives(owner, None, [], ["fact-1"], "[forgotten]")+ assert blanked == 1+ assert await _contents(session_id) == ["[forgotten]"]+++async def test_a_mark_that_lands_first_holds_the_write_back_until_it_commits(+ make_user,+):+ """The forget marks the pass in a transaction; the narrative's write waits+ for it, then reads the mark and writes the placeholder."""+ owner = await make_user()+ pass_id, session_id = await _dream_session(owner, erased=False)++ async with db.transaction() as tx:+ await db.execute_raw_with_schema(+ 'UPDATE {schema_prefix}"DreamPass" SET "erasedAt" = now() WHERE "id" = $1',+ pass_id,+ client=tx,+ )+ write = asyncio.create_task(+ add_dream_narrative(session_id, pass_id, _NARRATIVE, "[forgotten]")+ )+ await asyncio.sleep(0.5)+ assert not write.done(), "the write waits for the mark"+ await asyncio.wait_for(write, 10)++ assert await _contents(session_id) == ["[forgotten]"]+++async def test_a_write_that_lands_first_is_found_by_the_forgets_blank(make_user):+ """The narrative's write holds the pass's row while it goes in: the mark+ waits for it, and the forget's blank of the narratives after the mark+ finds it."""+ owner = await make_user()+ pass_id, session_id = await _dream_session(owner, erased=False)++ async with db.transaction() as tx:+ await db.query_raw_with_schema(+ 'SELECT 1 FROM {schema_prefix}"DreamPass" WHERE "id" = $1 FOR SHARE',+ pass_id,+ client=tx,+ )+ mark = asyncio.create_task(+ DreamPass.prisma().update(+ where={"id": pass_id}, data={"erasedAt": datetime_now()}+ )+ )+ await asyncio.sleep(0.5)+ assert not mark.done(), "the mark waits for the row the write holds"+ await tx.chatmessage.create(+ data={+ "sessionId": session_id,+ "role": "assistant",+ "sequence": 0,+ "content": _NARRATIVE,+ }+ )+ await asyncio.wait_for(mark, 10)+ await blank_dream_narratives(owner, None, [pass_id], [], "[forgotten]")++ assert await _contents(session_id) == ["[forgotten]"]+++async def test_the_blank_over_3000_dream_sessions_is_in_time(make_user):+ """The cost check's other half: 3,000 dream sessions each holding a+ 550-uuid read set and a narrative, against 1,000 reached uuids read by+ none, then one read by every pass, well inside the statement's+ timeout."""+ owner = await make_user()+ shared = "fact-read-by-every-pass"+ await db.execute_raw_with_schema(+ 'INSERT INTO {schema_prefix}"ChatSession" ("id", "userId", "updatedAt", '+ '"metadata", "credentials", "successfulAgentRuns", '+ '"successfulAgentSchedules") '+ "SELECT gen_random_uuid()::text, $1, now(), jsonb_build_object("+ "'kind', 'dream', 'dream_pass_id', gen_random_uuid()::text, $2::text, "+ "to_jsonb(ARRAY(SELECT gen_random_uuid()::text "+ "FROM generate_series(1, 549) WHERE session.n > 0) || $3::text)), "+ "'{{}}'::jsonb, '{{}}'::jsonb, '{{}}'::jsonb "+ "FROM generate_series(1, 3000) AS session(n)",+ owner,+ DREAM_READ_UUIDS,+ shared,+ )+ await db.execute_raw_with_schema(+ 'INSERT INTO {schema_prefix}"ChatMessage" ("id", "sessionId", "role", '+ '"sequence", "content") SELECT gen_random_uuid()::text, "id", '+ "'assistant', 0, $2 FROM {schema_prefix}\"ChatSession\" "+ 'WHERE "userId" = $1',+ owner,+ _NARRATIVE,+ )+ budget = NARRATIVE_BLANK_STATEMENT_TIMEOUT_MS / 1000 / 2+ unread = [str(uuid.uuid4()) for _ in range(1000)]++ started = time.monotonic()+ assert await blank_dream_narratives(owner, None, [], unread, "[forgotten]") == 0+ selection = time.monotonic() - started+ started = time.monotonic()+ blanked = await blank_dream_narratives(+ owner, None, [], [*unread[1:], shared], "[forgotten]"+ )+ every = time.monotonic() - started++ assert blanked == 3000+ assert selection < budget and every < budget, (selection, every)diff --git a/autogpt_platform/backend/backend/copilot/dream/apply.py b/autogpt_platform/backend/backend/copilot/dream/apply.pyindex 72d1950034..a6599ed29e 100644--- a/autogpt_platform/backend/backend/copilot/dream/apply.py+++ b/autogpt_platform/backend/backend/copilot/dream/apply.py@@ -8,12 +8,19 @@ Three side-effects, in order:3. Demotions / entity invalidations, written by ``demotions.py`` with therecall guard in each statement.+Every write and proposal must cite what the pass read, and no fact of+another scope; one that does not is dropped first and counted+(``citations.py``).+A ``ChatSession`` shell (``metadata.kind='dream'`` +``metadata.dream_pass_id``) is created up front so the MemoryEnvelopeprovenance can reference its id; the assistant message holding``summary_for_user`` is appended LAST, after the ops above, so a partway-failure leaves an empty dream rather than a narrative with no memory.-A pass with no operations at all creates neither — see the empty-pass+failure leaves an empty dream rather than a narrative with no memory. It is+written as the forgotten placeholder when a write or proposal was dropped+for resting on a forget, or a hard forget has marked the pass+(``forget_copies.py``): the summary may restate what was forgotten anywhere.+A pass with no operations left at all creates neither — see the empty-passguard at the top of ``apply_operations``."""@@ -22,9 +29,12 @@ from __future__ import annotationsimport asyncioimport loggingimport uuid as uuidlib-from collections.abc import Mapping+from collections.abc import Collection, Mappingfrom datetime import datetime, timezone-from typing import TYPE_CHECKING+from functools import partial+from typing import TYPE_CHECKING, Literal++from pydantic import BaseModel, Fieldfrom backend.copilot.graphiti.ingest import (IngestionCompletion,@@ -37,10 +47,12 @@ from backend.copilot.graphiti.memory_model import (MemoryStatus,SourceKind,)+from backend.copilot.graphiti.recall import FORGOTTEN_FACTfrom backend.copilot.graphiti.recall_citations import Citationsfrom backend.copilot.graphiti.scope import MemoryScopefrom backend.copilot.transports import resolve_default_chat_route+from .citations import source_description, validated_citationsfrom .demotions import apply_demotionsfrom .fetch import DREAM_EPISODE_NAME_PREFIXfrom .locks import LOCK_CHECK_TIMEOUT_SECONDS, DreamLockHandle, DreamLockLostError@@ -154,23 +166,55 @@ def _edge_metadata(envelope: MemoryEnvelope) -> dict:}-def _citations(- statement: str,- fact_uuids: list[str],- episode_uuids: list[str],- read: Citations | None,-) -> Citations:- """What a dream write rests on, which the ingestion worker checks against- forgets right before writing it (``graphiti/recall_citations.py``): what- it cites or, citing nothing, everything its pass read (``read``) and its- own statement, since an uncited claim may come from any of it."""- if fact_uuids or episode_uuids:- return Citations(fact_uuids=list(fact_uuids), episode_uuids=list(episode_uuids))- everything = read or Citations()- return Citations(- fact_uuids=everything.fact_uuids,- episode_uuids=everything.episode_uuids,- statement=statement,+class _Cited(BaseModel):+ """A pass's writes and proposals, each with the citations that name+ something the pass read (``citations.validated_citations``), by its index+ in the pass; ``uncited`` counts those dropped for citing nothing it read+ or a fact of another scope, ``cross_scope`` their citations of a fact of+ another scope."""++ writes: list[tuple[int, ConsolidatedFact, Citations]] = Field(default_factory=list)+ proposals: list[tuple[int, ProposedFinding, Citations]] = Field(+ default_factory=list+ )+ uncited: int = 0+ cross_scope: int = 0+++def _cite(+ ops: DreamOperations,+ known_facts: Collection[str],+ known_episodes: Collection[str],+ fact_scopes: Mapping[str, str],+) -> _Cited:+ """``ops``' writes and proposals checked against what the pass read."""+ check = partial(+ validated_citations,+ known_facts=known_facts,+ known_episodes=known_episodes,+ fact_scopes=fact_scopes,+ )+ items: list[ConsolidatedFact | ProposedFinding] = [*ops.writes, *ops.proposals]+ checked = [+ check(item.source_fact_uuids, item.source_episode_uuids, scope=item.scope)+ for item in items+ ]+ found = checked[: len(ops.writes)], checked[len(ops.writes) :]+ writes = [+ (i, fact, kept.citations)+ for i, (fact, kept) in enumerate(zip(ops.writes, found[0]))+ if kept.citations is not None+ ]+ proposals = [+ (i, finding, kept.citations)+ for i, (finding, kept) in enumerate(zip(ops.proposals, found[1]))+ if kept.citations is not None+ ]+ return _Cited(+ writes=writes,+ proposals=proposals,+ uncited=len(items) - len(writes) - len(proposals),+ cross_scope=sum(kept.cross_scope for kept in checked),)@@ -182,8 +226,10 @@ async def _write_consolidated_fact(session_id: str,completion: IngestionCompletion,*,- read: Citations | None = None,+ citations: Citations,) -> bool:+ """Queue one consolidated fact with what it cites: the worker checks the+ citations against forgets, then records them (``citations.py``)."""envelope = MemoryEnvelope(content=fact.content,source_kind=SourceKind.assistant_derived,@@ -198,14 +244,11 @@ async def _write_consolidated_fact(session_id=session_id,name=_episode_name(pass_id, "consolidate", counter),episode_body=envelope.model_dump_json(),- source_description=(- f"dream-pass consolidation; src_episodes="- f"{','.join(fact.source_episode_uuids[:5])}"- ),+ source_description=source_description("consolidation"),is_json=True,edge_metadata=_edge_metadata(envelope),completion=completion,- citations=_citations(fact.content, [], fact.source_episode_uuids, read),+ citations=citations,)@@ -217,8 +260,9 @@ async def _write_proposed_finding(session_id: str,completion: IngestionCompletion,*,- read: Citations | None = None,+ citations: Citations,) -> bool:+ """Queue one proposal with what it cites, like a consolidated fact."""envelope = MemoryEnvelope(content=finding.content,source_kind=SourceKind.assistant_derived,@@ -228,29 +272,75 @@ async def _write_proposed_finding(scope=finding.scope,provenance=_provenance(pass_id, "recombine"),)- description_parts: list[str] = ["dream-pass proposal"]- if finding.rationale:- description_parts.append(f"rationale={finding.rationale[:240]}")- if finding.source_fact_uuids:- description_parts.append(f"src_facts={','.join(finding.source_fact_uuids[:5])}")return await enqueue_episode(scope,session_id=session_id,name=_episode_name(pass_id, "recombine", counter),episode_body=envelope.model_dump_json(),- source_description="; ".join(description_parts),+ source_description=source_description("proposal", rationale=finding.rationale),is_json=True,edge_metadata=_edge_metadata(envelope),completion=completion,- citations=_citations(- finding.content,- finding.source_fact_uuids,- finding.source_episode_uuids,- read,- ),+ citations=citations,)+def _summary(+ item: ConsolidatedFact | ProposedFinding,+ citations: Citations,+ status: Literal["active", "tentative"],+) -> WriteSummary:+ """The snapshot line for a queued write: what it said and what it cites."""+ return WriteSummary(+ content=item.content,+ scope=item.scope,+ confidence=item.confidence,+ status=status,+ source_episode_uuids=list(citations.episode_uuids),+ source_fact_uuids=list(citations.fact_uuids),+ )+++async def _queue_cited(+ scope: MemoryScope,+ pass_id: str,+ cited: _Cited,+ session_id: str,+ writes_done: IngestionCompletion,+ proposals_done: IngestionCompletion,+) -> tuple[list[WriteSummary], list[WriteSummary]]:+ """Queue the cited writes, then the cited proposals, registering each+ queued write on ``writes_done`` and each proposal on ``proposals_done``;+ the snapshot lines of those queued."""+ writes: list[WriteSummary] = []+ for i, fact, citations in cited.writes:+ if await _write_consolidated_fact(+ scope,+ pass_id,+ i,+ fact,+ session_id=session_id,+ completion=writes_done,+ citations=citations,+ ):+ writes_done.register()+ writes.append(_summary(fact, citations, "active"))+ proposals: list[WriteSummary] = []+ for i, finding, citations in cited.proposals:+ if await _write_proposed_finding(+ scope,+ pass_id,+ i,+ finding,+ session_id=session_id,+ completion=proposals_done,+ citations=citations,+ ):+ proposals_done.register()+ proposals.append(_summary(finding, citations, "tentative"))+ return writes, proposals++async def _create_dream_session(scope: MemoryScope, pass_id: str) -> str:"""Create the dream-kind ChatSession shell and return its id.@@ -323,22 +413,21 @@ async def _create_dream_session(scope: MemoryScope, pass_id: str) -> str:async def _write_dream_summary_message(- session_id: str, pass_id: str, summary_for_user: str+ session_id: str, pass_id: str, summary_for_user: str, *, dropped: bool = False) -> None:"""Append the assistant narrative to an already-created dream session.Called at the END of ``apply_operations`` so the user-visible summary- only appears once the memory ops above have been attempted.+ only appears once the memory ops above have been attempted. ``dropped``+ (a write or proposal of the pass was dropped for resting on a forget)+ writes the placeholder instead, as the database does for a pass a hard+ forget has marked (``db.add_dream_narrative``)."""from backend.data.db_accessors import chat_dbbody = summary_for_user.strip() or "Dream pass completed with no narrative output."- await chat_db().add_chat_message(- session_id=session_id,- role="assistant",- sequence=0,- content=body,- metadata={"dream_pass_id": pass_id},+ await chat_db().add_dream_narrative(+ session_id, pass_id, FORGOTTEN_FACT if dropped else body, FORGOTTEN_FACT)@@ -413,6 +502,7 @@ async def apply_operations(*,known_fact_uuids: set[str] | None = None,known_episode_uuids: set[str] | None = None,+ fact_scopes: Mapping[str, str] | None = None,ingestion_drain_timeout: float = INGESTION_DRAIN_TIMEOUT_SECONDS,lock_handle: DreamLockHandle | None = None,lease: ApplyLease | None = None,@@ -437,27 +527,38 @@ async def apply_operations(``dropped_forgotten``.An empty pass — no writes, proposals, demotions, or entity- invalidations — returns zero counts and an empty snapshot WITHOUT- creating the dream session or writing any message; the- ``session_id`` key is absent so ``apply_stats.get("session_id")``- reads as ``None`` for both the orchestrator and the batch callback.- A pass WITH operations but an empty ``summary_for_user`` still- creates the session and writes the fallback narrative (the ops were- attempted; only the narrative is missing).+ invalidations, once the writes citing nothing the pass read are+ dropped — returns zero counts (``uncited_writes_dropped`` aside) and an+ empty snapshot WITHOUT creating the dream session or writing any+ message; the ``session_id`` key is absent so+ ``apply_stats.get("session_id")`` reads as ``None`` for both the+ orchestrator and the batch callback. A pass WITH operations but an+ empty ``summary_for_user`` still creates the session and writes the+ fallback narrative (the ops were attempted; only the narrative is+ missing).``known_fact_uuids`` is the set of edge uuids the dream passactually fetched (``DreamInput.known_fact_uuids``); demotionstargeting anything outside it are dropped before any Cypher runs(``demotions.py``). ``None`` means "look up the persisted input- bundle by pass_id".-- Each write and proposal is queued with what it cites; one that cites- nothing is taken to rest on everything the pass read, its- ``known_fact_uuids`` and ``known_episode_uuids``. The ingestion worker+ bundle by pass_id" for the demotions.++ Each write and proposal keeps the citations that name a fact in+ ``known_fact_uuids`` or an episode in ``known_episode_uuids``+ (``citations.py``). One left citing nothing, or citing a fact whose+ scope (``fact_scopes``, from ``citations.fact_scopes``) is not its own,+ is dropped whole before it is queued and counted in+ ``uncited_writes_dropped``, and each of its citations of a fact of+ another scope in ``cross_scope_citations_dropped``; episode citations+ are not scoped. Both routes pass their input bundle's sets and fact+ scopes; ``None`` reads as nothing read, so every write is dropped, and a+ fact with no scope given is ``citations.UNSCOPED``. The ingestion workerdrops, unwritten, any whose citations a forget reached after the pass- read the graph (``graphiti/recall_citations.py``). ``dropped_forgotten``- counts those dropped before apply returned: all of them on a drained- pass, possibly fewer when the drain was skipped or timed out.+ read the graph (``graphiti/recall_citations.py``), and records the+ citations of every write it makes (``graphiti/recall_derivation.py``).+ ``dropped_forgotten`` counts those dropped before apply returned: all of+ them on a drained pass, possibly fewer when the drain was skipped or+ timed out.``ingestion_drain_timeout`` bounds the in-line wait for the worker tofinish the enqueued episodes (see ``_drain_ingestion``). The sync path keeps the@@ -501,7 +602,22 @@ async def apply_operations(the engine is still booting)."""user_id = scope.owner_user_id- if not (ops.writes or ops.proposals or ops.demotions or ops.entity_invalidations):+ cited = _cite(+ ops,+ known_fact_uuids or set(),+ known_episode_uuids or set(),+ fact_scopes or {},+ )+ if cited.uncited:+ logger.warning(+ f"Dream pass {pass_id} for user {user_id[:12]}: dropped "+ f"{cited.uncited} write(s) and proposal(s) citing nothing the pass "+ f"read or a fact of another scope ({cited.cross_scope} citation(s) "+ "of another scope)"+ )+ if not (+ cited.writes or cited.proposals or ops.demotions or ops.entity_invalidations+ ):# Empty pass — nothing landed in memory, so don't manufacture a# user-visible artifact for it. Creating the session shell +# placeholder narrative here is what produced one untitled empty@@ -522,6 +638,9 @@ async def apply_operations("dropped_forgotten": 0,"pending_writes": 0,"failed_writes": 0,+ "provenance_pending": 0,+ "uncited_writes_dropped": cited.uncited,+ "cross_scope_citations_dropped": cited.cross_scope,"protected_demotions": 0,"indeterminate_demotion_writes": 0,"demotion_accounting_complete": True,@@ -548,56 +667,10 @@ async def apply_operations(# saved of each. Registered once per successful enqueue; the worker# records what became of each episode as it finishes it.writes_done, proposals_done = IngestionCompletion(), IngestionCompletion()- read = Citations(- fact_uuids=sorted(known_fact_uuids or ()),- episode_uuids=sorted(known_episode_uuids or ()),+ write_summaries, proposal_summaries = await _queue_cited(+ scope, pass_id, cited, session_id, writes_done, proposals_done)- write_summaries: list[WriteSummary] = []- for i, fact in enumerate(ops.writes):- if await _write_consolidated_fact(- scope,- pass_id,- i,- fact,- session_id=session_id,- completion=writes_done,- read=read,- ):- writes_done.register()- write_summaries.append(- WriteSummary(- content=fact.content,- scope=fact.scope,- confidence=fact.confidence,- status="active",- source_episode_uuids=list(fact.source_episode_uuids),- )- )-- proposal_summaries: list[WriteSummary] = []- for i, prop in enumerate(ops.proposals):- if await _write_proposed_finding(- scope,- pass_id,- i,- prop,- session_id=session_id,- completion=proposals_done,- read=read,- ):- proposals_done.register()- proposal_summaries.append(- WriteSummary(- content=prop.content,- scope=prop.scope,- confidence=prop.confidence,- status="tentative",- source_episode_uuids=list(prop.source_episode_uuids),- source_fact_uuids=list(prop.source_fact_uuids),- )- )-# Renew the dream lock right before the longest non-LLM tail (the# ingestion drain plus the demotions / summary write that follow), to a# budget sized for that tail. The renewal proves ownership at the instant@@ -628,25 +701,33 @@ async def apply_operations(destroyed = await apply_demotions(scope, pass_id, ops, known_fact_uuids)# Narrative summary last — only surface the user-facing dream story- # once the memory ops above have been attempted.- await _write_dream_summary_message(session_id, pass_id, ops.summary_for_user)-+ # once the memory ops above have been attempted, and none of it once a+ # write it may restate was dropped for resting on a forget. A drop the+ # drain did not wait for (the batch path's, a timed-out drain's) is not+ # known here: a hard forget's mark covers that pass instead.trackers = (writes_done, proposals_done)dropped = sum(tracker.dropped_forgotten for tracker in trackers)+ await _write_dream_summary_message(+ session_id, pass_id, ops.summary_for_user, dropped=dropped > 0+ )pending = sum(tracker.pending for tracker in trackers)failed = sum(tracker.failed for tracker in trackers)+ provenance_pending = sum(tracker.provenance_pending for tracker in trackers)logger.info("Dream pass %s applied for user %s: "- "writes=%d proposals=%d dropped_forgotten=%d pending=%d failed=%d "- "demoted=%d (failed=%d) protected=%d entity_edges=%d indeterminate=%d "- "accounting_complete=%s ingestion_drain_status=%s",+ "writes=%d proposals=%d uncited=%d dropped_forgotten=%d pending=%d "+ "failed_writes=%d provenance_pending=%d demoted=%d (failed=%d) "+ "protected=%d entity_edges=%d indeterminate=%d accounting_complete=%s "+ "ingestion_drain_status=%s",pass_id,user_id[:12],writes_done.written,proposals_done.written,+ cited.uncited,dropped,pending,failed,+ provenance_pending,destroyed.demoted,destroyed.failed,destroyed.protected,@@ -674,10 +755,20 @@ async def apply_operations(# Writes and proposals dropped unwritten: a forget reached what they# rest on after the pass read the graph."dropped_forgotten": dropped,- # Writes and proposals not written yet (queued, waiting for the- # graph's lock, or being written), and those whose write failed.+ # Writes and proposals not written yet: queued, waiting for the+ # graph's lock, or being written."pending_writes": pending,+ # Writes and proposals the worker did not make: their citation marker+ # could not be written (it fails closed) or the graph write raised."failed_writes": failed,+ # Writes made whose derivation record failed after the write: their+ # marker stays until reconciled (``graphiti/recall_reconcile.py``).+ "provenance_pending": provenance_pending,+ # Writes and proposals dropped before they were queued: they cited+ # nothing the pass read, or a fact of another scope.+ "uncited_writes_dropped": cited.uncited,+ # Their citations of a fact of another scope.+ "cross_scope_citations_dropped": cited.cross_scope,# Distinct facts an acknowledged write spared and the accounting read# found live; provisional when the accounting is incomplete."protected_demotions": destroyed.protected,diff --git a/autogpt_platform/backend/backend/copilot/dream/apply_citations_test.py b/autogpt_platform/backend/backend/copilot/dream/apply_citations_test.pyindex f77aea93c1..e9a6057391 100644--- a/autogpt_platform/backend/backend/copilot/dream/apply_citations_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/apply_citations_test.py@@ -1,9 +1,12 @@-"""apply hands every dream write what it rests on, for the ingestion worker-to check against forgets right before writing it-(``graphiti/recall_citations.py``), and reports the writes it dropped.+"""apply checks every dream write's citations against what its pass read,+drops and counts one left citing nothing, and hands the rest, with what they+cite, to the ingestion worker, which checks them against forgets right before+writing (``graphiti/recall_citations.py``) and records them+(``graphiti/recall_derivation.py``).-The live runs, a forget between the dream's read and its writes, are-``graphiti/recall_dream_citations_integration_test.py``.+The live runs are ``graphiti/recall_dream_citations_integration_test.py``+(a forget between the dream's read and its writes) and+``graphiti/recall_cascade_integration_test.py`` (a forget after them)."""from unittest.mock import AsyncMock@@ -11,20 +14,32 @@ from unittest.mock import AsyncMockimport pytestfrom backend.copilot.graphiti.ingest import IngestionCompletion+from backend.copilot.graphiti.migrations.legacy_citations import described_citationsfrom backend.copilot.graphiti.recall_citations import Citationsfrom backend.copilot.graphiti.scope import MemoryScopefrom . import apply as apply_mod+from .citations import UNSCOPED, source_description, validated_citationsfrom .schemas import ConsolidatedFact, DreamOperations, ProposedFinding_SCOPE = MemoryScope.for_user("u-1234567890ab")+_READ = {"known_fact_uuids": {"f1", "f2"}, "known_episode_uuids": {"ep1", "ep2"}}+++async def _worker_wrote_them(completion: IngestionCompletion, _: float) -> bool:+ completion.written = completion.registered+ return True+++_real_summary_writer = apply_mod._write_dream_summary_message@pytest.fixture(autouse=True)def enqueue(mocker) -> AsyncMock:+ """The queue, and a worker that writes every episode queued."""queued = AsyncMock(return_value=True)mocker.patch.object(apply_mod, "enqueue_episode", queued)- mocker.patch.object(apply_mod, "wait_for_ingestion", AsyncMock(return_value=True))+ mocker.patch.object(apply_mod, "wait_for_ingestion", _worker_wrote_them)mocker.patch.object(apply_mod, "_create_dream_session", AsyncMock(return_value="s"))mocker.patch.object(apply_mod, "_write_dream_summary_message", AsyncMock())return queued@@ -34,14 +49,61 @@ def _cited(enqueue: AsyncMock) -> list[Citations]:return [call.kwargs["citations"] for call in enqueue.await_args_list]+def _checked(+ facts: list[str], episodes: list[str], **known: set[str]+) -> Citations | None:+ """``validated_citations`` for a write in the default scope, every+ source unscoped."""+ return validated_citations(+ facts,+ episodes,+ scope=UNSCOPED,+ known_facts=known["known_facts"],+ known_episodes=known["known_episodes"],+ fact_scopes={},+ ).citations+++class TestValidatedCitations:+ def test_keeps_what_the_pass_read_in_the_models_order_once_each(self) -> None:+ cited = _checked(+ ["f2", "made-up", "f1", "f2"],+ ["ep2", "ep1", "ep2"],+ known_facts={"f1", "f2"},+ known_episodes={"ep1", "ep2"},+ )++ assert cited == Citations(fact_uuids=["f2", "f1"], episode_uuids=["ep2", "ep1"])++ def test_files_a_uuid_under_the_kind_the_pass_read_it_as(self) -> None:+ cited = _checked(["ep1"], ["f1"], known_facts={"f1"}, known_episodes={"ep1"})++ assert cited == Citations(fact_uuids=["f1"], episode_uuids=["ep1"])++ @pytest.mark.parametrize(+ "facts, episodes", [([], []), (["made-up"], ["also-made-up"])]+ )+ def test_nothing_the_pass_read_is_none(+ self, facts: list[str], episodes: list[str]+ ) -> None:+ cited = _checked(facts, episodes, known_facts={"f1"}, known_episodes={"ep1"})++ assert cited is None++@pytest.mark.asyncio-async def test_each_write_carries_what_it_cites(enqueue: AsyncMock) -> None:+async def test_each_write_carries_what_it_cites_of_what_its_pass_read(+ enqueue: AsyncMock,+) -> None:+ """A consolidation cites facts now, as a proposal does; a made-up uuid+ is dropped and no write carries a statement to compare."""ops = DreamOperations(writes=[ConsolidatedFact(content="Alice works on Atlas",confidence=0.9,- source_episode_uuids=["ep1"],+ source_episode_uuids=["ep1", "made-up"],+ source_fact_uuids=["f1"],)],proposals=[@@ -49,60 +111,140 @@ async def test_each_write_carries_what_it_cites(enqueue: AsyncMock) -> None:content="Alice may lead Atlas",confidence=0.5,rationale="implied",- source_fact_uuids=["f1"],+ source_fact_uuids=["f2", "made-up"],source_episode_uuids=["ep2"],)],)- await apply_mod.apply_operations(- _SCOPE, "p1", ops, known_fact_uuids={"f1", "f9"}, known_episode_uuids={"ep1"}- )+ stats = await apply_mod.apply_operations(_SCOPE, "p1", ops, **_READ)assert _cited(enqueue) == [- Citations(episode_uuids=["ep1"]),- Citations(fact_uuids=["f1"], episode_uuids=["ep2"]),+ Citations(fact_uuids=["f1"], episode_uuids=["ep1"]),+ Citations(fact_uuids=["f2"], episode_uuids=["ep2"]),]+ assert stats["uncited_writes_dropped"] == 0@pytest.mark.asyncio-async def test_a_write_citing_nothing_rests_on_everything_the_pass_read(+async def test_writes_citing_nothing_the_pass_read_are_dropped_and_counted(enqueue: AsyncMock,) -> None:ops = DreamOperations(- writes=[ConsolidatedFact(content="Alice works on Atlas", confidence=0.9)],- proposals=[- ProposedFinding(content="Alice leads", confidence=0.5, rationale="r")+ writes=[+ ConsolidatedFact(content="cites nothing", confidence=0.9),+ ConsolidatedFact(+ content="cites only made-up uuids",+ confidence=0.9,+ source_episode_uuids=["made-up"],+ source_fact_uuids=["also-made-up"],+ ),+ ConsolidatedFact(+ content="Alice works on Atlas",+ confidence=0.9,+ source_episode_uuids=["ep1"],+ ),],+ proposals=[ProposedFinding(content="a hunch", confidence=0.5, rationale="r")],)- await apply_mod.apply_operations(- _SCOPE, "p1", ops, known_fact_uuids={"f2", "f1"}, known_episode_uuids={"ep1"}- )+ stats = await apply_mod.apply_operations(_SCOPE, "p1", ops, **_READ)- assert _cited(enqueue) == [- Citations(- fact_uuids=["f1", "f2"],- episode_uuids=["ep1"],- statement="Alice works on Atlas",- ),- Citations(- fact_uuids=["f1", "f2"], episode_uuids=["ep1"], statement="Alice leads"- ),+ assert [call.kwargs["name"] for call in enqueue.await_args_list] == [+ "dream_p1_consolidate_002"]+ assert stats["consolidated_count"] == 1+ assert stats["proposal_count"] == 0+ assert stats["uncited_writes_dropped"] == 3+ snapshot = stats["snapshot"]+ assert [w.content for w in snapshot.writes] == ["Alice works on Atlas"]@pytest.mark.asyncio-async def test_without_what_the_pass_read_it_is_checked_by_its_statement(+async def test_a_pass_left_with_only_uncited_writes_creates_no_session(enqueue: AsyncMock,) -> None:- fact = ConsolidatedFact(content="Alice works on Atlas", confidence=0.9)+ ops = DreamOperations(+ writes=[ConsolidatedFact(content="cites nothing", confidence=0.9)],+ summary_for_user="I consolidated one fact.",+ )++ stats = await apply_mod.apply_operations(_SCOPE, "p1", ops, **_READ)++ enqueue.assert_not_awaited()+ apply_mod._create_dream_session.assert_not_awaited()+ assert "session_id" not in stats+ assert (stats["consolidated_count"], stats["uncited_writes_dropped"]) == (0, 1)- await apply_mod._write_consolidated_fact(- _SCOPE, "p1", 0, fact, "s", IngestionCompletion()++async def test_without_what_the_pass_read_every_write_is_dropped(+ enqueue: AsyncMock,+) -> None:+ ops = DreamOperations(+ writes=[+ ConsolidatedFact(+ content="Alice works on Atlas",+ confidence=0.9,+ source_episode_uuids=["ep1"],+ )+ ])- assert _cited(enqueue) == [Citations(statement="Alice works on Atlas")]+ stats = await apply_mod.apply_operations(_SCOPE, "p1", ops)++ enqueue.assert_not_awaited()+ assert stats["uncited_writes_dropped"] == 1+++async def test_the_snapshot_and_the_episode_description_carry_the_checked_citations(+ enqueue: AsyncMock,+) -> None:+ ops = DreamOperations(+ writes=[+ ConsolidatedFact(+ content="Alice works on Atlas",+ confidence=0.9,+ source_episode_uuids=["ep1", "made-up"],+ source_fact_uuids=["f1"],+ )+ ],+ proposals=[+ ProposedFinding(+ content="Alice may lead Atlas",+ confidence=0.5,+ rationale="she runs the standups",+ source_fact_uuids=["f2"],+ )+ ],+ )++ stats = await apply_mod.apply_operations(_SCOPE, "p1", ops, **_READ)++ write, proposal = stats["snapshot"].writes[0], stats["snapshot"].proposals[0]+ assert (write.source_fact_uuids, write.source_episode_uuids) == (["f1"], ["ep1"])+ assert (proposal.source_fact_uuids, proposal.source_episode_uuids) == (["f2"], [])+ descriptions = [c.kwargs["source_description"] for c in enqueue.await_args_list]+ assert descriptions == [+ "dream-pass consolidation",+ "dream-pass proposal; rationale=she runs the standups",+ ], "the marker and the records carry what it cites"+++def test_a_rationale_cannot_forge_a_citation_in_the_description() -> None:+ """Codex's probe: a rationale holding the delimiter and a key. Its+ ``;`` is written as ``,``, so the backfill's reader finds a key where+ the dream never puts one and attributes nothing."""+ description = source_description(+ "proposal", rationale="ordinary; src_facts=11111111-2222"+ )++ assert description == (+ "dream-pass proposal; rationale=ordinary, src_facts=11111111-2222"+ )+ read_back = described_citations(description)+ assert (read_back.facts, read_back.ambiguous) == ([], True)@pytest.mark.asyncio@@ -117,20 +259,56 @@ async def test_the_writes_the_worker_dropped_are_reported(mocker) -> None:mocker.patch.object(apply_mod, "wait_for_ingestion", worker_dropped_one)ops = DreamOperations(writes=[- ConsolidatedFact(content="one", confidence=0.9),- ConsolidatedFact(content="two", confidence=0.9),+ ConsolidatedFact(content="one", confidence=0.9, source_fact_uuids=["f1"]),+ ConsolidatedFact(content="two", confidence=0.9, source_fact_uuids=["f2"]),])- stats = await apply_mod.apply_operations(_SCOPE, "p1", ops)+ stats = await apply_mod.apply_operations(_SCOPE, "p1", ops, **_READ)assert stats["consolidated_count"] == 1assert stats["dropped_forgotten"] == 1assert stats["pending_writes"] == 0[email protected]("dropped", [0, 1])+async def test_a_write_dropped_for_a_forget_keeps_the_narrative_out(+ mocker, dropped: int+) -> None:+ """The narrative may restate a dropped write anywhere, in the model's+ own words: once one was dropped for resting on a forget, the session+ opens with the placeholder instead."""++ async def worker(completion: IngestionCompletion, _: float) -> bool:+ completion.dropped_forgotten += dropped+ completion.written = completion.registered - dropped+ return True++ narrative = AsyncMock()+ mocker.patch.object(apply_mod, "wait_for_ingestion", worker)+ mocker.patch(+ "backend.data.db_accessors.chat_db",+ return_value=mocker.MagicMock(add_dream_narrative=narrative),+ )+ mocker.patch.object(apply_mod, "_write_dream_summary_message", _real_summary_writer)+ said = "Alice works on Atlas, I noticed."+ ops = DreamOperations(+ writes=[+ ConsolidatedFact(content=said, confidence=0.9, source_fact_uuids=["f1"])+ ],+ summary_for_user=said,+ )++ await apply_mod.apply_operations(_SCOPE, "p1", ops, **_READ)++ session_id, pass_id, content, erased = narrative.await_args.args+ assert (session_id, pass_id, erased) == ("s", "p1", "[forgotten]")+ assert content == ("[forgotten]" if dropped else said)++@pytest.mark.asyncioasync def test_an_empty_pass_drops_nothing() -> None:stats = await apply_mod.apply_operations(_SCOPE, "p1", DreamOperations())- assert stats["dropped_forgotten"] == 0+ assert (stats["dropped_forgotten"], stats["uncited_writes_dropped"]) == (0, 0)diff --git a/autogpt_platform/backend/backend/copilot/dream/apply_fence_test.py b/autogpt_platform/backend/backend/copilot/dream/apply_fence_test.pyindex 60639d2143..ce98f248fb 100644--- a/autogpt_platform/backend/backend/copilot/dream/apply_fence_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/apply_fence_test.py@@ -141,7 +141,7 @@ class TestTheBatchRoute:assert newer.error is None and not newer.skippedassert [call.args[1] for call in apply.await_args_list] == [newer.pass_id]- error = f"expired: forced by an admin-triggered dream pass {newer.pass_id}"+ error = "expired: forced"await _assert_ended_without_applying(fake_dream_db, fake_dream_redis, error)assert fake_dream_db.rows["p1"]["status"] is DreamPassStatus.EXPIREDassert fake_dream_db.rows[newer.pass_id]["status"] is DreamPassStatus.COMPLETE@@ -221,7 +221,7 @@ class TestTheBatchRoute:assert 1.9 < loop.time() - started < 4assert newer.error is None and not newer.skippedassert [call.args[1] for call in apply.await_args_list] == [newer.pass_id]- error = f"expired: forced by an admin-triggered dream pass {newer.pass_id}"+ error = "expired: forced"await _assert_ended_without_applying(fake_dream_db, fake_dream_redis, error)assert _charged(charges) == ["consolidate", "recombine", "sanitize"]@@ -248,7 +248,7 @@ class TestTheBatchRoute:apply.assert_not_awaited()await _assert_ended_without_applying(- fake_dream_db, fake_dream_redis, "apply: dream lease could not be renewed"+ fake_dream_db, fake_dream_redis, "apply: lease_unrenewed")assert _LOCK_KEY not in fake_dream_redis.storediff --git a/autogpt_platform/backend/backend/copilot/dream/apply_test.py b/autogpt_platform/backend/backend/copilot/dream/apply_test.pyindex 73ed2dfd55..a6baf1d3e8 100644--- a/autogpt_platform/backend/backend/copilot/dream/apply_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/apply_test.py@@ -3,7 +3,7 @@ calls are produced for each operation type.These tests do NOT touch FalkorDB or Prisma. apply.py is a purefan-out: it builds MemoryEnvelopes for writes/proposals and delegates-to ``enqueue_episode`` / ``create_chat_session`` / ``add_chat_message``+to ``enqueue_episode`` / ``create_chat_session`` / ``add_dream_narrative``and, through its destructive stage (``demotions.py``), to``supersede_unless_recalled`` / ``invalidate_entity_direct_neighbors``.Each of those is mocked here.@@ -20,6 +20,7 @@ import pytestfrom backend.copilot.graphiti.guarded_writes import NeighbourWrites, WriteOutcomefrom backend.copilot.graphiti.ingest import IngestionCompletion+from backend.copilot.graphiti.recall_citations import Citationsfrom backend.copilot.graphiti.scope import MemoryScopefrom backend.data.db_accessors import orgs_db as real_orgs_db@@ -38,6 +39,15 @@ from .schemas import (ProposedFinding,)+# What these passes read. Every write and proposal below cites some of it:+# one citing nothing the pass read is dropped (``citations.py``).+_READ_FACT = "f1"+_READ_EPISODE = "ep-1"+_READ = {"known_fact_uuids": {_READ_FACT}, "known_episode_uuids": {_READ_EPISODE}}+_WRITE = ConsolidatedFact(+ content="A likes B", confidence=0.8, source_episode_uuids=[_READ_EPISODE]+)+def _bundle_with_known_facts(*uuids: str) -> DreamInput:return DreamInput(@@ -104,14 +114,14 @@ def _stub_boundaries(mocker):return_value=mocker.MagicMock(session_id="s1"))database.update_chat_session_title = AsyncMock(return_value=True)- database.add_chat_message = AsyncMock(return_value=None)+ database.add_dream_narrative = AsyncMock(return_value=None)mocker.patch("backend.data.db_accessors.chat_db", return_value=database)mocker.patch("backend.copilot.db.create_chat_session", database.create_chat_session)mocker.patch("backend.copilot.db.update_chat_session_title",database.update_chat_session_title,)- mocker.patch("backend.copilot.db.add_chat_message", database.add_chat_message)+ mocker.patch("backend.copilot.db.add_dream_narrative", database.add_dream_narrative)# _create_dream_session's tenant lookup. Unmocked it runs REAL Prisma# queries on this test's function-scoped event loop whenever an earlier# test already connected Prisma (its except swallows the failure when@@ -144,12 +154,17 @@ def _stub_boundaries(mocker):async def test_writes_become_active_envelopes():ops = DreamOperations(writes=[- ConsolidatedFact(content="A likes B", confidence=0.8, scope="real:global")+ ConsolidatedFact(+ content="A likes B",+ confidence=0.8,+ scope="real:global",+ source_episode_uuids=[_READ_EPISODE],+ )],summary_for_user="ok",)stats = await apply_mod.apply_operations(- scope=MemoryScope.for_user("u-1234567890ab"), pass_id="p-abc", ops=ops+ scope=MemoryScope.for_user("u-1234567890ab"), pass_id="p-abc", ops=ops, **_READ)assert stats["consolidated_count"] == 1@@ -184,7 +199,7 @@ async def test_proposals_become_tentative_envelopes():summary_for_user="ok",)await apply_mod.apply_operations(- scope=MemoryScope.for_user("u-x"), pass_id="p-2", ops=ops+ scope=MemoryScope.for_user("u-x"), pass_id="p-2", ops=ops, **_READ)# First call was the consolidate write (there were no writes, so first call IS the proposal)call_kwargs = apply_mod.enqueue_episode.await_args.kwargs@@ -431,7 +446,7 @@ async def test_empty_pass_creates_no_session_and_no_message():)copilot_db.create_chat_session.assert_not_awaited()- copilot_db.add_chat_message.assert_not_awaited()+ copilot_db.add_dream_narrative.assert_not_awaited()apply_mod.enqueue_episode.assert_not_awaited()assert stats.get("session_id") is Noneassert stats["consolidated_count"] == 0@@ -450,18 +465,21 @@ async def test_ops_with_empty_summary_still_create_session_with_placeholder():from backend.copilot import db as copilot_dbops = DreamOperations(- writes=[ConsolidatedFact(content="A likes B", confidence=0.8)],+ writes=[_WRITE],summary_for_user="",)stats = await apply_mod.apply_operations(- scope=MemoryScope.for_user("u-ph"), pass_id="p-ph", ops=ops+ scope=MemoryScope.for_user("u-ph"), pass_id="p-ph", ops=ops, **_READ)copilot_db.create_chat_session.assert_awaited_once()- copilot_db.add_chat_message.assert_awaited_once()- msg_kwargs = copilot_db.add_chat_message.await_args.kwargs- assert msg_kwargs["role"] == "assistant"- assert msg_kwargs["content"] == "Dream pass completed with no narrative output."+ copilot_db.add_dream_narrative.assert_awaited_once()+ session_id, pass_id, content, erased = (+ copilot_db.add_dream_narrative.await_args.args+ )+ assert (session_id, pass_id) == (stats["session_id"], "p-ph")+ assert content == "Dream pass completed with no narrative output."+ assert erased == "[forgotten]"assert isinstance(stats["session_id"], str) and stats["session_id"]@@ -473,11 +491,11 @@ async def test_dream_session_titled_with_utc_date():from backend.copilot import db as copilot_dbops = DreamOperations(- writes=[ConsolidatedFact(content="A likes B", confidence=0.8)],+ writes=[_WRITE],summary_for_user="ok",)stats = await apply_mod.apply_operations(- scope=MemoryScope.for_user("u-title"), pass_id="p-title", ops=ops+ scope=MemoryScope.for_user("u-title"), pass_id="p-title", ops=ops, **_READ)copilot_db.update_chat_session_title.assert_awaited_once()@@ -501,15 +519,15 @@ async def test_title_failure_does_not_abort_apply(mocker):AsyncMock(side_effect=ConnectionError("db blip")),)ops = DreamOperations(- writes=[ConsolidatedFact(content="A likes B", confidence=0.8)],+ writes=[_WRITE],summary_for_user="ok",)stats = await apply_mod.apply_operations(- scope=MemoryScope.for_user("u-tf"), pass_id="p-tf", ops=ops+ scope=MemoryScope.for_user("u-tf"), pass_id="p-tf", ops=ops, **_READ)assert stats["consolidated_count"] == 1- copilot_db.add_chat_message.assert_awaited_once()+ copilot_db.add_dream_narrative.assert_awaited_once()@pytest.mark.asyncio@@ -527,9 +545,10 @@ async def test_expert_dream_session_and_write_keep_expert_scope(mocker):expert_scope,"p1",0,- ConsolidatedFact(content="A likes B", confidence=0.8),+ _WRITE,"session-1",IngestionCompletion(),+ citations=Citations(episode_uuids=[_READ_EPISODE]),)assert apply_mod.enqueue_episode.call_args.args[0] == expert_scope@@ -592,9 +611,10 @@ async def test_summary_written_after_memory_ops(mocker):scope=MemoryScope.for_user("u-1"),pass_id="p-1",ops=DreamOperations(- writes=[ConsolidatedFact(content="A likes B", confidence=0.8)],+ writes=[_WRITE],summary_for_user="done",),+ **_READ,)assert calls == ["write", "summary"], calls@@ -619,11 +639,11 @@ async def test_apply_waits_for_ingestion_drain_before_reporting_counts(mocker):apply_mod, "wait_for_ingestion", AsyncMock(return_value=True))ops = DreamOperations(- writes=[ConsolidatedFact(content="A likes B", confidence=0.8)],+ writes=[_WRITE],summary_for_user="ok",)stats = await apply_mod.apply_operations(- scope=MemoryScope.for_user("u-drain"), pass_id="p-drain", ops=ops+ scope=MemoryScope.for_user("u-drain"), pass_id="p-drain", ops=ops, **_READ)drain.assert_awaited_once()@@ -650,15 +670,12 @@ async def test_drain_timeout_reports_partial_visibility_not_failure(mocker, caplmocker.patch.object(apply_mod, "wait_for_ingestion", wrote_one_of_two)ops = DreamOperations(- writes=[- ConsolidatedFact(content="A likes B", confidence=0.8),- ConsolidatedFact(content="B likes C", confidence=0.8),- ],+ writes=[_WRITE, _WRITE.model_copy(update={"content": "B likes C"})],summary_for_user="ok",)with caplog.at_level(logging.WARNING, logger=apply_mod.logger.name):stats = await apply_mod.apply_operations(- scope=MemoryScope.for_user("u-slow"), pass_id="p-slow", ops=ops+ scope=MemoryScope.for_user("u-slow"), pass_id="p-slow", ops=ops, **_READ)assert stats["ingestion_drain_status"] is IngestionDrainStatus.timed_out@@ -686,18 +703,20 @@ async def test_counts_only_what_the_worker_wrote_of_each_kind(mocker):mocker.patch.object(apply_mod, "wait_for_ingestion", worker)ops = DreamOperations(- writes=[- ConsolidatedFact(content="A likes B", confidence=0.8),- ConsolidatedFact(content="B likes C", confidence=0.8),- ],+ writes=[_WRITE, _WRITE.model_copy(update={"content": "B likes C"})],proposals=[- ProposedFinding(content="A may like C", confidence=0.5, rationale="r")+ ProposedFinding(+ content="A may like C",+ confidence=0.5,+ rationale="r",+ source_episode_uuids=[_READ_EPISODE],+ )],summary_for_user="ok",)stats = await apply_mod.apply_operations(- scope=MemoryScope.for_user("u-kinds"), pass_id="p-kinds", ops=ops+ scope=MemoryScope.for_user("u-kinds"), pass_id="p-kinds", ops=ops, **_READ)assert stats["ingestion_drain_status"] is IngestionDrainStatus.timed_out@@ -715,7 +734,7 @@ async def test_zero_drain_timeout_skips_wait_and_reports_skipped(mocker, caplog)apply_mod, "wait_for_ingestion", AsyncMock(return_value=True))ops = DreamOperations(- writes=[ConsolidatedFact(content="A likes B", confidence=0.8)],+ writes=[_WRITE],summary_for_user="ok",)with caplog.at_level(logging.INFO, logger=apply_mod.logger.name):@@ -723,6 +742,7 @@ async def test_zero_drain_timeout_skips_wait_and_reports_skipped(mocker, caplog)scope=MemoryScope.for_user("u-batch"),pass_id="p-batch",ops=ops,+ **_READ,ingestion_drain_timeout=apply_mod.BATCH_INGESTION_DRAIN_TIMEOUT_SECONDS,)@@ -759,13 +779,14 @@ async def test_sync_path_renews_lock_before_drain(mocker):lock_handle = mocker.MagicMock()lock_handle.extend = AsyncMock(return_value=True)ops = DreamOperations(- writes=[ConsolidatedFact(content="A likes B", confidence=0.8)],+ writes=[_WRITE],summary_for_user="ok",)await apply_mod.apply_operations(scope=MemoryScope.for_user("u-lock"),pass_id="p-lock",ops=ops,+ **_READ,lock_handle=lock_handle,)@@ -784,7 +805,7 @@ async def test_failed_lock_renewal_aborts_before_drain_and_demotions(mocker):lock_handle = mocker.MagicMock()lock_handle.extend = AsyncMock(return_value=False)ops = DreamOperations(- writes=[ConsolidatedFact(content="A likes B", confidence=0.8)],+ writes=[_WRITE],summary_for_user="ok",)@@ -793,6 +814,7 @@ async def test_failed_lock_renewal_aborts_before_drain_and_demotions(mocker):scope=MemoryScope.for_user("u-lost"),pass_id="p-lost",ops=ops,+ **_READ,lock_handle=lock_handle,)@@ -815,10 +837,7 @@ async def test_a_lock_renewal_redis_does_not_answer_aborts_before_drain(mocker):lock_handle = mocker.MagicMock()lock_handle.extend = never_answers- ops = DreamOperations(- writes=[ConsolidatedFact(content="A likes B", confidence=0.8)],- summary_for_user="ok",- )+ ops = DreamOperations(writes=[_WRITE], summary_for_user="ok")with pytest.raises(DreamLockLostError):await asyncio.wait_for(@@ -827,6 +846,7 @@ async def test_a_lock_renewal_redis_does_not_answer_aborts_before_drain(mocker):pass_id="p-slow",ops=ops,lock_handle=lock_handle,+ **_READ,),5,)@@ -1013,7 +1033,11 @@ async def test_apply_operations_returns_snapshot_with_per_op_detail(mocker):summary_for_user="ok",)stats = await apply_mod.apply_operations(- scope=MemoryScope.for_user("u-snap"), pass_id="p-snap", ops=ops+ scope=MemoryScope.for_user("u-snap"),+ pass_id="p-snap",+ ops=ops,+ known_fact_uuids={"f1", "d1"},+ known_episode_uuids={"ep-1", "ep-2"},)snap = stats["snapshot"]diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_callbacks.py b/autogpt_platform/backend/backend/copilot/dream/batch_callbacks.pyindex 27db55b4d6..7fe0aa0f45 100644--- a/autogpt_platform/backend/backend/copilot/dream/batch_callbacks.py+++ b/autogpt_platform/backend/backend/copilot/dream/batch_callbacks.py@@ -60,9 +60,12 @@ from .batch_outcome import (from .batch_state import claim_apply_gate, content_for, read_state, write_phase_to_statefrom .batch_submit import PHASE_RESPONSE_MODELS, read_input_bundle, submit_phasefrom .cancel import end_batch_pass_if_stopped, pass_closed+from .citations import fact_scopesfrom .clamp import clamp_pass_operationsfrom .lease import ApplyLease, admit_batch_apply, renew_batch_leasefrom .llm import parse_json_with_prose_fallback+from .locks import DreamLockLostError+from .pass_errors import pass_errorfrom .provider_batch import anthropic_api_keyfrom .schemas import DreamOperations, DreamPhase, IngestionDrainStatusfrom .store import (@@ -84,10 +87,10 @@ logger = logging.getLogger(__name__)NAMESPACE = "dream_pass"# The error a batch pass closes with when the usage attribution it started-# under could not be established for one of its results (#15208): a fixed,-# content-free string. Its phases are charged only under the attribution+# under could not be established for one of its results (#15208): a fixed+# code (``pass_errors``). Its phases are charged only under the attribution# its row carries, if any (``batch_costs``).-UNATTRIBUTED_ERROR = "batch usage attribution unavailable"+UNATTRIBUTED_ERROR = pass_error("batch", "usage_attribution_unavailable")NEXT_PHASE: dict[DreamPhase, DreamPhase | None] = {"consolidate": "recombine",@@ -204,11 +207,11 @@ async def _handle_dream_batch_result("Dream batch handler missing user_id/pass_id/phase — payload=%s",payload,)- await end_dead_end(bp, "batch payload missing user_id/pass_id/phase")+ await end_dead_end(bp, pass_error("batch", "payload_incomplete"))returnif phase not in NEXT_PHASE:logger.warning("Dream batch handler unknown phase=%r", phase)- await end_dead_end(bp, f"unknown batch phase {phase!r}")+ await end_dead_end(bp, pass_error("batch", "unknown_phase"))returnawait _handle_guarded(bp, phase, rows)@@ -238,7 +241,7 @@ async def _handle_guarded("Dream batch input missing; refusing payload-only scope for pass=%s",bp.pass_id,)- await fail_pass(bp, "batch DreamInput missing; memory scope unavailable")+ await fail_pass(bp, pass_error("batch", "input_missing"))returnscoped = bp.model_copy(update={@@ -250,14 +253,14 @@ async def _handle_guarded(logger.error("Dream batch payload memory scope mismatch for pass=%s", bp.pass_id)- await fail_pass(scoped, "batch payload memory scope mismatch")+ await fail_pass(scoped, pass_error("batch", "scope_mismatch"))returnawait _handle_phase_result(bp, input_bundle, phase, rows)except Exception:logger.exception("Dream batch handler crashed for pass=%s phase=%s", bp.pass_id, phase)- await _fail_after_crash(scoped, f"{phase}: handler crashed")+ await _fail_after_crash(scoped, pass_error(phase, "crashed"))async def _fail_after_crash(bp: BatchPass, error: str) -> None:@@ -284,7 +287,7 @@ async def _handle_phase_result(unexpected raise (Redis blip, apply bug) is the crash guard's."""if not rows:logger.warning("Dream batch handler got empty rows for pass=%s", bp.pass_id)- await fail_pass(bp, f"{phase}: provider returned no rows")+ await fail_pass(bp, pass_error(phase, "no_rows"))return# Single-request-per-batch today; the first (and only) row is the# phase result. When we group batches in the future the BatchExecutor@@ -310,7 +313,10 @@ async def _landed_output(await write_phase_to_state(pass_id=bp.pass_id, phase=phase, row=row, model=bp.phase_models.get(phase))- await fail_pass(bp, f"{phase}: {row.error}")+ logger.warning(+ "Dream batch pass=%s phase=%s errored: %s", bp.pass_id, phase, row.error+ )+ await fail_pass(bp, pass_error(phase, "provider_error"))return None# Validate the row's content matches the phase's Pydantic schema# BEFORE persisting — corrupted content shouldn't pollute the@@ -326,7 +332,13 @@ async def _landed_output(await write_phase_to_state(pass_id=bp.pass_id, phase=phase, row=row, model=bp.phase_models.get(phase))- await fail_pass(bp, f"{phase}: invalid output shape — {type(exc).__name__}")+ logger.warning(+ "Dream batch pass=%s phase=%s answered off its schema: %s",+ bp.pass_id,+ phase,+ type(exc).__name__,+ )+ await fail_pass(bp, pass_error(phase, "schema_mismatch"))return Noneawait write_phase_to_state(pass_id=bp.pass_id,@@ -353,7 +365,7 @@ async def _chain_next_phase(state = await read_state(bp.pass_id)api_key = anthropic_api_key()if api_key is None:- await fail_pass(bp, f"{next_phase}: no Anthropic API key configured")+ await fail_pass(bp, pass_error(next_phase, "no_api_key"))returntry:submission = await submit_phase(@@ -367,13 +379,13 @@ async def _chain_next_phase(consolidated_json=content_for(state, "consolidate"),recombined_json=content_for(state, "recombine"),)- except Exception as exc:+ except Exception:logger.exception("Failed to submit %s phase for pass=%s — marking errored",next_phase,bp.pass_id,)- await fail_pass(bp, f"{next_phase}: submit failed: {type(exc).__name__}: {exc}")+ await fail_pass(bp, pass_error(next_phase, "submit_failed"))returnawait _advance_job(bp, next_phase, submission.provider_batch_id)await record_next_batch(bp.pass_id, next_phase, submission.provider_batch_id)@@ -413,7 +425,7 @@ async def _finalize_complete(bp: BatchPass, input_bundle: DreamInput) -> None:)except Exception:logger.exception("Failed to import dream apply for pass=%s", bp.pass_id)- await fail_pass(bp, "apply: import failed")+ await fail_pass(bp, pass_error("apply", "import_failed"))returnstate = await read_state(bp.pass_id)ops = await _terminal_ops(bp, state, input_bundle)@@ -432,15 +444,23 @@ async def _finalize_complete(bp: BatchPass, input_bundle: DreamInput) -> None:ops,known_fact_uuids=input_bundle.known_fact_uuids,known_episode_uuids=input_bundle.known_episode_uuids,+ fact_scopes=fact_scopes(input_bundle),ingestion_drain_timeout=BATCH_INGESTION_DRAIN_TIMEOUT_SECONDS,lease=lease,)- except Exception as exc:+ except DreamLockLostError:+ logger.exception(+ "apply_operations lost the dream lock for batch pass=%s — marking errored",+ bp.pass_id,+ )+ await fail_pass(bp, pass_error("apply", "lock_lost"))+ return+ except Exception:logger.exception("apply_operations crashed for batch pass=%s — marking errored",bp.pass_id,)- await fail_pass(bp, f"apply: {type(exc).__name__}: {exc}")+ await fail_pass(bp, pass_error("apply", "crashed"))returnawait _finish_applied(bp, state, apply_stats, ops, drain_status_from_stats(apply_stats)@@ -463,9 +483,7 @@ async def _claim_apply(bp: BatchPass, ops: DreamOperations) -> ApplyLease | Noneif gate == "error":# We cannot tell first-vs-duplicate apart, and "complete with no# writes" would silently drop the dream.- await fail_pass(- bp, "apply: gate unavailable (redis) — cannot guarantee at-most-once"- )+ await fail_pass(bp, pass_error("apply", "gate_unavailable"))return Noneif gate == "duplicate":await finish_duplicate(bp, ops)@@ -480,12 +498,17 @@ async def _terminal_ops(malformed result has failed the pass."""sanitize_row = state.get("sanitize")if sanitize_row is None or not sanitize_row.get("content"):- await fail_pass(bp, "sanitize: missing terminal phase content")+ await fail_pass(bp, pass_error("sanitize", "missing_content"))return Nonetry:ops = DreamOperations.model_validate(json.loads(sanitize_row["content"]))except (json.JSONDecodeError, ValidationError) as exc:- await fail_pass(bp, f"sanitize: shape validation failed: {type(exc).__name__}")+ logger.warning(+ "Dream batch pass=%s sanitize answered off its schema: %s",+ bp.pass_id,+ type(exc).__name__,+ )+ await fail_pass(bp, pass_error("sanitize", "schema_mismatch"))return None# Enforce the same per-pass operation caps the sync path applies# before writing — the model can over-emit past the prompt's limits.diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_callbacks_test.py b/autogpt_platform/backend/backend/copilot/dream/batch_callbacks_test.pyindex 8794ccf8e0..7ddd766af1 100644--- a/autogpt_platform/backend/backend/copilot/dream/batch_callbacks_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/batch_callbacks_test.py@@ -228,10 +228,7 @@ class TestPhaseChaining:)apply.assert_not_awaited()- assert (- mark_errored.await_args.kwargs["error"]- == "batch DreamInput missing; memory scope unavailable"- )+ assert mark_errored.await_args.kwargs["error"] == "batch: input_missing"# No token anywhere (no bundle, no row): the pass's ownership of the# lock is unknown, so nothing is released; its TTL, or the reaper,# settles it.@@ -282,10 +279,7 @@ class TestPhaseChaining:)apply.assert_not_awaited()- assert (- mark_errored.await_args.kwargs["error"]- == "batch payload memory scope mismatch"- )+ assert mark_errored.await_args.kwargs["error"] == "batch: scope_mismatch"release_lock.assert_awaited_once_with(MemoryScope.for_expert("u1", "expert-1"), "tok-expert")@@ -342,10 +336,7 @@ class TestPhaseChaining:)submit_phase.assert_not_awaited()- assert (- mark_errored.await_args.kwargs["error"]- == "batch payload memory scope mismatch"- )+ assert mark_errored.await_args.kwargs["error"] == "batch: scope_mismatch"release_lock.assert_awaited_once_with(MemoryScope.for_expert("u1", "expert-1"), "tok-expert")@@ -454,7 +445,7 @@ class TestPhaseChaining:flips to complete."""from backend.copilot.dream.batch_state import write_phase_to_statefrom backend.copilot.dream.batch_submit import persist_input_bundle- from backend.copilot.dream.fetch import DreamInput+ from backend.copilot.dream.fetch import DreamInput, FactRow_, _, string_store = fake_redis# The orchestrator holds the dream lock while persisting the bundle;@@ -468,6 +459,19 @@ class TestPhaseChaining:group_id="user_u1",window_start=now,window_end=now,+ facts=[+ FactRow(+ uuid="fact-1",+ source="A",+ target="B",+ name="likes",+ fact="A likes B",+ scope="project:bread",+ confidence=0.7,+ status="active",+ created_at=None,+ )+ ],known_fact_uuids={"fact-1"},known_episode_uuids={"episode-1"},),@@ -510,6 +514,8 @@ class TestPhaseChaining:# for the clamp — apply must not re-read the bundle from Redis.assert apply.call_args.kwargs["known_fact_uuids"] == {"fact-1"}assert apply.call_args.kwargs["known_episode_uuids"] == {"episode-1"}+ # ...and each fact's scope, which a write's fact citations must share.+ assert apply.call_args.kwargs["fact_scopes"] == {"fact-1": "project:bread"}# The batch path must NOT run the 300s in-line ingestion drain: apply# executes inside this handler, which BatchExecutor.walk_once awaits# serially — a long drain would stall every other user's batch poll.@@ -799,7 +805,7 @@ class TestPhaseChaining:apply.assert_not_awaited()mark_complete.assert_not_awaited()mark_errored.assert_awaited_once()- assert "gate unavailable" in mark_errored.call_args.kwargs["error"]+ assert mark_errored.call_args.kwargs["error"] == "apply: gate_unavailable"class TestErrorPaths:@@ -820,7 +826,7 @@ class TestErrorPaths:final = await read_status(kind="dream_pass", job_id="j-dead")assert final is not Noneassert final.state == "errored"- assert "missing" in (final.error or "")+ assert final.error == "batch: payload_incomplete"@pytest.mark.asyncioasync def test_unknown_phase_marks_job_errored_not_stuck(self, fake_redis):@@ -841,7 +847,7 @@ class TestErrorPaths:final = await read_status(kind="dream_pass", job_id="j-odd")assert final is not Noneassert final.state == "errored"- assert "unknown batch phase" in (final.error or "")+ assert final.error == "batch: unknown_phase"@pytest.mark.asyncio@pytest.mark.parametrize("error", ["content moderation", "max_tokens"])@@ -864,7 +870,7 @@ class TestErrorPaths:],)mark_errored.assert_awaited_once()- assert error in mark_errored.call_args.kwargs["error"]+ assert mark_errored.call_args.kwargs["error"] == "consolidate: provider_error"record_cost.assert_not_awaited()redis, _, _ = fake_redisstored = json.loads(redis.hset.await_args.args[2])@@ -895,7 +901,7 @@ class TestErrorPaths:)submit_phase.assert_not_awaited()mark_errored.assert_awaited_once()- assert "invalid output shape" in mark_errored.call_args.kwargs["error"]+ assert mark_errored.call_args.kwargs["error"] == "consolidate: schema_mismatch"record_cost.assert_not_awaited()redis, _, _ = fake_redisstored = json.loads(redis.hset.await_args.args[2])@@ -1051,7 +1057,7 @@ class TestErrorPaths:)mark_errored.assert_awaited_once()- assert "handler crashed" in mark_errored.call_args.kwargs["error"]+ assert mark_errored.call_args.kwargs["error"] == "consolidate: crashed"# Released with the ownership token the input bundle carried.release_lock.assert_awaited_once_with(MemoryScope.for_user("u1"), "tok-u1")@@ -1454,7 +1460,7 @@ class TestDreamPassRecord:assert fake_dream_db.statuses("p1") == [DreamPassStatus.ERRORED]row = fake_dream_db.rows["p1"]- assert row["error"] == "recombine: provider down"+ assert row["error"] == "recombine: provider_error"# The errored phase used nothing billable; consolidate did.assert [p.phase for p in row["usage"].phases] == ["consolidate"]assert row["completed_at"] is not None@@ -1485,15 +1491,15 @@ class TestDreamPassRecord:DreamPassStatus.ERRORED,]row = fake_dream_db.rows["p1"]- assert row["error"] == "apply: RuntimeError: FalkorDB unreachable"+ assert row["error"] == "apply: crashed", "no exception text is recorded"assert row["operations"]["planned"] == DreamOperations(summary_for_user="ok")@pytest.mark.asyncio@pytest.mark.parametrize("payload_fix, error",[- ({"phase": "daydream"}, "unknown batch phase 'daydream'"),- ({"phase": None}, "batch payload missing user_id/pass_id/phase"),+ ({"phase": "daydream"}, "batch: unknown_phase"),+ ({"phase": None}, "batch: payload_incomplete"),],ids=["unknown_phase", "missing_phase"],)@@ -1697,7 +1703,7 @@ class TestDreamPassRecord:submit_phase.assert_not_awaited()row = fake_dream_db.rows["p1"]assert row["status"] is DreamPassStatus.ERRORED- assert row["error"] == "consolidate: handler crashed"+ assert row["error"] == "consolidate: crashed"assert row.get("usage") is Nonestatus = await job_status.read_status(kind="dream_pass", job_id="j1")assert status is not None and status.state == "errored"@@ -1863,12 +1869,14 @@ class TestDreamPassRecord:async def _cancel_p1() -> str:assert (await cancel_dream_pass("p1", user_id="u1", reason="testing")).cancelled- return "cancelled: testing"+ return "cancelled: by_admin"async def _expire_p1() -> str:- assert await write_stop("p1", expired("lease lapsed", not_updated_since=None))- return "expired: lease lapsed"+ assert await write_stop(+ "p1", expired("expired: lease_lapsed", not_updated_since=None)+ )+ return "expired: lease_lapsed"class TestAStoppedPass:@@ -2039,7 +2047,7 @@ class TestAStoppedPass:fake_dream_db,fake_dream_redis,lock_key,- "batch usage attribution unavailable",+ "batch: usage_attribution_unavailable",)assert fake_dream_db.rows["p1"]["status"] is DreamPassStatus.CANCELLED@@ -2137,7 +2145,7 @@ class TestAStoppedPass:DreamPassStatus.CANCELLED,]await self._assert_ended(- fake_dream_db, fake_dream_redis, lock_key, "cancelled: testing"+ fake_dream_db, fake_dream_redis, lock_key, "cancelled: by_admin")@pytest.mark.asyncio@@ -2258,7 +2266,7 @@ async def test_dispatch_invalid_attribution_closes_job_without_billing(assert string_store["dream:inflight:u1"] == "unrelated-token"row = fake_dream_db.rows["p1"]assert row["status"] is DreamPassStatus.ERRORED- assert row["error"] == "batch usage attribution unavailable"+ assert row["error"] == "batch: usage_attribution_unavailable"assert input_bundle_key("p1") not in string_storeassert "dream:batch:state:p1" not in state_storebill.assert_not_called()@@ -2321,7 +2329,7 @@ async def test_a_result_its_row_does_not_attribute_ends_the_pass(row = fake_dream_db.rows["p1"]assert row["status"] is DreamPassStatus.ERRORED- assert row["error"] == "batch usage attribution unavailable"+ assert row["error"] == "batch: usage_attribution_unavailable"assert [c.trial_id if c else None for c in seen] == ([] if row_attribution is None else ["trial-other"])diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_deliveries.py b/autogpt_platform/backend/backend/copilot/dream/batch_deliveries.pyindex eac53c923c..ada6693112 100644--- a/autogpt_platform/backend/backend/copilot/dream/batch_deliveries.py+++ b/autogpt_platform/backend/backend/copilot/dream/batch_deliveries.py@@ -42,6 +42,7 @@ from .batch_costs import recorded_usagefrom .batch_outcome import BatchPass, clean_up_after, fail_pass, mark_job_erroredfrom .batch_submit import read_input_bundlefrom .fetch import DreamInput+from .pass_errors import pass_errorfrom .pass_record import stop_errorfrom .provider_batch import provider_batch_stoppedfrom .schemas import DreamOperations, DreamPassResult, IngestionDrainStatus@@ -54,11 +55,9 @@ logger = logging.getLogger(__name__)DISPATCH_CHECK_READ_TIMEOUT_SECONDS = 3.0# How a pass whose last batch came back twice closes: the first delivery-# claimed apply and never finished.-DUPLICATE_ERROR = (- "apply: an earlier delivery claimed apply and never finished; closed "- "without applying again, and what it wrote may have landed"-)+# claimed apply and never finished, so it closes without applying again,+# and what that delivery wrote may have landed.+DUPLICATE_ERROR = pass_error("apply", "duplicate_delivery")async def should_dispatch(entry: PendingEntry) -> bool:@@ -83,9 +82,9 @@ async def drop_closed(entry: PendingEntry) -> None:reaper's, from the closed row."""bp = BatchPass.from_payload(entry.payload or {})row, owner = await _owner_of(bp)- reason = "pass already closed"+ reason = pass_error("batch", "already_closed")if row is not None:- reason = stop_error(row) or f"pass already {row.status.value.lower()}"+ reason = stop_error(row) or reasonlogger.info(f"Dream batch {entry.provider_batch_id} of pass {bp.pass_id} dropped "f"unpolled: {reason}"diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_deliveries_test.py b/autogpt_platform/backend/backend/copilot/dream/batch_deliveries_test.pyindex db936444f9..7f66d867a3 100644--- a/autogpt_platform/backend/backend/copilot/dream/batch_deliveries_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/batch_deliveries_test.py@@ -93,14 +93,17 @@ class TestShouldDispatch:assert input_bundle_key("p1") not in fake_dream_redis.storestatus = await job_status.read_status(kind="dream_pass", job_id="j1")assert status is not None- assert (status.state, status.error) == ("errored", "cancelled: testing")+ assert (status.state, status.error) == ("errored", "cancelled: by_admin")# The job keeps what the landed phase used, which the closed row# refused.assert status.result is not Nonephases = status.result["usage"]["phases"]assert [phase["phase"] for phase in phases] == ["consolidate"]row = fake_dream_db.rows["p1"]- assert (row["status"], row["error"]) == (DreamPassStatus.CANCELLED, "testing")+ assert (row["status"], row["error"]) == (+ DreamPassStatus.CANCELLED,+ "cancelled: by_admin",+ )assert row.get("usage") is None# Every step finished, so the drop cleared the mark the cancel set.assert (row["cleanup_pending_at"], row["lease_token"]) == (None, None)@@ -280,7 +283,7 @@ class TestADeadEnd:row = fake_dream_db.rows["p1"]assert (row["status"], row["error"]) == (DreamPassStatus.ERRORED,- "unknown batch phase 'daydream'",+ "batch: unknown_phase",)assert _charged(charges) == ["consolidate"]assert state_key("p1") not in fake_dream_redis.hashesdiff --git a/autogpt_platform/backend/backend/copilot/dream/batch_handoff.py b/autogpt_platform/backend/backend/copilot/dream/batch_handoff.pyindex 2c3fc7e55b..dc0e2f6e87 100644--- a/autogpt_platform/backend/backend/copilot/dream/batch_handoff.py+++ b/autogpt_platform/backend/backend/copilot/dream/batch_handoff.py@@ -31,6 +31,7 @@ from .batch_submit import persist_input_bundle, phase_models_for_config, submit_from .cancel import stopped_errorfrom .fetch import DreamInputfrom .locks import BATCH_LOCK_TTL_SECONDS, DreamLockHandle+from .pass_errors import pass_errorfrom .pass_record import stop_errorfrom .pass_run import DreamPassRunfrom .provider_batch import cancel_provider_batch@@ -40,10 +41,10 @@ from .store import read_pass, record_submitted, record_taken_backlogger = logging.getLogger(__name__)# How a pass ends whose row refused its submit and cannot say why.-REFUSED_SUBMIT_ERROR = "stopped: row refused the submitted write"+REFUSED_SUBMIT_ERROR = pass_error("submit", "row_refused")# How a pass ends whose submit write failed and whose row, read again, does# not hold its batch, or cannot be read.-UNRECORDED_SUBMIT_ERROR = "anthropic_batch: batch not on the pass's record; taken back"+UNRECORDED_SUBMIT_ERROR = pass_error("submit", "not_on_record")async def submit_dream_pass_batch(@@ -59,7 +60,7 @@ async def submit_dream_pass_batch(if not api_key:# Shouldn't happen — routing.py only picks anthropic_batch when# the key is present. Guard for type-narrowing + future safety.- return run.failure("anthropic_batch: no Anthropic API key (routing bug)")+ return run.failure(pass_error("submit", "no_api_key"))stopped = await stopped_error(run.pass_id)if stopped is not None:return run.failure(stopped)@@ -70,8 +71,12 @@ async def submit_dream_pass_batch(await persist_input_bundle(run.pass_id, input_bundle, lock_token=lock_handle.token)- except Exception as exc:- return run.failure(f"anthropic_batch: input persist failed: {exc}")+ except Exception:+ logger.warning(+ f"Dream pass {run.pass_id}: could not persist its input bundle",+ exc_info=True,+ )+ return run.failure(pass_error("submit", "input_persist_failed"))# An empty job id tells the callbacks there is no JobStatus row to update.try:@@ -84,8 +89,12 @@ async def submit_dream_pass_batch(api_key=api_key,input_bundle=input_bundle,)- except Exception as exc:- return run.failure(f"anthropic_batch: phase 1 submit failed: {exc}")+ except Exception:+ logger.warning(+ f"Dream pass {run.pass_id}: could not submit its first batch",+ exc_info=True,+ )+ return run.failure(pass_error("consolidate", "submit_failed"))return await _hand_off(run, lock_handle, submission.provider_batch_id, input_bundle)@@ -116,7 +125,11 @@ async def _hand_off(provider_batch_id,)if not await lock_handle.extend(BATCH_LOCK_TTL_SECONDS):- error = "anthropic_batch: dream lock lost before handoff — batch revoked"+ error = pass_error("submit", "lock_lost")+ logger.warning(+ f"Dream pass {run.pass_id}: lost its dream lock before the handoff; "+ f"taking batch {provider_batch_id} back"+ )await _take_back(run, lock_handle, provider_batch_id, error)return run.failure(error)stopped = await _record_submit(run, lock_handle, provider_batch_id, input_bundle)diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_handoff_test.py b/autogpt_platform/backend/backend/copilot/dream/batch_handoff_test.pyindex 88e958ca61..3935c37a4a 100644--- a/autogpt_platform/backend/backend/copilot/dream/batch_handoff_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/batch_handoff_test.py@@ -81,7 +81,7 @@ async def test_a_pass_cancelled_while_it_gathered_submits_nothing(provider.submit.assert_not_awaited()assert (result.error, result.execution_path) == (- "cancelled: testing",+ "cancelled: by_admin","anthropic_batch",)assert input_bundle_key(result.pass_id) not in fake_dream_redis.store@@ -107,7 +107,7 @@ async def test_a_pass_cancelled_while_it_submitted_takes_its_batch_back(provider.submit.assert_awaited_once()provider.revoke.assert_awaited_once_with("batch-1")provider.cancel.assert_awaited_once_with("batch-1")- assert result.error == "cancelled: testing"+ assert result.error == "cancelled: by_admin"assert input_bundle_key(result.pass_id) not in fake_dream_redis.storeassert _LOCK_KEY not in fake_dream_redis.store_assert_cancelled(fake_dream_db, result.pass_id)@@ -237,12 +237,11 @@ async def test_a_refused_submit_whose_row_cannot_be_read_is_not_handed_off(_assert_cancelled(fake_dream_db, result.pass_id)-async def test_a_pass_whose_row_is_gone_is_not_handed_off(+async def test_a_pass_whose_row_was_never_inserted_submits_nothing(fake_dream_db, fake_dream_redis, gather, provider, mocker):- """A row that was never inserted (or was deleted with its user or- expert) refuses the submit too: the pass has nothing that could stop it- later, so it stops here."""+ """A pass whose insert failed stops before it gathers: a hard forget+ could not find it to mark it, so it reads nothing and submits nothing."""mocker.patch.object(fake_dream_db,"create_dream_pass",@@ -251,6 +250,27 @@ async def test_a_pass_whose_row_is_gone_is_not_handed_off(result = await orchestrator_mod.execute_dream_pass("u")+ assert result.error == "pass: record_unconfirmed"+ gather.assert_not_awaited()+ provider.submit.assert_not_awaited()+ assert _LOCK_KEY not in fake_dream_redis.store+++async def test_a_pass_whose_row_was_deleted_is_not_handed_off(+ fake_dream_db, fake_dream_redis, gather, provider+):+ """A row deleted with its user or expert after the gather refuses the+ submit: the pass has nothing that could stop it later, so it stops+ here."""++ async def submitted_after_the_row_went(**_kwargs) -> MagicMock:+ fake_dream_db.rows.clear()+ return MagicMock(provider_batch_id="batch-1")++ provider.submit.side_effect = submitted_after_the_row_went++ result = await orchestrator_mod.execute_dream_pass("u")+assert result.error == batch_handoff_mod.REFUSED_SUBMIT_ERROR_assert_revoked(provider, fake_dream_redis, result.pass_id)@@ -273,7 +293,7 @@ def _assert_cancelled(fake_dream_db, pass_id: str) -> None:row = fake_dream_db.rows[pass_id]assert (row["status"], row["error"], row["cancel_generation"]) == (DreamPassStatus.CANCELLED,- "testing",+ "cancelled: by_admin",1,)diff --git a/autogpt_platform/backend/backend/copilot/dream/batch_outcome.py b/autogpt_platform/backend/backend/copilot/dream/batch_outcome.pyindex 73ae4b6116..8ff1ee4b51 100644--- a/autogpt_platform/backend/backend/copilot/dream/batch_outcome.py+++ b/autogpt_platform/backend/backend/copilot/dream/batch_outcome.py@@ -274,6 +274,11 @@ def _applied_result(dropped_forgotten=_stat_count(apply_stats, "dropped_forgotten"),pending_writes=_stat_count(apply_stats, "pending_writes"),failed_writes=_stat_count(apply_stats, "failed_writes"),+ provenance_pending=_stat_count(apply_stats, "provenance_pending"),+ uncited_writes_dropped=_stat_count(apply_stats, "uncited_writes_dropped"),+ cross_scope_citations_dropped=_stat_count(+ apply_stats, "cross_scope_citations_dropped"+ ),protected_demotions=_stat_count(apply_stats, "protected_demotions"),indeterminate_demotion_writes=_stat_count(apply_stats, "indeterminate_demotion_writes"diff --git a/autogpt_platform/backend/backend/copilot/dream/cancel.py b/autogpt_platform/backend/backend/copilot/dream/cancel.pyindex 1a09ed4b9b..4acbc611b3 100644--- a/autogpt_platform/backend/backend/copilot/dream/cancel.py+++ b/autogpt_platform/backend/backend/copilot/dream/cancel.py@@ -47,6 +47,7 @@ that cannot confirm its lock does not apply.import loggingfrom collections.abc import Awaitable+from typing import Literalfrom pydantic import BaseModel@@ -55,6 +56,7 @@ from backend.data.dream_pass_models import OPEN_STATUSES, DreamPassRecordfrom .batch_outcome import BatchPass, fail_passfrom .locks import DreamLockHandle+from .pass_errors import pass_errorfrom .pass_record import cancelled, stop_errorfrom .pass_run import DreamPassRun, PassEndedfrom .provider_batch import cancel_provider_batch@@ -63,7 +65,10 @@ from .store import read_dream_pass, read_open_passes, read_pass, write_stoplogger = logging.getLogger(__name__)# How a pass that lost its scope's lock before apply ends.-LOCK_LOST_ERROR = "apply: dream lock lost before apply"+LOCK_LOST_ERROR = pass_error("apply", "lock_lost")++# Who cancels a pass, as its row records it.+CancelCode = Literal["by_admin", "by_wipe"]class DreamPassCancel(BaseModel):@@ -76,16 +81,24 @@ class DreamPassCancel(BaseModel):async def cancel_dream_pass(- pass_id: str, *, user_id: str, reason: str+ pass_id: str,+ *,+ user_id: str,+ reason: str,+ by: CancelCode = "by_admin",) -> DreamPassCancel:"""Cancel *user_id*'s pass *pass_id* while it is open, then read its row- back; a batch it closed has its provider batch in flight cancelled too,+ back. The row records ``cancelled: <by>``; *reason* goes to the log only+ (a pass records only fixed codes, ``pass_errors.py``). A batch it closed+ has its provider batch in flight cancelled too,best-effort (the executor drops that batch at its next poll whatever theprovider says, see ``batch_deliveries.should_dispatch``). Unlike a pass'sown writes, a store call that fails or runs out of time raises: the calleris answering a request, or about to erase the memory the pass wouldwrite."""- closed = await write_stop(pass_id, cancelled(reason, owner_user_id=user_id))+ closed = await write_stop(+ pass_id, cancelled(pass_error("cancelled", by), owner_user_id=user_id)+ )record = await read_dream_pass(pass_id, user_id=user_id)if closed:logger.info(f"Dream pass {pass_id} cancelled: {reason}")@@ -95,7 +108,7 @@ async def cancel_dream_pass(async def cancel_open_passes(- scope: MemoryScope, *, user_id: str, reason: str+ scope: MemoryScope, *, user_id: str, reason: str, by: CancelCode = "by_wipe") -> list[DreamPassCancel]:"""Cancel every open pass of *scope*, one ``cancel_dream_pass`` each, forthe wipe to call before it erases the scope's memory. Raises when the@@ -106,7 +119,8 @@ async def cancel_open_passes(also wait for the scope's lock before it erases."""rows = await read_open_passes(scope)return [- await cancel_dream_pass(row.id, user_id=user_id, reason=reason) for row in rows+ await cancel_dream_pass(row.id, user_id=user_id, reason=reason, by=by)+ for row in rows]diff --git a/autogpt_platform/backend/backend/copilot/dream/cancel_test.py b/autogpt_platform/backend/backend/copilot/dream/cancel_test.pyindex c477a1aa4f..5081a2103c 100644--- a/autogpt_platform/backend/backend/copilot/dream/cancel_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/cancel_test.py@@ -32,7 +32,6 @@ from .cancel import (end_batch_pass_if_stopped,stop_if_stopped,)-from .pass_record import MAX_ERROR_CHARSfrom .pass_run import DreamPassRun, PassEndedfrom .schemas import PhaseUsage@@ -96,7 +95,7 @@ def _cancelled(pass_id: str = "p1", **overrides) -> DreamPassRecord:pass_id,status=DreamPassStatus.CANCELLED,cancel_generation=1,- error="testing",+ error="cancelled: by_admin",completed_at=_NOW,**overrides,)@@ -113,7 +112,7 @@ class TestCancelDreamPass:assert pass_id == "p1"assert (update.status, update.error, update.owner_user_id) == (DreamPassStatus.CANCELLED,- "testing",+ "cancelled: by_admin","u1",)assert update.bump_cancel_generation is True@@ -171,11 +170,17 @@ class TestCancelDreamPass:provider_cancel.assert_not_awaited()- async def test_a_long_reason_is_capped_like_any_error(self, db):- await cancel_dream_pass("p1", user_id="u1", reason="x" * (MAX_ERROR_CHARS * 2))+ async def test_the_reason_goes_to_the_log_and_the_row_records_a_code(+ self, db, caplog+ ):+ """A recorded error is a fixed code: an admin's words may quote the+ memory the pass was dreaming about."""+ with caplog.at_level(logging.INFO, logger=cancel.logger.name):+ await cancel_dream_pass("p1", user_id="u1", reason="Alice runs Atlas")_, update = db.update_dream_pass.await_args.args- assert update.error == "x" * MAX_ERROR_CHARS+ assert update.error == "cancelled: by_admin"+ assert "Alice runs Atlas" in caplog.text@pytest.mark.parametrize("stalled",@@ -220,7 +225,9 @@ class TestCancelOpenPasses:db.list_open_dream_passes.assert_awaited_once_with(_SCOPE.scope_key, limit=None)sent = [call.args for call in db.update_dream_pass.await_args_list]assert [pass_id for pass_id, _ in sent] == ["a", "b"]- assert {(u.error, u.owner_user_id) for _, u in sent} == {("wiped", "u1")}+ assert {(u.error, u.owner_user_id) for _, u in sent} == {+ ("cancelled: by_wipe", "u1")+ }async def test_a_scope_the_store_cannot_list_raises(self, db):db.list_open_dream_passes.side_effect = ConnectionError("db down")@@ -239,26 +246,36 @@ class TestTheSyncCheck:await stop_if_stopped(run)result = ended.value.result- assert result.error == "cancelled: testing"+ assert result.error == "cancelled: by_admin"assert result.skipped is Falseassert result.usage is not None and result.usage.phases == run.phasesdb.get_dream_pass.assert_awaited_once_with(run.pass_id)- async def test_an_expired_row_ends_the_pass_too(self, db):+ @pytest.mark.parametrize(+ "error, ends_with",+ [+ ("expired: stalled", "expired: stalled"),+ ("consolidate: reaped", "consolidate: reaped"),+ ("no progress since then; expired by dream pass p9", "expired: stopped"),+ (None, "expired: stopped"),+ ],+ ids=["code", "reaped", "not_a_code", "none"],+ )+ async def test_an_expired_row_ends_the_pass_too(self, db, error, ends_with):+ """With the stop's own code, or ``expired: stopped`` for an error+ that is not one."""run = DreamPassRun.begin("u1", "sync_baseline")db.get_dream_pass.return_value = _row(run.pass_id,status=DreamPassStatus.EXPIRED,cancel_generation=1,- error="no progress since then; expired by dream pass p9",+ error=error,)with pytest.raises(PassEnded) as ended:await stop_if_stopped(run)- assert ended.value.result.error == (- "expired: no progress since then; expired by dream pass p9"- )+ assert ended.value.result.error == ends_with@pytest.mark.parametrize("row",@@ -305,7 +322,7 @@ class TestTheBatchCheck:provider.assert_not_awaited()fail.assert_awaited_once_with(- _BATCH_PASS, "cancelled: testing", provider_batch_id="msgbatch_1"+ _BATCH_PASS, "cancelled: by_admin", provider_batch_id="msgbatch_1")async def test_a_stopped_row_without_a_batch_skips_the_provider(self, db, ends):@@ -316,7 +333,7 @@ class TestTheBatchCheck:provider.assert_not_awaited()fail.assert_awaited_once_with(- _BATCH_PASS, "cancelled: testing", provider_batch_id=None+ _BATCH_PASS, "cancelled: by_admin", provider_batch_id=None)async def test_a_pass_nobody_stopped_goes_on(self, db, ends):diff --git a/autogpt_platform/backend/backend/copilot/dream/citations.py b/autogpt_platform/backend/backend/copilot/dream/citations.pynew file mode 100644index 0000000000..a05d82c3fd--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/citations.py@@ -0,0 +1,114 @@+"""What a dream write rests on, checked against what its pass read.++Every consolidated fact and proposal must cite the facts and episodes it was+drawn from. apply keeps a citation only when it names a fact or an episode+the pass read (its input bundle), filed under the kind the bundle says it+is, in the model's order and once each; a write left citing nothing is+dropped before it is queued and counted in ``uncited_writes_dropped``. So a+made-up uuid never reaches the graph, and no dream write rests on nothing.++Scope binds the fact citations, as the prompts tell the model ("group by+scope", a finding stays in the scope of the facts it cites). A fact's scope+is its own, ``UNSCOPED`` (the default everywhere) when it names none. A+write citing a fact of another scope is dropped whole, not trimmed to its+own scope's citations: trimmed, it could restate the other fact and escape+that fact's forget. It is counted in ``uncited_writes_dropped`` too, and+each such citation in ``cross_scope_citations_dropped``. Episode citations+are not scoped: a chat turn is raw material that can hold facts of any+scope, a project fact consolidated from one is legitimate, and the cascade+reaches it through ``derived_from_episodes`` when a forget hides that turn.+A source in the right scope that the write does not truly rest on is not+caught: no deterministic check can tell.++What survives goes to the ingestion worker, which checks it against the+forgets made since the pass read the graph (``graphiti/recall_citations.py``)+and then records it on the dream's episode and on the facts only dream+episodes state (``graphiti/recall_derivation.py``). A later forget of+anything it cites retracts those facts (``graphiti/recall_cascade.py``). The+episode's free-text ``source_description`` no longer lists what it cites:+the marker and the records carry that, and only the derivation backfill+reads the lists older descriptions hold+(``graphiti/migrations/legacy_citations.py``).+"""++from collections.abc import Collection, Mapping, Sequence++from pydantic import BaseModel, ValidationError++from backend.copilot.graphiti.recall_citations import Citations++from .fetch import DreamInput++# The scope of a fact or an envelope that names none: ``MemoryFact``'s and+# ``MemoryEnvelope``'s default, and how the prompts list an unscoped fact.+UNSCOPED = "real:global"+++class CheckedCitations(BaseModel):+ """What apply keeps of a write's citations: ``None`` when the write is+ dropped, for citing nothing the pass read or a fact of another scope;+ ``cross_scope`` counts its citations of a fact of another scope."""++ citations: Citations | None = None+ cross_scope: int = 0+++def validated_citations(+ fact_uuids: Sequence[str],+ episode_uuids: Sequence[str],+ *,+ scope: str,+ known_facts: Collection[str],+ known_episodes: Collection[str],+ fact_scopes: Mapping[str, str],+) -> CheckedCitations:+ """The write's citations that name something its pass read, each under+ the kind the pass read it as; none at all when one names a fact of+ another scope than the write's ``scope``. ``fact_scopes`` gives each+ fact's scope, ``UNSCOPED`` when absent; an episode's is not checked."""+ cited = list(dict.fromkeys([*fact_uuids, *episode_uuids]))+ facts = [uuid for uuid in cited if uuid in known_facts]+ episodes = [uuid for uuid in cited if uuid in known_episodes]+ wanted = scope_key(scope)+ elsewhere = [uuid for uuid in facts if scope_key(fact_scopes.get(uuid)) != wanted]+ if elsewhere or not (facts or episodes):+ return CheckedCitations(cross_scope=len(elsewhere))+ return CheckedCitations(+ citations=Citations(fact_uuids=facts, episode_uuids=episodes)+ )+++def scope_key(scope: str | None) -> str:+ """``scope`` as the prompts list it: whitespace collapsed, ``UNSCOPED``+ when it names none."""+ return " ".join((scope or "").split()) or UNSCOPED+++def fact_scopes(input_bundle: DreamInput) -> dict[str, str]:+ """The scope of every fact a pass read, ``UNSCOPED`` when it names none."""+ return {fact.uuid: scope_key(fact.scope) for fact in input_bundle.facts}+++class _EnvelopeScope(BaseModel):+ scope: str | None = None+++def envelope_scope(content: str | None) -> str:+ """The scope a ``MemoryEnvelope`` body declares; ``UNSCOPED`` for any+ other episode (a chat turn is plain text). The derivation backfill reads+ an older dream write's own scope this way."""+ try:+ return scope_key(_EnvelopeScope.model_validate_json(content or "").scope)+ except ValidationError:+ return UNSCOPED+++def source_description(kind: str, *, rationale: str | None = None) -> str:+ """The dream episode's ``source_description``, for people reading the+ graph: ``dream-pass <kind>`` and the proposal's rationale, each ``;`` in+ it written as ``,`` so no part of it reads as a field of its own. What+ the write cites is not in it: the marker and the records carry that."""+ parts = [f"dream-pass {kind}"]+ if rationale:+ parts.append(f"rationale={rationale[:240].replace(';', ',')}")+ return "; ".join(parts)diff --git a/autogpt_platform/backend/backend/copilot/dream/citations_routes_test.py b/autogpt_platform/backend/backend/copilot/dream/citations_routes_test.pynew file mode 100644index 0000000000..9b53cca5de--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/citations_routes_test.py@@ -0,0 +1,331 @@+"""Writes dropped for what they cite, end to end on both routes: the+sanitizer lets through one consolidation citing a fact the pass read, one+citing nothing, one citing a fact of another scope beside that fact, and a+proposal citing only a uuid the pass never read. The first is queued with+what it cites; the other three are dropped before they reach the graph and+reach ``uncited_writes_dropped`` in the result, the durable record and the+admin job status, the third's citation of another scope+``cross_scope_citations_dropped`` too. The worker then drops the queued+write for resting on a forget, fails+another and makes a third whose record fails: ``dropped_forgotten``,+``failed_writes`` and ``provenance_pending`` report them in the same places+on the sync route, which waits for the worker, and not on the batch route,+which does not: there the queued write is counted pending, not written.+Only the LLM, the ingestion queue and worker, and the chat store are+stubbed; the rest runs on ``conftest.py``'s in-memory Redis and store."""++import asyncio+import json+from datetime import datetime, timedelta, timezone+from unittest.mock import AsyncMock, MagicMock++import pytest+from prisma.enums import (+ DreamPassPhase,+ DreamPassRoute,+ DreamPassStatus,+ DreamPassTrigger,+)++from backend.copilot.graphiti.ingest import IngestionCompletion+from backend.copilot.graphiti.recall_citations import Citations+from backend.copilot.graphiti.scope import MemoryScope+from backend.copilot.inference.complete import StructuredCompletion+from backend.copilot.inference.context import InferenceUsage, RouteDecision+from backend.data.dream_pass_models import DreamPassApplied, DreamPassDraft+from backend.executor.batch_executor import PendingEntry+from backend.executor.scheduler import execute_dream_pass_with_status+from backend.util.llm.providers import BatchResultRow++from . import apply as apply_mod+from . import batch_callbacks as batch_callbacks_mod+from . import orchestrator as orchestrator_mod+from .batch_callbacks import handle_dream_batch_result+from .batch_submit import persist_input_bundle+from .fetch import DreamInput, FactRow+from .job_status import read_status, write_initial_status+from .pass_record import dream_pass_result_from_row+from .schemas import (+ ConsolidatedFact,+ ConsolidationOutput,+ DreamOperations,+ DreamPassResult,+ ProposedFinding,+ RecombinationOutput,+)++_OPS = DreamOperations(+ writes=[+ ConsolidatedFact(+ content="Nick ships on Fridays",+ confidence=0.9,+ source_fact_uuids=["f-read"],+ ),+ ConsolidatedFact(content="Nick ships on Mondays", confidence=0.9),+ ConsolidatedFact(+ content="Project X releases go through staging first",+ confidence=0.9,+ source_fact_uuids=["f-read", "f-project"],+ ),+ ],+ proposals=[+ ProposedFinding(+ content="Nick prefers small releases",+ confidence=0.5,+ rationale="a hunch",+ source_fact_uuids=["f-never-read"],+ )+ ],+ summary_for_user="Consolidated a fact.",+)+++def _bundle(user_id: str) -> DreamInput:+ now = datetime.now(timezone.utc)+ fact = FactRow(+ uuid="f-read",+ source="Nick",+ target="Fridays",+ name="ships_on",+ fact="Nick ships on Fridays",+ scope="real:global",+ confidence=0.8,+ status="active",+ created_at="2026-01-01T00:00:00+00:00",+ )+ elsewhere = fact.model_copy(update={"uuid": "f-project", "scope": "project:x"})+ return DreamInput(+ user_id=user_id,+ group_id=MemoryScope.for_user(user_id).group_id,+ window_start=now - timedelta(days=14),+ window_end=now,+ facts=[fact, elsewhere],+ known_fact_uuids={fact.uuid, elsewhere.uuid},+ )+++async def _worker_reports(completion: IngestionCompletion, _: float) -> bool:+ """The worker's outcomes, one of each, as its counters carry them: a+ write dropped for resting on a forget, one it failed to make, one made+ (written) whose record is pending."""+ completion.dropped_forgotten += 1+ completion.failed += 1+ completion.written += 1+ completion.provenance_pending += 1+ return True+++def queued(mocker) -> AsyncMock:+ """The ingestion queue and its worker's outcomes; apply's chat store."""+ enqueue = AsyncMock(return_value=True)+ mocker.patch.object(apply_mod, "enqueue_episode", enqueue)+ mocker.patch.object(apply_mod, "wait_for_ingestion", _worker_reports)+ database = MagicMock()+ database.create_chat_session = AsyncMock()+ database.update_chat_session_title = AsyncMock()+ database.add_dream_narrative = AsyncMock()+ mocker.patch("backend.data.db_accessors.chat_db", return_value=database)+ mocker.patch(+ "backend.data.db_accessors.orgs_db",+ return_value=MagicMock(+ get_user_default_team=AsyncMock(return_value=(None, None))+ ),+ )+ return enqueue+++def _answer(value) -> StructuredCompletion:+ return StructuredCompletion(+ value=value, usage=InferenceUsage(model="m", payer="platform_allowance")+ )+++def _route(scope, job, *, config=None) -> RouteDecision:+ return RouteDecision(+ engine="provider_sync",+ auth_provider="platform",+ provider="open_router",+ model=f"{job.tier}-model",+ payer="platform_allowance",+ execution_path="sync_baseline",+ cost_log_provider="open_router",+ reason="test",+ )+++async def _job_result(job_id: str) -> DreamPassResult:+ status = await read_status(kind="dream_pass", job_id=job_id)+ assert status is not None and status.state == "complete"+ return DreamPassResult.model_validate(status.result)+++def _reported(result: DreamPassResult | DreamPassApplied) -> tuple[int, ...]:+ """Written, proposed, uncited, cross-scope citations, and the worker's+ outcomes: dropped as forgotten, failed, provenance pending, and those not+ written yet."""+ return (+ result.consolidated_count,+ result.proposal_count,+ result.uncited_writes_dropped,+ result.cross_scope_citations_dropped,+ result.dropped_forgotten,+ result.failed_writes,+ result.provenance_pending,+ result.pending_writes,+ )+++def _only_the_cited_write_was_queued(enqueue: AsyncMock) -> None:+ [call] = enqueue.await_args_list+ assert call.kwargs["citations"] == Citations(fact_uuids=["f-read"])+++def scheduler_loop(mocker):+ """The scheduler's shared event loop: one loop of the test's own that+ its ``run_async`` runs on."""+ loop = asyncio.new_event_loop()+ mocker.patch(+ "backend.executor.scheduler.run_async",+ side_effect=lambda coro, timeout=None: loop.run_until_complete(coro),+ )+ yield loop+ loop.close()+++def test_the_sync_route_reports_the_dropped_writes_everywhere(+ mocker, queued, fake_dream_db, scheduler_loop+) -> None:+ mocker.patch.object(orchestrator_mod, "resolve_route", side_effect=_route)+ mocker.patch.object(+ orchestrator_mod, "resolve_dream_execution_path", return_value="sync_baseline"+ )+ mocker.patch.object(+ orchestrator_mod, "is_feature_enabled", AsyncMock(return_value=False)+ )+ mocker.patch.object(+ orchestrator_mod, "check_dream_budget", AsyncMock(return_value=(True, None))+ )+ mocker.patch.object(+ orchestrator_mod,+ "gather_dream_input",+ AsyncMock(return_value=_bundle("u-sync")),+ )+ mocker.patch.object(+ orchestrator_mod,+ "record_phase_cost",+ AsyncMock(side_effect=lambda ctx, usage: usage),+ )+ mocker.patch.object(+ orchestrator_mod,+ "structured_complete",+ AsyncMock(+ side_effect=[+ _answer(ConsolidationOutput(facts=[])),+ _answer(RecombinationOutput(proposals=[])),+ _answer(_OPS),+ ]+ ),+ )+ scheduler_loop.run_until_complete(+ write_initial_status(kind="dream_pass", job_id="j-sync", user_id="u")+ )++ execute_dream_pass_with_status("u-sync", "j-sync")++ job = scheduler_loop.run_until_complete(_job_result("j-sync"))+ assert job.error is None+ assert _reported(job) == (1, 0, 3, 1, 1, 1, 1, 0)+ applied = fake_dream_db.rows[job.pass_id]["operations"]["applied"]+ assert _reported(applied) == (1, 0, 3, 1, 1, 1, 1, 0)+ record = dream_pass_result_from_row(fake_dream_db.record(job.pass_id))+ assert _reported(record) == (1, 0, 3, 1, 1, 1, 1, 0)+ _only_the_cited_write_was_queued(queued)+++def _entry(phase: str) -> PendingEntry:+ now = datetime.now(timezone.utc)+ custom_id = f"p-batch:{phase}"+ return PendingEntry(+ provider="anthropic",+ provider_batch_id=f"msgbatch_{phase}",+ callback_namespace="dream_pass",+ submitted_at=now,+ next_poll_at=now,+ payload={+ "user_id": "u-batch",+ "cost_context": {+ "user_id": "u-batch",+ "trial_id": None,+ "generation": None,+ },+ "pass_id": "p-batch",+ "job_id": "j-batch",+ "phase": phase,+ "phase_models": {+ "consolidate": "claude-sonnet-5",+ "recombine": "claude-opus-5-5",+ "sanitize": "claude-sonnet-5",+ },+ "custom_ids": [custom_id],+ "phase_for_custom_id": {custom_id: phase},+ },+ )+++def _row(phase: str, content: str) -> BatchResultRow:+ return BatchResultRow(+ custom_id=f"p-batch:{phase}", content=content, input_tokens=10, output_tokens=20+ )+++async def test_the_batch_route_reports_the_dropped_writes_everywhere(+ mocker, queued, fake_dream_db, fake_dream_redis+) -> None:+ scope = MemoryScope.for_user("u-batch")+ fake_dream_redis.store[scope.redis_key("dream_lock")] = "tok-batch"+ await persist_input_bundle("p-batch", _bundle("u-batch"), lock_token="tok-batch")+ fake_dream_db.seed(+ DreamPassDraft(+ id="p-batch",+ user_id="u-batch",+ scope_key=scope.scope_key,+ route=DreamPassRoute.ANTHROPIC_BATCH,+ trigger=DreamPassTrigger.CRON,+ status=DreamPassStatus.SUBMITTED,+ phase=DreamPassPhase.CONSOLIDATE,+ started_at=datetime.now(timezone.utc),+ )+ )+ await write_initial_status(kind="dream_pass", job_id="j-batch", user_id="u-batch")+ mocker.patch.object(+ batch_callbacks_mod,+ "submit_phase",+ AsyncMock(+ side_effect=[+ MagicMock(provider_batch_id="msgbatch_recombine"),+ MagicMock(provider_batch_id="msgbatch_sanitize"),+ ]+ ),+ )+ mocker.patch.object(batch_callbacks_mod, "anthropic_api_key", return_value="sk")+ mocker.patch(+ "backend.copilot.inference.record.persist_and_record_usage", AsyncMock()+ )++ for phase, content in (+ ("consolidate", '{"facts": []}'),+ ("recombine", '{"proposals": []}'),+ ("sanitize", json.dumps(_OPS.model_dump())),+ ):+ await handle_dream_batch_result(_entry(phase), [_row(phase, content)])++ row = fake_dream_db.rows["p-batch"]+ assert row["status"] is DreamPassStatus.COMPLETE+ assert _reported(row["operations"]["applied"]) == (0, 0, 3, 1, 0, 0, 0, 1)+ record = dream_pass_result_from_row(fake_dream_db.record("p-batch"))+ assert _reported(record) == (0, 0, 3, 1, 0, 0, 0, 1)+ assert _reported(await _job_result("j-batch")) == (0, 0, 3, 1, 0, 0, 0, 1)+ _only_the_cited_write_was_queued(queued)diff --git a/autogpt_platform/backend/backend/copilot/dream/citations_scope_test.py b/autogpt_platform/backend/backend/copilot/dream/citations_scope_test.pynew file mode 100644index 0000000000..73052902e2--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/citations_scope_test.py@@ -0,0 +1,220 @@+"""A dream write's fact citations must name facts of its own scope, and a+write citing a fact of another scope is dropped whole (``citations.py``):+the rule the prompts give the model ("group by scope", a finding stays in+the scope of the facts it cites), enforced by apply. Episode citations are+not scoped: a chat turn can hold facts of any scope. The live run is+``graphiti/recall_citation_scope_integration_test.py``.+"""++from datetime import datetime, timezone+from unittest.mock import AsyncMock++import pytest++from backend.copilot.graphiti.memory_model import MemoryEnvelope+from backend.copilot.graphiti.recall_citations import Citations+from backend.copilot.graphiti.scope import MemoryScope++from . import apply as apply_mod+from .citations import UNSCOPED, fact_scopes, scope_key, validated_citations+from .fetch import DreamInput, EpisodeRow, FactRow+from .schemas import ConsolidatedFact, DreamOperations, ProposedFinding++_SCOPE = MemoryScope.for_user("u-1234567890ab")+_SCOPES = {"f-global": UNSCOPED, "f-bread": "project:bread"}+_READ = {+ "known_fact_uuids": {"f-global", "f-bread"},+ "known_episode_uuids": {"ep-chat", "ep-stored"},+ "fact_scopes": _SCOPES,+}+++def _check(scope: str, *cited: str):+ return validated_citations(+ [uuid for uuid in cited if uuid.startswith("f")],+ [uuid for uuid in cited if uuid.startswith("ep")],+ scope=scope,+ known_facts={"f-global", "f-bread"},+ known_episodes={"ep-chat", "ep-stored"},+ fact_scopes=_SCOPES,+ )+++class TestValidatedCitations:+ def test_a_write_citing_a_fact_of_another_scope_is_dropped_whole(+ self,+ ) -> None:+ """Trimmed to its own scope's citations it could restate the other+ fact and escape that fact's forget."""+ checked = _check("project:bread", "f-bread", "f-global", "ep-chat")++ assert (checked.citations, checked.cross_scope) == (None, 1)++ def test_a_write_citing_only_another_scope_cites_nothing(self) -> None:+ """Codex's probe: a project conclusion citing a global fact."""+ checked = _check("project:unrelated", "f-global")++ assert (checked.citations, checked.cross_scope) == (None, 1)++ def test_an_episode_citation_is_not_scoped(self) -> None:+ """A chat turn holds facts of any scope."""+ checked = _check("project:bread", "f-bread", "ep-chat")++ assert checked.citations == Citations(+ fact_uuids=["f-bread"], episode_uuids=["ep-chat"]+ )+ assert checked.cross_scope == 0++ def test_a_project_write_citing_only_chat_turns_is_kept(self) -> None:+ checked = _check("project:unrelated", "ep-chat", "ep-stored")++ assert checked.citations == Citations(episode_uuids=["ep-chat", "ep-stored"])++ def test_an_unknown_uuid_is_not_counted_as_cross_scope(self) -> None:+ checked = _check("project:bread", "made-up")++ assert (checked.citations, checked.cross_scope) == (None, 0)++ def test_a_fact_with_no_scope_given_is_unscoped(self) -> None:+ checked = validated_citations(+ ["f1"],+ [],+ scope="real:global",+ known_facts={"f1"},+ known_episodes=set(),+ fact_scopes={},+ )++ assert checked.citations == Citations(fact_uuids=["f1"])++ def test_scopes_compare_as_the_prompts_list_them(self) -> None:+ assert scope_key(" project:bread\n") == "project:bread"+ assert scope_key(None) == scope_key("") == UNSCOPED+++def test_a_pass_reads_each_facts_scope() -> None:+ """A fact's own, unset read as unscoped. An episode's envelope scope is+ not read: episode citations are not scoped."""+ envelope = MemoryEnvelope(content="bread notes", scope="project:bread")+ bundle = DreamInput(+ user_id="u",+ group_id="g",+ window_start=datetime(2026, 9, 1, tzinfo=timezone.utc),+ window_end=datetime(2026, 9, 28, tzinfo=timezone.utc),+ facts=[_fact("f-bread", "project:bread"), _fact("f-old", None)],+ episodes=[_episode("ep-stored", envelope.model_dump_json())],+ )++ assert fact_scopes(bundle) == {"f-bread": "project:bread", "f-old": UNSCOPED}+++def enqueue(mocker) -> AsyncMock:+ queued = AsyncMock(return_value=True)+ mocker.patch.object(apply_mod, "enqueue_episode", queued)+ mocker.patch.object(apply_mod, "wait_for_ingestion", AsyncMock(return_value=True))+ mocker.patch.object(apply_mod, "_create_dream_session", AsyncMock(return_value="s"))+ mocker.patch.object(apply_mod, "_write_dream_summary_message", AsyncMock())+ return queued+++async def test_apply_drops_every_write_citing_a_fact_of_another_scope(+ enqueue: AsyncMock,+) -> None:+ ops = DreamOperations(+ writes=[+ ConsolidatedFact(+ content="Sunrise Bakery changes suppliers every quarter",+ scope="project:unrelated",+ confidence=0.9,+ source_fact_uuids=["f-global"],+ ),+ ConsolidatedFact(+ content="The bread project uses the new flour",+ scope="project:bread",+ confidence=0.9,+ source_fact_uuids=["f-bread", "f-global"],+ ),+ ConsolidatedFact(+ content="The bread project bakes on Fridays",+ scope="project:bread",+ confidence=0.9,+ source_fact_uuids=["f-bread"],+ source_episode_uuids=["ep-chat"],+ ),+ ],+ proposals=[+ ProposedFinding(+ content="The bread project may need a second mill",+ scope="project:bread",+ confidence=0.5,+ rationale="volume",+ source_episode_uuids=["ep-chat"],+ )+ ],+ )++ stats = await apply_mod.apply_operations(_SCOPE, "p1", ops, **_READ)++ assert [call.kwargs["citations"] for call in enqueue.await_args_list] == [+ Citations(fact_uuids=["f-bread"], episode_uuids=["ep-chat"]),+ Citations(episode_uuids=["ep-chat"]),+ ]+ assert (+ stats["uncited_writes_dropped"],+ stats["cross_scope_citations_dropped"],+ ) == (+ 2,+ 2,+ )+++async def test_an_empty_pass_still_counts_what_it_dropped(enqueue: AsyncMock) -> None:+ ops = DreamOperations(+ writes=[+ ConsolidatedFact(+ content="a project conclusion from a global fact",+ scope="project:unrelated",+ confidence=0.9,+ source_fact_uuids=["f-global"],+ )+ ]+ )++ stats = await apply_mod.apply_operations(_SCOPE, "p1", ops, **_READ)++ enqueue.assert_not_awaited()+ assert (+ stats["uncited_writes_dropped"],+ stats["cross_scope_citations_dropped"],+ ) == (+ 1,+ 1,+ )+++def _fact(uuid: str, scope: str | None) -> FactRow:+ return FactRow(+ uuid=uuid,+ source="Nick",+ target="Bread",+ name="bakes",+ fact="Nick bakes bread",+ scope=scope,+ confidence=0.8,+ status="active",+ created_at=None,+ )+++def _episode(uuid: str, content: str | None) -> EpisodeRow:+ return EpisodeRow(+ uuid=uuid,+ name="n",+ content=content,+ source_description=None,+ valid_at=None,+ created_at=None,+ )diff --git a/autogpt_platform/backend/backend/copilot/dream/conftest.py b/autogpt_platform/backend/backend/copilot/dream/conftest.pyindex 73088f201b..bd4601c976 100644--- a/autogpt_platform/backend/backend/copilot/dream/conftest.py+++ b/autogpt_platform/backend/backend/copilot/dream/conftest.py@@ -33,7 +33,10 @@ import pytest_asynciofrom backend.copilot import usage_activation_fixturesfrom backend.copilot.trial_cost_context import TrialCostContextfrom backend.data.dream_pass_models import (+ DREAM_TEXT_FIELDS,+ ERASED_TEXT,OPEN_STATUSES,+ UNKNOWN_READS,DreamPassDraft,DreamPassOperations,DreamPassRecord,@@ -41,6 +44,8 @@ from backend.data.dream_pass_models import (DreamPhaseOutputs,)+from .pass_errors import PASS_ERROR_STAGES, UNRECORDED, is_pass_error+@pytest_asyncio.fixture(scope="session", loop_scope="session")async def server():@@ -59,16 +64,33 @@ class FakeAsyncRedis:Covers the surface the dream code uses: ``get``/``set`` (with ``nx``,``xx`` and ``ex``), ``delete``, ``expire``, ``ttl``, ``exists``, ``incr``,- the hash commands, and ``eval`` for the two single-key lock scripts in- ``locks.py`` (compare-and-delete, compare-and-extend). TTLs are recorded,- not enforced.+ the hash commands, the set commands the reaper's sweep of pending dream+ records uses (``graphiti/provenance_pending.py``), and ``eval`` for the+ two single-key lock scripts in ``locks.py`` (compare-and-delete,+ compare-and-extend). TTLs are recorded, not enforced."""def __init__(self) -> None:self.store: dict[str, str] = {}self.hashes: dict[str, dict[str, str]] = {}+ self.sets: dict[str, set[str]] = {}self.ttls: dict[str, int] = {}+ async def sadd(self, name: str, *values: Any) -> int:+ bucket = self.sets.setdefault(name, set())+ added = {self._s(value) for value in values} - bucket+ bucket |= added+ return len(added)++ async def srem(self, name: str, *values: Any) -> int:+ bucket = self.sets.get(name, set())+ removed = {self._s(value) for value in values} & bucket+ bucket -= removed+ return len(removed)++ async def srandmember(self, name: str, number: int) -> list[str]:+ return sorted(self.sets.get(name, set()))[:number]+@staticmethoddef _s(value: Any) -> str:return value.decode() if isinstance(value, bytes) else str(value)@@ -197,9 +219,12 @@ class FakeDreamDb:stop bumps the cancel generation, ``clear`` empties its columns, and aterminal row, another user's row (for an update naming the owner) or arow written after the update's instant is not written; an update marked- ``closed_row`` writes a terminal row only. ``writes`` holds- every draft and update that was written, in order, as sent. ``fail``- makes every call raise, as an unreachable database would.+ ``closed_row`` writes a terminal row only. The read set only grows, and an+ error that is not one of the fixed codes is recorded as ``UNRECORDED``.+ An update that closes a row a hard forget marked (``erased_at``) blanks+ its text and error, as ``erase_dream_passes`` does to a closed one.+ ``writes`` holds every draft and update that was written, in order, as+ sent. ``fail`` makes every call raise, as an unreachable database would."""def __init__(self) -> None:@@ -228,10 +253,14 @@ class FakeDreamDb:row["status"] = update.statusif update.phase is not None and _order(update.phase) > _order(row["phase"]):row["phase"] = update.phase+ if update.read_uuids is not None:+ row["read_uuids"] = sorted({*row["read_uuids"], *update.read_uuids})for field, value in dict(update).items():if value is None or field in _NOT_WRITTEN_AS_IS:continue- if field in ("phase_outputs", "operations"):+ if field == "error":+ row["error"] = recorded_error(value)+ elif field in ("phase_outputs", "operations"):row[field].update({k: v for k, v in dict(value).items() if v is not None})@@ -242,6 +271,11 @@ class FakeDreamDb:row.update({column: None for column in update.clear})row["cancel_generation"] += int(update.bump_cancel_generation)row["updated_at"] = datetime.now(timezone.utc)+ closes = update.closed_row or (+ update.status is not None and update.status not in OPEN_STATUSES+ )+ if row.get("erased_at") is not None and closes:+ _blank_row(row)return Trueasync def get_dream_pass(self, pass_id: str) -> DreamPassRecord | None:@@ -265,6 +299,37 @@ class FakeDreamDb:if row["scope_key"] == scope_key and row["status"] in OPEN_STATUSES][:limit]+ async def erase_dream_passes(+ self, user_id: str, scope_key: str, *, reached: list[str], named: list[str]+ ) -> list[str]:+ """Mark the user's passes of the scope a hard forget reached (one+ that read a reached uuid or may have read anything, open, closed with+ a cleanup pending, named, or marked before), blank those marked now+ and closed, and keep every+ error of the scope only as a fixed code; the ids marked, now or+ before. ``data/dream_pass.ERASE_SQL``."""+ self._raise_if_down()+ marked: list[str] = []+ for pass_id, row in self.rows.items():+ if (row["user_id"], row["scope_key"]) != (user_id, scope_key):+ continue+ row["error"] = blank_error(row.get("error"))+ if not (+ row.get("erased_at") is not None+ or row["status"] in OPEN_STATUSES+ or row.get("cleanup_pending_at") is not None+ or pass_id in named+ or set(row["read_uuids"]) & set(reached)+ or UNKNOWN_READS in row["read_uuids"]+ ):+ continue+ marked.append(pass_id)+ if row.get("erased_at") is None:+ row["erased_at"] = datetime.now(timezone.utc)+ if row["status"] not in OPEN_STATUSES:+ _blank_row(row)+ return marked+async def list_expired_dream_passes(self,expired_before: datetime,@@ -352,6 +417,7 @@ class FakeDreamDb:**dict(draft),"phase_outputs": {},"operations": {},+ "read_uuids": [],"cancel_generation": 0,"created_at": now,"updated_at": now,@@ -394,6 +460,7 @@ _NOT_WRITTEN_AS_IS = frozenset("not_updated_since","clear","closed_row",+ "read_uuids",})@@ -412,6 +479,55 @@ def _writable(row: dict[str, Any], update: DreamPassUpdate) -> bool:)+def _blank_row(row: dict[str, Any]) -> None:+ """Blank the dream's text in *row*'s JSON columns, and its error unless+ it is a fixed code."""+ row["phase_outputs"], row["operations"] = blank_text(_json_columns(row))+ row["error"] = blank_error(row.get("error"))+++def _json_columns(row: dict[str, Any]) -> list[dict[str, Any]]:+ """*row*'s phase outputs and operations as the database keeps them."""+ return [+ DreamPhaseOutputs(**row["phase_outputs"]).model_dump(mode="json"),+ DreamPassOperations(**row["operations"]).model_dump(mode="json"),+ ]+++def blank_text(value: Any) -> Any:+ """*value* with every string under one of ``DREAM_TEXT_FIELDS`` reading+ ``ERASED_TEXT``, as the migration's ``dream_text_blanked`` does."""+ if isinstance(value, dict):+ return {+ key: (+ ERASED_TEXT+ if key in DREAM_TEXT_FIELDS and isinstance(item, str)+ else blank_text(item)+ )+ for key, item in value.items()+ }+ if isinstance(value, list):+ return [blank_text(item) for item in value]+ return value+++def blank_error(error: str | None) -> str | None:+ """*error* when it is one of the fixed codes, else ``ERASED_TEXT`` after+ its stage when that is one of the fixed stages, as the migration's+ ``dream_error_blanked`` does."""+ if error is None or is_pass_error(error):+ return error+ stage, separator, _ = error.partition(": ")+ if separator and stage in PASS_ERROR_STAGES:+ return f"{stage}: {ERASED_TEXT}"+ return ERASED_TEXT+++def recorded_error(error: str) -> str:+ """*error* as the store records it: a fixed code, else ``UNRECORDED``."""+ return error if is_pass_error(error) else UNRECORDED++def _due(row: dict[str, Any], due_before: datetime) -> bool:"""Whether a marked row's cleanup is due, as the cleanup scan decides."""lease = row.get("lease_expires_at")diff --git a/autogpt_platform/backend/backend/copilot/dream/dedup.py b/autogpt_platform/backend/backend/copilot/dream/dedup.pyindex 9a46d9cb27..37074ce7e8 100644--- a/autogpt_platform/backend/backend/copilot/dream/dedup.py+++ b/autogpt_platform/backend/backend/copilot/dream/dedup.py@@ -186,15 +186,17 @@ def dedupe_near_duplicate_writes(Greedy by content length (desc) so the most specific phrasing survives;output preserves the writes' original order. Writes are only comparedwithin the same ``scope`` (facts must never merge across scopes), and a- dropped write's ``source_episode_uuids`` are unioned into its cluster's- survivor so provenance is never lost. Returns (kept, dropped)."""+ dropped write's ``source_episode_uuids`` and ``source_fact_uuids`` are+ unioned into its cluster's survivor so provenance is never lost: a forget+ of anything a dropped duplicate cited still reaches the survivor.+ Returns (kept, dropped)."""by_len_desc = sorted(range(len(writes)), key=lambda i: len(writes[i].content), reverse=True)tokens = [_content_tokens(w.content) for w in writes]bigrams = [_word_bigrams(w.content) for w in writes]kept_idx: list[int] = []- absorbed: dict[int, list[str]] = {}+ absorbed: dict[int, list[ConsolidatedFact]] = {}for i in by_len_desc:survivor = next((@@ -209,20 +211,34 @@ def dedupe_near_duplicate_writes(kept_idx.append(i)absorbed[i] = []else:- absorbed[survivor].extend(writes[i].source_episode_uuids)+ absorbed[survivor].append(writes[i])kept = [_absorb_provenance(writes[i], absorbed[i]) for i in sorted(kept_idx)]return kept, len(writes) - len(kept)def _absorb_provenance(- write: ConsolidatedFact, absorbed_uuids: list[str]+ write: ConsolidatedFact, absorbed: list[ConsolidatedFact]) -> ConsolidatedFact:- """Union dropped near-duplicates' episode uuids into the survivor."""- extra = [- u for u in dict.fromkeys(absorbed_uuids) if u not in write.source_episode_uuids- ]- if not extra:+ """Union dropped near-duplicates' episode and fact uuids into the+ survivor, after its own."""+ episodes = list(+ dict.fromkeys(+ [+ *write.source_episode_uuids,+ *(uuid for dup in absorbed for uuid in dup.source_episode_uuids),+ ]+ )+ )+ facts = list(+ dict.fromkeys(+ [+ *write.source_fact_uuids,+ *(uuid for dup in absorbed for uuid in dup.source_fact_uuids),+ ]+ )+ )+ if (episodes, facts) == (write.source_episode_uuids, write.source_fact_uuids):return writereturn write.model_copy(- update={"source_episode_uuids": [*write.source_episode_uuids, *extra]}+ update={"source_episode_uuids": episodes, "source_fact_uuids": facts})diff --git a/autogpt_platform/backend/backend/copilot/dream/demotions_stateful_test.py b/autogpt_platform/backend/backend/copilot/dream/demotions_stateful_test.pyindex 681008f129..f74cc7fc95 100644--- a/autogpt_platform/backend/backend/copilot/dream/demotions_stateful_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/demotions_stateful_test.py@@ -341,7 +341,7 @@ def world(mocker) -> SimpleNamespace:database = MagicMock()database.create_chat_session = AsyncMock()database.update_chat_session_title = AsyncMock()- database.add_chat_message = AsyncMock()+ database.add_dream_narrative = AsyncMock()mocker.patch("backend.data.db_accessors.chat_db", return_value=database)mocker.patch("backend.data.db_accessors.orgs_db",diff --git a/autogpt_platform/backend/backend/copilot/dream/forget_copies.py b/autogpt_platform/backend/backend/copilot/dream/forget_copies.pynew file mode 100644index 0000000000..1f2d0010b6--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/forget_copies.py@@ -0,0 +1,86 @@+"""What a hard forget blanks outside the graph: the dream's copies, in+Postgres, of the text it derived ("accept names, blank the copies").++A hard forget erases, in the graph, the text the dream derived from what it+forgot (``graphiti/recall_erase.py``). The dream kept that text in Postgres+too: on its pass's ``DreamPass`` row (each phase's output, the operations+apply was handed and what apply reported, an error quoting the model) and in+the narrative its dream session opens with.++``blank_dream_copies`` marks with ``erasedAt``, never cleared, every pass of+the scope the forget reached, in one statement over all of the scope's+passes (``data/dream_pass.erase_dream_passes``): one that read a fact or+episode the forget reached (its read set, every one it gathered, recorded+before its first model call, ``DreamPass.readUuids``), one that may have+read anything (a pass from before read sets were recorded, whose read set+holds ``UNKNOWN_READS``), one that may still be writing (open, or closed+with its cleanup pending: a stop from outside closes the row while the pass+may still be applying), and one whose dream writes the forget erased (a+dream write's episode is named after its pass, ``apply._episode_name``; a+pass did not read its own writes). A marked pass that is closed has its+text blanked in that statement; an open one in the statement that closes+it, however it closes (``data/dream_pass_update.py``), the reaper's expiry+being the backstop for one that never ends. Then the narrative of every+pass marked (now or by an earlier forget, so repeating a forget whose+narrative blank failed finishes it) or named is blanked, and so is every+narrative whose session's copy of its pass's read set holds a reached uuid,+its pass's row aged out or not, and every narrative of the scope whose+session keeps no read set (written before sessions kept one) or one holding+``UNKNOWN_READS``; one written after the mark is written blank+(``copilot/db.add_dream_narrative``). Every pass of the scope keeps its+error only when it is one of the fixed codes (``pass_errors.py``).++Every sentence of a marked pass goes, each ``content``, ``rationale``,+``reason`` and ``summary_for_user`` reading ``recall.FORGOTTEN_FACT``, not+only those derived from the forgotten fact: the dream may restate anything+it read anywhere, in its own words, a summary citing nothing. The uuids,+scopes, counts and usage stay. A pass whose row has aged out+(``retention.py``) keeps only its narrative, which is blanked through its+pass id or its session's read set, or as one of its scope with no known read+set. Entity names and name embeddings are kept.+"""++import asyncio+import re+from collections.abc import Collection++from backend.copilot.graphiti.recall import FORGOTTEN_FACT+from backend.copilot.graphiti.scope import MemoryScope+from backend.data.db_accessors import chat_db++from .store import erase_passes++# How long blanking the narratives may take, the RPC included.+NARRATIVE_TIMEOUT_SECONDS = 10.0+# A dream write's episode name (``apply._episode_name``).+_DREAM_WRITE = re.compile(r"^dream_(?P<pass_id>.+)_(?:consolidate|recombine)_\d{3}$")+++async def blank_dream_copies(+ scope: MemoryScope, *, reached: Collection[str], dream_writes: Collection[str]+) -> list[str]:+ """Mark the passes of ``scope`` a hard forget reached (the uuids it+ ``reached``, the episode names of the ``dream_writes`` it erased),+ blanking what they keep now or as they close, and blank their+ narratives and those of passes that read what it reached. Returns the+ ids marked, now or before. Raises when a write fails or runs out of+ time; repeating it finishes what a failed one left."""+ named = sorted({pass_id for name in dream_writes if (pass_id := pass_of(name))})+ read = sorted(reached)+ marked = await erase_passes(scope, reached=read, named=named)+ async with asyncio.timeout(NARRATIVE_TIMEOUT_SECONDS):+ await chat_db().blank_dream_narratives(+ scope.owner_user_id,+ scope.expert_id,+ sorted({*marked, *named}),+ read,+ FORGOTTEN_FACT,+ )+ return marked+++def pass_of(episode_name: str) -> str | None:+ """The pass a dream write's episode is named after; None for any other+ episode."""+ match = _DREAM_WRITE.match(episode_name)+ return match.group("pass_id") if match else Nonediff --git a/autogpt_platform/backend/backend/copilot/dream/forget_copies_test.py b/autogpt_platform/backend/backend/copilot/dream/forget_copies_test.pynew file mode 100644index 0000000000..196e2496ff--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/forget_copies_test.py@@ -0,0 +1,548 @@+"""Unit tests for ``forget_copies``: which passes a hard forget marks and+what it blanks of the dream's copies in Postgres, now or as a marked pass+closes, against the in-memory pass record (``conftest.py``, which follows+the SQL's rules) and a mocked chat database.++The real statements are ``backend/data/dream_pass_blank_test.py`` (the pass+row and its transitions) and ``backend/copilot/db_dream_narrative_test.py``+(the narrative); the review's reproductions run end to end in+``graphiti/recall_forget_copies_integration_test.py``.+"""++import asyncio+from datetime import datetime, timezone+from typing import Any+from unittest.mock import AsyncMock, MagicMock++import pytest+from prisma.enums import (+ DreamPassPhase,+ DreamPassRoute,+ DreamPassStatus,+ DreamPassTrigger,+)+from pytest_mock import MockerFixture++from backend.copilot.graphiti.recall import FORGOTTEN_FACT+from backend.copilot.graphiti.scope import MemoryScope+from backend.data.dream_pass_models import (+ ERASED_TEXT,+ UNKNOWN_READS,+ DreamPassApplied,+ DreamPassDraft,+ DreamPassOperations,+ DreamPhaseOutputs,+)++from . import forget_copies+from .apply import _episode_name+from .conftest import FakeDreamDb+from .forget_copies import blank_dream_copies, pass_of+from .pass_record import cancelled, cleanup_finished, expired, failed+from .schemas import (+ ConsolidatedFact,+ ConsolidationOutput,+ DemotionSummary,+ DreamDemotion,+ DreamOperations,+ DreamOperationsSnapshot,+ DreamPassResult,+ ProposedFinding,+ RecombinationOutput,+ WriteSummary,+)+from .store import record_batch_complete, record_sync_outcome, write_stop++_SCOPE = MemoryScope.for_user("u1")+_NOW = datetime.now(timezone.utc)+_FORGOTTEN = "fact-forgotten"+_SENTENCE = "Alice runs the Atlas launch"+++def chat(mocker: MockerFixture) -> MagicMock:+ database = MagicMock()+ database.blank_dream_narratives = AsyncMock(return_value=1)+ mocker.patch.object(forget_copies, "chat_db", return_value=database)+ return database+++def _seed(+ fake: FakeDreamDb,+ pass_id: str,+ *,+ cites: str = _FORGOTTEN,+ status: DreamPassStatus = DreamPassStatus.COMPLETE,+ scope: MemoryScope = _SCOPE,+ **columns: Any,+) -> None:+ """A pass that read ``cites`` and whose every phase consolidated+ ``_SENTENCE`` citing it, proposed a finding on it and demoted another+ fact."""+ columns.setdefault("read_uuids", [cites])+ write = ConsolidatedFact(+ content=_SENTENCE, confidence=0.8, source_fact_uuids=[cites]+ )+ proposal = ProposedFinding(+ content="Atlas ships this quarter",+ confidence=0.6,+ rationale=f"{_SENTENCE}, so it ships",+ source_fact_uuids=[cites],+ )+ demotion = DreamDemotion(edge_uuid="fact-other", reason=f"stale: {_SENTENCE}")+ ops = DreamOperations(+ writes=[write],+ proposals=[proposal],+ demotions=[demotion],+ summary_for_user=f"I noticed {_SENTENCE}.",+ )+ applied = DreamPassApplied(+ consolidated_count=1,+ summary_for_user=f"I noticed {_SENTENCE}.",+ snapshot=DreamOperationsSnapshot(+ writes=[WriteSummary(content=_SENTENCE, source_fact_uuids=[cites])],+ demotions=[+ DemotionSummary(+ edge_uuid="fact-other",+ reason=f"stale: {_SENTENCE}",+ new_status="superseded",+ )+ ],+ ),+ )+ fake.seed(+ DreamPassDraft(+ id=pass_id,+ user_id=scope.owner_user_id,+ scope_key=scope.scope_key,+ route=DreamPassRoute.SYNC,+ trigger=DreamPassTrigger.CRON,+ status=status,+ phase=DreamPassPhase.SANITIZE,+ ),+ phase_outputs=DreamPhaseOutputs(+ consolidate=ConsolidationOutput(facts=[write]),+ recombine=RecombinationOutput(proposals=[proposal]),+ sanitize=ops,+ ).model_dump(mode="json"),+ operations=DreamPassOperations(planned=ops, applied=applied).model_dump(+ mode="json"+ ),+ **columns,+ )+++def _texts(value: Any) -> list[str]:+ """Every string under ``value`` (a row's JSON column)."""+ if isinstance(value, dict):+ return [text for item in value.values() for text in _texts(item)]+ if isinstance(value, list):+ return [text for item in value for text in _texts(item)]+ return [value] if isinstance(value, str) else []+++def _row_text(fake: FakeDreamDb, pass_id: str) -> list[str]:+ row = fake.rows[pass_id]+ return _texts(row["phase_outputs"]) + _texts(row["operations"])+++async def test_a_closed_pass_that_read_what_was_forgotten_is_blanked(+ fake_dream_db: FakeDreamDb, chat: MagicMock+) -> None:+ """Every sentence goes, in every phase's output, the plan and apply's+ report; the uuids, scope, counts and statuses stay."""+ _seed(fake_dream_db, "p1")++ marked = await blank_dream_copies(_SCOPE, reached={_FORGOTTEN}, dream_writes=[])++ assert marked == ["p1"]+ assert not any(_SENTENCE in text for text in _row_text(fake_dream_db, "p1"))+ record = fake_dream_db.record("p1")+ assert record.erased_at is not None+ assert record.phase_outputs.consolidate is not None+ [write] = record.phase_outputs.consolidate.facts+ assert (write.content, write.source_fact_uuids) == (FORGOTTEN_FACT, [_FORGOTTEN])+ assert record.operations.applied is not None+ assert record.operations.applied.summary_for_user == FORGOTTEN_FACT+ assert record.operations.applied.consolidated_count == 1+ snapshot = record.operations.applied.snapshot+ assert snapshot is not None+ [demoted] = snapshot.demotions+ assert (demoted.edge_uuid, demoted.new_status) == ("fact-other", "superseded")+ chat.blank_dream_narratives.assert_awaited_once_with(+ "u1", None, ["p1"], [_FORGOTTEN], FORGOTTEN_FACT+ )+++async def test_a_summary_citing_nothing_is_blanked_by_what_its_pass_read(+ fake_dream_db: FakeDreamDb, chat: MagicMock+) -> None:+ """The round-2 review's uncited summary: no write of the pass cites the+ forgotten fact, but the pass read it, and that is what marks it."""+ _seed(fake_dream_db, "p1", cites="fact-unrelated", read_uuids=[_FORGOTTEN])++ assert await blank_dream_copies(_SCOPE, reached={_FORGOTTEN}, dream_writes=[]) == [+ "p1"+ ]+ assert not any(_SENTENCE in text for text in _row_text(fake_dream_db, "p1"))+++async def test_a_pass_from_before_read_sets_goes_with_any_forget_in_its_scope(+ fake_dream_db: FakeDreamDb, chat: MagicMock+) -> None:+ """The round-3 review's legacy pass: what it read is unknown, so a forget+ of anything in its scope marks it; another scope's is left."""+ _seed(fake_dream_db, "p1", cites="fact-unrelated", read_uuids=[UNKNOWN_READS])+ expert = MemoryScope.for_expert("u1", "e1")+ _seed(+ fake_dream_db,+ "p2",+ cites="fact-unrelated",+ read_uuids=[UNKNOWN_READS],+ scope=expert,+ )++ marked = await blank_dream_copies(_SCOPE, reached={_FORGOTTEN}, dream_writes=[])++ assert marked == ["p1"]+ assert not any(_SENTENCE in text for text in _row_text(fake_dream_db, "p1"))+ assert any(_SENTENCE in text for text in _row_text(fake_dream_db, "p2"))+++async def test_the_narratives_blank_is_held_to_the_forgets_scope(+ fake_dream_db: FakeDreamDb, chat: MagicMock+) -> None:+ """An expert's forget blanks the narratives of unknown provenance of that+ expert's scope only (``copilot/db.blank_dream_narratives``)."""+ expert = MemoryScope.for_expert("u1", "e1")++ await blank_dream_copies(expert, reached={_FORGOTTEN}, dream_writes=[])++ chat.blank_dream_narratives.assert_awaited_once_with(+ "u1", "e1", [], [_FORGOTTEN], FORGOTTEN_FACT+ )+++async def test_a_pass_whose_write_was_erased_is_blanked_by_its_name(+ fake_dream_db: FakeDreamDb, chat: MagicMock+) -> None:+ """A dream write the cascade erased names its pass, which may cite+ nothing else the forget reached."""+ _seed(fake_dream_db, "p1", cites="fact-unrelated")++ await blank_dream_copies(+ _SCOPE,+ reached={_FORGOTTEN},+ dream_writes=[_episode_name("p1", "consolidate", 1)],+ )++ assert not any(_SENTENCE in text for text in _row_text(fake_dream_db, "p1"))+ chat.blank_dream_narratives.assert_awaited_once_with(+ "u1", None, ["p1"], [_FORGOTTEN], FORGOTTEN_FACT+ )+++async def test_a_pass_the_forget_did_not_reach_is_left_alone(+ fake_dream_db: FakeDreamDb, chat: MagicMock+) -> None:+ """A pass that read none of it, and another scope's that did; the+ narratives' blank still looks for sessions whose pass read it."""+ _seed(fake_dream_db, "p1", cites="fact-unrelated")+ _seed(fake_dream_db, "p2", scope=MemoryScope.for_expert("u1", "e1"))+ before = {pass_id: _row_text(fake_dream_db, pass_id) for pass_id in ("p1", "p2")}++ marked = await blank_dream_copies(_SCOPE, reached={_FORGOTTEN}, dream_writes=[])++ assert marked == []+ assert {p: _row_text(fake_dream_db, p) for p in ("p1", "p2")} == before+ assert all(fake_dream_db.record(p).erased_at is None for p in ("p1", "p2"))+ chat.blank_dream_narratives.assert_awaited_once_with(+ "u1", None, [], [_FORGOTTEN], FORGOTTEN_FACT+ )+++async def test_a_pass_marked_before_is_returned_to_every_later_forget(+ fake_dream_db: FakeDreamDb, chat: MagicMock+) -> None:+ """The round-2 review's retry: a forget whose narrative blank failed is+ repeated after the pass closed. The pass is found by its mark, whatever+ the repeat reached, and its row is not written again."""+ _seed(fake_dream_db, "p1", status=DreamPassStatus.APPLYING)+ chat.blank_dream_narratives.side_effect = [OSError("down"), 1]+ with pytest.raises(OSError):+ await blank_dream_copies(_SCOPE, reached={_FORGOTTEN}, dream_writes=[])+ await record_sync_outcome(_result("p1", summary_for_user=f"I noticed {_SENTENCE}."))+ written = len(fake_dream_db.writes)+ closed = fake_dream_db.record("p1")++ assert await blank_dream_copies(+ _SCOPE, reached={"fact-other"}, dream_writes=[]+ ) == ["p1"]++ assert fake_dream_db.record("p1") == closed+ assert len(fake_dream_db.writes) == written+ assert chat.blank_dream_narratives.await_args.args == (+ "u1",+ None,+ ["p1"],+ ["fact-other"],+ FORGOTTEN_FACT,+ )+++ "status",+ [+ DreamPassStatus.QUEUED,+ DreamPassStatus.RUNNING,+ DreamPassStatus.SUBMITTED,+ DreamPassStatus.APPLYING,+ ],+)+async def test_every_running_pass_of_the_scope_is_marked_and_blanked_as_it_ends(+ fake_dream_db: FakeDreamDb, chat: MagicMock, status: DreamPassStatus+) -> None:+ """Its next phase reads its row, and one that gathered its input before+ the forget may not name what it read yet: it is marked whether or not its+ row names the forget, its text kept until it closes, and its narrative+ (one apply may have written already) blanked now with the closed pass's+ beside it."""+ _seed(fake_dream_db, "p-cites", status=status)+ _seed(fake_dream_db, "p-names-nothing", cites="fact-unrelated", status=status)+ _seed(fake_dream_db, "p-closed")+ before = _row_text(fake_dream_db, "p-cites")++ marked = await blank_dream_copies(+ _SCOPE,+ reached={_FORGOTTEN},+ dream_writes=[_episode_name("p-cites", "recombine", 2)],+ )++ assert sorted(marked) == ["p-cites", "p-closed", "p-names-nothing"]+ assert _row_text(fake_dream_db, "p-cites") == before+ assert not any(_SENTENCE in t for t in _row_text(fake_dream_db, "p-closed"))+ chat.blank_dream_narratives.assert_awaited_once_with(+ "u1",+ None,+ ["p-cites", "p-closed", "p-names-nothing"],+ [_FORGOTTEN],+ FORGOTTEN_FACT,+ )++ await record_sync_outcome(+ _result("p-names-nothing", summary_for_user=f"I noticed {_SENTENCE}.")+ )++ assert not any(_SENTENCE in t for t in _row_text(fake_dream_db, "p-names-nothing"))+ assert _row_text(fake_dream_db, "p-cites") == before, "still running"+++def _result(pass_id: str, **fields: Any) -> DreamPassResult:+ return DreamPassResult(user_id="u1", pass_id=pass_id, **fields)+++ "close",+ [+ "sync_outcome",+ "sync_failure",+ "batch_complete",+ "batch_failure",+ "cancel",+ "expiry",+ ],+)+async def test_every_way_a_marked_pass_closes_blanks_it(+ fake_dream_db: FakeDreamDb, chat: MagicMock, close: str+) -> None:+ """Whichever transition closes a marked pass blanks its text, what it+ writes as it closes included; an error that is not a fixed code is never+ recorded."""+ _seed(fake_dream_db, "p1", status=DreamPassStatus.APPLYING)+ await blank_dream_copies(_SCOPE, reached={_FORGOTTEN}, dream_writes=[])+ said = f"I noticed {_SENTENCE}."+ error = f"recombine: non-JSON — {_SENTENCE}"++ match close:+ case "sync_outcome":+ await record_sync_outcome(_result("p1", summary_for_user=said))+ case "sync_failure":+ await record_sync_outcome(_result("p1", error=error))+ case "batch_complete":+ await record_batch_complete(_result("p1", summary_for_user=said), None)+ case "batch_failure":+ assert await write_stop("p1", failed(error, None, _NOW, marked=True))+ case "cancel":+ assert await write_stop(+ "p1", cancelled("cancelled: by_admin", owner_user_id="u1")+ )+ case "expiry":+ assert await write_stop(+ "p1", expired("expired: forced", not_updated_since=None)+ )++ row = fake_dream_db.record("p1")+ assert row.status not in (DreamPassStatus.APPLYING,)+ assert not any(_SENTENCE in text for text in _row_text(fake_dream_db, "p1"))+ assert _SENTENCE not in (row.error or "")+++async def test_the_cleanups_end_blanks_a_marked_pass_again(+ fake_dream_db: FakeDreamDb, chat: MagicMock+) -> None:+ """The one write a closed row takes blanks it too, so a pass marked in+ the instant it closed is blanked by the cleanup after it at the latest."""+ _seed(fake_dream_db, "p1", status=DreamPassStatus.COMPLETE, cleanup_pending_at=_NOW)+ fake_dream_db.rows["p1"]["erased_at"] = _NOW++ assert await fake_dream_db.update_dream_pass("p1", cleanup_finished())++ assert not any(_SENTENCE in text for text in _row_text(fake_dream_db, "p1"))+++async def test_a_pass_with_no_mark_closes_as_it_did(+ fake_dream_db: FakeDreamDb, chat: MagicMock+) -> None:+ _seed(fake_dream_db, "p1", status=DreamPassStatus.APPLYING)+ said = f"I noticed {_SENTENCE}."++ await record_sync_outcome(_result("p1", summary_for_user=said))++ record = fake_dream_db.record("p1")+ assert record.operations.applied is not None+ assert record.operations.applied.summary_for_user == said+++async def test_a_closed_pass_with_its_cleanup_pending_is_marked_and_blanked(+ fake_dream_db: FakeDreamDb, chat: MagicMock+) -> None:+ """A stop from outside closes the row while the pass may still be+ applying, and may write the narrative after the forget: it is marked+ however little its row names."""+ _seed(+ fake_dream_db,+ "p1",+ cites="fact-unrelated",+ status=DreamPassStatus.CANCELLED,+ cleanup_pending_at=_NOW,+ )++ assert await blank_dream_copies(_SCOPE, reached={_FORGOTTEN}, dream_writes=[]) == [+ "p1"+ ]+ assert not any(_SENTENCE in text for text in _row_text(fake_dream_db, "p1"))+++ "error, kept",+ [+ (+ f"recombine: LLM returned non-JSON content — first 200 chars: {_SENTENCE}",+ f"recombine: {ERASED_TEXT}",+ ),+ ("consolidate: schema_mismatch", "consolidate: schema_mismatch"),+ ("apply: SecretToken123", f"apply: {ERASED_TEXT}"),+ (f"cancelled by an admin: {_SENTENCE}", ERASED_TEXT),+ ],+ ids=["quoted", "code", "not_a_code", "no_stage"],+)+async def test_every_error_of_the_scope_is_kept_only_as_a_fixed_code(+ fake_dream_db: FakeDreamDb, chat: MagicMock, error: str, kept: str+) -> None:+ """An error the store did not write (it records codes only) may quote+ anything, and no uuid finds it: every pass of the scope keeps its error+ only when it is one of the fixed codes, its stage when that is one,+ reached or not."""+ _seed(fake_dream_db, "p1", cites="fact-unrelated", status=DreamPassStatus.CANCELLED)+ fake_dream_db.rows["p1"]["error"] = error++ await blank_dream_copies(_SCOPE, reached={_FORGOTTEN}, dream_writes=[])++ assert fake_dream_db.record("p1").error == kept+ assert fake_dream_db.record("p1").erased_at is None, "not reached"+++async def test_every_pass_of_the_scope_is_looked_at(+ fake_dream_db: FakeDreamDb, chat: MagicMock+) -> None:+ """The review's 1,001 passes: the oldest, the one the forget reached,+ is behind a thousand newer ones, and no page or limit leaves it out."""+ _seed(fake_dream_db, "p-old")+ for index in range(1000):+ _seed(fake_dream_db, f"p-{index:04d}", cites="fact-unrelated")++ marked = await blank_dream_copies(_SCOPE, reached={_FORGOTTEN}, dream_writes=[])++ assert marked == ["p-old"]+ assert not any(_SENTENCE in text for text in _row_text(fake_dream_db, "p-old"))+++async def test_a_pass_whose_row_aged_out_keeps_only_its_narrative_to_blank(+ fake_dream_db: FakeDreamDb, chat: MagicMock+) -> None:+ await blank_dream_copies(+ _SCOPE,+ reached={_FORGOTTEN},+ dream_writes=[_episode_name("p-gone", "consolidate", 1)],+ )++ chat.blank_dream_narratives.assert_awaited_once_with(+ "u1", None, ["p-gone"], [_FORGOTTEN], FORGOTTEN_FACT+ )+++async def test_a_failed_write_raises(+ fake_dream_db: FakeDreamDb, chat: MagicMock+) -> None:+ fake_dream_db.fail = True++ with pytest.raises(ConnectionError):+ await blank_dream_copies(_SCOPE, reached={_FORGOTTEN}, dream_writes=[])+ chat.blank_dream_narratives.assert_not_awaited()+++async def test_a_narrative_write_that_hangs_runs_out_of_time(+ fake_dream_db: FakeDreamDb, chat: MagicMock, monkeypatch: pytest.MonkeyPatch+) -> None:+ monkeypatch.setattr(forget_copies, "NARRATIVE_TIMEOUT_SECONDS", 0.05)+ chat.blank_dream_narratives = AsyncMock(side_effect=_hang)+ _seed(fake_dream_db, "p1")++ with pytest.raises(TimeoutError):+ await blank_dream_copies(_SCOPE, reached={_FORGOTTEN}, dream_writes=[])+++async def _hang(*args: object) -> None:+ await asyncio.sleep(5)+++ "name, pass_id",+ [+ (_episode_name("p-1", "consolidate", 1), "p-1"),+ (_episode_name("0198c5a2-7f5e", "recombine", 12), "0198c5a2-7f5e"),+ ("conversation_s-1", None),+ ("finding_s-1", None),+ ("dream_p-1_sanitize_001", None),+ ("dream_p-1_consolidate_1", None),+ ],+)+def test_a_dream_writes_episode_names_its_pass(name: str, pass_id: str | None) -> None:+ assert pass_of(name) == pass_iddiff --git a/autogpt_platform/backend/backend/copilot/dream/guard.py b/autogpt_platform/backend/backend/copilot/dream/guard.pyindex 404d0ef968..e4702446a2 100644--- a/autogpt_platform/backend/backend/copilot/dream/guard.py+++ b/autogpt_platform/backend/backend/copilot/dream/guard.py@@ -51,6 +51,7 @@ from backend.data.dream_pass_models import OPEN_STATUSES, DreamPassRecordfrom backend.util.feature_flag import Flag, evaluate_feature_flagfrom .locks import DEFAULT_LOCK_TTL_SECONDS+from .pass_errors import PassErrorCode, pass_errorfrom .pass_record import expiredfrom .pass_run import DreamPassRun, PassEndedfrom .store import read_open_passes, read_pass, write_stop@@ -153,11 +154,15 @@ async def _expire(run: DreamPassRun, row: DreamPassRecord, *, stale: bool) -> bo"""Expire *row* and say whether it no longer holds the scope: expired now,closed by its own pass meanwhile, or unknown because the store did notanswer in time (the pass goes on)."""- reason = _expiry_reason(run, row, stale=stale)unchanged_since = row.updated_at if stale else None+ error = pass_error("expired", _expiry_code(row, stale=stale))try:- if await write_stop(row.id, expired(reason, not_updated_since=unchanged_since)):- logger.warning(f"Dream pass {run.pass_id} expired pass {row.id}: {reason}")+ if await write_stop(row.id, expired(error, not_updated_since=unchanged_since)):+ logger.warning(+ f"Dream pass {run.pass_id} expired pass {row.id} as {error} "+ f"({row.status.value.lower()}, lease until {row.lease_expires_at}, "+ f"last written {row.updated_at})"+ )return Truecurrent = await read_pass(row.id)except Exception:@@ -169,12 +174,8 @@ async def _expire(run: DreamPassRun, row: DreamPassRecord, *, stale: bool) -> boreturn current is None or current.status not in OPEN_STATUSES-def _expiry_reason(run: DreamPassRun, row: DreamPassRecord, *, stale: bool) -> str:- """What the expired row's error says happened."""+def _expiry_code(row: DreamPassRecord, *, stale: bool) -> PassErrorCode:+ """Why the row is expired, as its error records it."""if not stale:- return f"forced by an admin-triggered dream pass {run.pass_id}"- if row.lease_expires_at is not None:- lapsed = f"lease lapsed at {row.lease_expires_at.isoformat()}"- else:- lapsed = f"no progress since {row.updated_at.isoformat()}"- return f"{lapsed}; expired by dream pass {run.pass_id}"+ return "forced"+ return "lease_lapsed" if row.lease_expires_at is not None else "stalled"diff --git a/autogpt_platform/backend/backend/copilot/dream/guard_test.py b/autogpt_platform/backend/backend/copilot/dream/guard_test.pyindex 01c35ae34b..873b1a8ef7 100644--- a/autogpt_platform/backend/backend/copilot/dream/guard_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/guard_test.py@@ -146,10 +146,14 @@ class TestOpenPasses:assert update.not_updated_since == stale.updated_atassert update.owner_user_id is Noneassert update.completed_at is not None- assert update.error is not None- assert "no progress since" in update.error and run.pass_id in update.error+ assert update.error == "expired: stalled"[record] = [r for r in caplog.records if "expired pass other" in r.getMessage()]assert record.levelno == logging.WARNING+ message = record.getMessage()+ assert f"Dream pass {run.pass_id} expired pass other as expired: stalled" in (+ message+ ), "the code and both passes are logged"+ assert f"last written {stale.updated_at}" in message, "and when"async def test_a_lapsed_lease_is_stale_however_recently_written(self, db):leased = _row(@@ -164,7 +168,7 @@ class TestOpenPasses:_, update = db.update_dream_pass.await_args.argsassert update.status is DreamPassStatus.EXPIREDassert update.not_updated_since == leased.updated_at- assert update.error is not None and "lease lapsed" in update.error+ assert update.error == "expired: lease_lapsed"async def test_a_stale_pass_that_moved_before_its_expiry_still_blocks(self, db):"""The expiry is refused because the row was written after the guard@@ -200,7 +204,7 @@ class TestOpenPasses:assert update.status is DreamPassStatus.EXPIREDassert update.bump_cancel_generation is Trueassert update.not_updated_since is None- assert update.error == f"forced by an admin-triggered dream pass {run.pass_id}"+ assert update.error == "expired: forced"async def test_stale_passes_are_expired_before_a_fresh_one_skips(self, db):db.list_open_dream_passes.return_value = [diff --git a/autogpt_platform/backend/backend/copilot/dream/lease.py b/autogpt_platform/backend/backend/copilot/dream/lease.pyindex 2facbf7fcc..d8721979f5 100644--- a/autogpt_platform/backend/backend/copilot/dream/lease.py+++ b/autogpt_platform/backend/backend/copilot/dream/lease.py@@ -61,6 +61,7 @@ from .locks import (DreamLockHandle,extend_dream_lock,)+from .pass_errors import PassErrorStage, pass_errorfrom .pass_run import DreamPassRun, PassEndedfrom .store import record_lease@@ -86,18 +87,7 @@ class ApplyLease(BaseModel):return await _renewed(extend, self.pass_id) is True-def lock_lost_error(step: str) -> str:- """How a pass that lost its scope's lock before *step* ends."""- return f"{step}: dream lock lost before {step}"---def lease_unknown_error(step: str) -> str:- """How a pass ends whose lease could not be renewed before *step*, where- ownership unknown fails closed."""- return f"{step}: dream lease could not be renewed"---async def checkpoint(run: DreamPassRun, step: str) -> None:+async def checkpoint(run: DreamPassRun, step: PassErrorStage) -> None:"""The sync pass's check before each phase: its row shows no stop, thenits lease is renewed (``renew_sync_lease``); raises ``PassEnded`` wheneither ends the pass."""@@ -105,7 +95,7 @@ async def checkpoint(run: DreamPassRun, step: str) -> None:await renew_sync_lease(run, step)-async def renew_sync_lease(run: DreamPassRun, step: str) -> None:+async def renew_sync_lease(run: DreamPassRun, step: PassErrorStage) -> None:"""Renew the sync pass's lease before *step*; a renewal Redis cannotanswer goes on, warned (a phase writes nothing; apply's admission failsclosed). Raises ``PassEnded`` once the lock is no longer the pass's."""@@ -130,7 +120,7 @@ async def admit_sync_apply()-async def renew_batch_lease(bp: BatchPass, next_step: str) -> bool:+async def renew_batch_lease(bp: BatchPass, next_step: PassErrorStage) -> bool:"""On a callback whose phase has landed: stretch the batch pass's lockfor another ``BATCH_LOCK_TTL_SECONDS`` (a whole batch lifetime) andrecord it on the row. ``False`` once the lock turned out to be no longer@@ -148,7 +138,7 @@ async def renew_batch_lease(bp: BatchPass, next_step: str) -> bool:extend_dream_lock(scope, token, BATCH_LOCK_TTL_SECONDS), bp.pass_id)if renewed is False:- error = await stopped_error(bp.pass_id) or lock_lost_error(next_step)+ error = await stopped_error(bp.pass_id) or pass_error(next_step, "lock_lost")logger.warning(f"Dream batch pass {bp.pass_id} stops: {error}")await fail_pass(bp, error, holds_lock=False)return False@@ -191,7 +181,11 @@ async def admit_batch_apply(bp: BatchPass, lock_token: str | None) -> ApplyLeaseasync def _renew_sync(- run: DreamPassRun, lock: DreamLockHandle, step: str, *, fail_closed: bool+ run: DreamPassRun,+ lock: DreamLockHandle,+ step: PassErrorStage,+ *,+ fail_closed: bool,) -> None:"""Stretch the sync pass's lock another ``DEFAULT_LOCK_TTL_SECONDS`` andrecord it on its row. Raise ``PassEnded`` with the run's failure once@@ -208,9 +202,10 @@ async def _renew_sync(raise PassEnded(run.failure(error))-def _ended_error(step: str, renewed: bool | None) -> str:- """Why a pass's renewal ended it: its lock lost, or ownership unknown."""- return lock_lost_error(step) if renewed is False else lease_unknown_error(step)+def _ended_error(step: PassErrorStage, renewed: bool | None) -> str:+ """Why a pass's renewal ended it: its lock lost, or ownership unknown,+ which fails closed."""+ return pass_error(step, "lock_lost" if renewed is False else "lease_unrenewed")async def _renewed(extend: Awaitable[bool], pass_id: str) -> bool | None:diff --git a/autogpt_platform/backend/backend/copilot/dream/lease_admission_test.py b/autogpt_platform/backend/backend/copilot/dream/lease_admission_test.pyindex bae14ba3f8..42728ad700 100644--- a/autogpt_platform/backend/backend/copilot/dream/lease_admission_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/lease_admission_test.py@@ -46,7 +46,11 @@ from .schemas import (_SCOPE = MemoryScope.for_user("u1")_LOCK_KEY = _SCOPE.redis_key("dream_lock")-_FACT = ConsolidatedFact(content="Nick ships on Fridays", confidence=0.9)+# Cites the one episode each pass read (``_input``): a write citing nothing+# the pass read is dropped before it is written.+_FACT = ConsolidatedFact(+ content="Nick ships on Fridays", confidence=0.9, source_episode_uuids=["e1"]+)_OPS = DreamOperations(writes=[_FACT], summary_for_user="ok")_ANSWERS: tuple[BaseModel, ...] = (ConsolidationOutput(), RecombinationOutput(), _OPS)_BILLED = InferenceUsage(@@ -162,9 +166,7 @@ class TestTheSyncRoute:assert 1.9 < loop.time() - started < 4assert newer.error is None and not newer.skippedassert graph.writes == [newer.pass_id]- assert result.error == (- f"expired: forced by an admin-triggered dream pass {newer.pass_id}"- )+ assert result.error == "expired: forced"assert fake_dream_db.rows[result.pass_id]["status"] is DreamPassStatus.EXPIREDasync def test_a_lock_that_lapses_while_apply_starts_stops_its_writes(@@ -187,7 +189,7 @@ class TestTheSyncRoute:assert newer.error is Noneassert graph.writes == [newer.pass_id]- assert result.error is not None and result.error.startswith("Dream lock lost")+ assert result.error == "apply: lock_lost"class TestTheBatchRoute:@@ -214,7 +216,7 @@ class TestTheBatchRoute:assert graph.writes == [newer.pass_id]status = await job_status.read_status(kind="dream_pass", job_id="j1")assert status is not None and status.state == "errored"- assert (status.error or "").startswith("apply: DreamLockLostError")+ assert status.error == "apply: lock_lost"async def test_a_fence_answer_held_past_the_lease_still_writes_nothing(self, mocker, fake_dream_db, fake_dream_redis, graph@@ -273,7 +275,11 @@ async def _seed_batch_pass(fake_dream_db, fake_dream_redis) -> None:await persist_input_bundle("p1",DreamInput(- user_id="u1", group_id=_SCOPE.group_id, window_start=now, window_end=now+ user_id="u1",+ group_id=_SCOPE.group_id,+ window_start=now,+ window_end=now,+ known_episode_uuids={"e1"},),lock_token="our-token",)@@ -360,4 +366,5 @@ def _input() -> DreamInput:window_start=now,window_end=now,episodes=[episode],+ known_episode_uuids={episode.uuid},)diff --git a/autogpt_platform/backend/backend/copilot/dream/lease_batch_test.py b/autogpt_platform/backend/backend/copilot/dream/lease_batch_test.pyindex 3fbc660b00..c465b35378 100644--- a/autogpt_platform/backend/backend/copilot/dream/lease_batch_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/lease_batch_test.py@@ -119,7 +119,7 @@ class TestTheBatchLease:provider.assert_not_awaited()row = fake_dream_db.rows["p1"]- error = "recombine: dream lock lost before recombine"+ error = "recombine: lock_lost"assert (row["status"], row["error"]) == (DreamPassStatus.ERRORED, error)assert [p.phase for p in row["usage"].phases] == ["consolidate"]assert fake_dream_redis.store[_LOCK_KEY] == "newer-token"diff --git a/autogpt_platform/backend/backend/copilot/dream/lease_test.py b/autogpt_platform/backend/backend/copilot/dream/lease_test.pyindex 56b4d70f84..ad15b3dc82 100644--- a/autogpt_platform/backend/backend/copilot/dream/lease_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/lease_test.py@@ -157,7 +157,7 @@ class TestTheSyncLease:result = await orchestrator_mod.execute_dream_pass("u")- assert result.error == "recombine: dream lock lost before recombine"+ assert result.error == "recombine: lock_lost"assert phases.await_count == 1assert result.usage is not Noneassert [p.phase for p in result.usage.phases] == ["consolidate"]@@ -182,7 +182,7 @@ class TestTheSyncLease:with caplog.at_level(logging.WARNING):result = await orchestrator_mod.execute_dream_pass("u")- assert result.error == "apply: dream lease could not be renewed"+ assert result.error == "apply: lease_unrenewed"assert phases.await_count == 3assert result.usage is not None and len(result.usage.phases) == 3apply.assert_not_awaited()@@ -220,14 +220,16 @@ class TestTheSyncLease:its row, not just the lock it lost."""run = DreamPassRun.begin("u", "sync_baseline")fake_dream_db.seed(_draft(run.pass_id))- assert await write_stop(run.pass_id, expired("stale", not_updated_since=None))+ assert await write_stop(+ run.pass_id, expired("expired: forced", not_updated_since=None)+ )async with locks_mod.dream_lock(_SCOPE, token=run.lease_token) as handle:run.hold(handle)fake_dream_redis.store[_LOCK_KEY] = "newer-token"with pytest.raises(PassEnded) as ended:await renew_sync_lease(run, "apply")- assert ended.value.result.error == "expired: stale"+ assert ended.value.result.error == "expired: forced"assert fake_dream_redis.store[_LOCK_KEY] == "newer-token"@@ -266,8 +268,8 @@ class TestAClosingRow:[lambda: outcome(_result(), None, marked=True),lambda: failed("boom", None, datetime.now(timezone.utc), marked=True),- lambda: cancelled("testing", owner_user_id="u"),- lambda: expired("stale", not_updated_since=None),+ lambda: cancelled("cancelled: by_admin", owner_user_id="u"),+ lambda: expired("expired: forced", not_updated_since=None),],ids=["batch complete", "batch errored", "cancelled", "expired"],)@@ -336,7 +338,7 @@ class TestAClosingRow:)assert await fake_dream_db.update_dream_pass(- "p1", reaped("lapsed", not_updated_since=now)+ "p1", reaped("consolidate: reaped", not_updated_since=now))row = fake_dream_db.rows["p1"]assert (row["status"], row["lease_token"]) == (DreamPassStatus.EXPIRED, "tok")@@ -347,7 +349,10 @@ class TestAClosingRow:assert await fake_dream_db.update_dream_pass("p1", cleanup_finished())assert (row["cleanup_pending_at"], row["lease_token"]) == (None, None)- assert (row["status"], row["error"]) == (DreamPassStatus.EXPIRED, "lapsed")+ assert (row["status"], row["error"]) == (+ DreamPassStatus.EXPIRED,+ "consolidate: reaped",+ )def test_the_cleanup_write_goes_to_closed_rows_and_only_clears(self):with pytest.raises(ValidationError, match="a closed row only has columns"):diff --git a/autogpt_platform/backend/backend/copilot/dream/llm.py b/autogpt_platform/backend/backend/copilot/dream/llm.pyindex 3afe0240fc..f45cd72475 100644--- a/autogpt_platform/backend/backend/copilot/dream/llm.py+++ b/autogpt_platform/backend/backend/copilot/dream/llm.py@@ -14,19 +14,26 @@ JSON:Every failure is an ``InferenceError`` carrying the call's usage: theusage stays available for telemetry even when the caller absorbs the cost.+Its message and code never quote the model answer; only a debug log may show its start."""from __future__ import annotationsimport json+import loggingfrom typing import TypeVarfrom pydantic import BaseModel, ValidationErrorfrom backend.copilot.inference.context import InferenceError, InferenceUsage+logger = logging.getLogger(__name__)+T = TypeVar("T", bound=BaseModel)+# How much of an answer that cannot be parsed a debug log shows.+_EXCERPT_CHARS = 200+def parse_structured_output(content: str, response_model: type[T], usage: InferenceUsage@@ -38,15 +45,19 @@ def parse_structured_output(schema."""if not content:- raise InferenceError("LLM returned empty content", usage)+ raise InferenceError("LLM returned empty content", usage, code="empty_content")try:return response_model.model_validate(parse_json_with_prose_fallback(content))except InferenceError as exc:exc.usage = usageraiseexcept ValidationError as exc:+ logger.debug("LLM JSON did not match %s: %s", response_model.__name__, exc)raise InferenceError(- f"LLM JSON did not match {response_model.__name__}: {exc}", usage+ f"LLM JSON did not match {response_model.__name__} "+ f"({exc.error_count()} errors)",+ usage,+ code="schema_mismatch",) from exc@@ -62,8 +73,7 @@ def parse_json_with_prose_fallback(content: str) -> object:"Looking at the inputs, I need to..." prose-prefix case.Raises ``InferenceError`` if neither layer recovers a parseable- document, surfacing the first 200 chars of the offending content so- the caller can log enough to debug what the model emitted.+ document; a debug log shows the start of the offending content."""cleaned = _strip_json_code_fence(content)try:@@ -73,15 +83,19 @@ def parse_json_with_prose_fallback(content: str) -> object:extracted = _extract_first_json_object(cleaned)if extracted is None:- raise InferenceError(- f"LLM returned non-JSON content — first 200 chars: {content[:200]}"- )+ logger.debug("LLM returned non-JSON content: %s", content[:_EXCERPT_CHARS])+ raise InferenceError("LLM returned non-JSON content", code="non_json_content")try:return json.loads(extracted)except json.JSONDecodeError as exc:+ logger.debug(+ "LLM returned non-JSON content even after extraction (%s): %s",+ exc,+ content[:_EXCERPT_CHARS],+ )raise InferenceError(- f"LLM returned non-JSON content even after extraction: {exc} — "- f"first 200 chars: {content[:200]}"+ "LLM returned non-JSON content even after extraction",+ code="non_json_content",) from excdiff --git a/autogpt_platform/backend/backend/copilot/dream/llm_test.py b/autogpt_platform/backend/backend/copilot/dream/llm_test.pyindex a054dc592b..b31e38e5ac 100644--- a/autogpt_platform/backend/backend/copilot/dream/llm_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/llm_test.py@@ -130,13 +130,24 @@ def test_parse_structured_output_validates_fenced_json():@pytest.mark.parametrize(- "content,match",- [("", "empty"), ("prose only", "non-JSON"), ('{"other": 1}', "did not match")],+ "content,match,code",+ [+ ("", "empty", "empty_content"),+ ("prose only", "non-JSON", "non_json_content"),+ ("prose {only", "non-JSON", "non_json_content"),+ ('{"other": 1}', "did not match", "schema_mismatch"),+ ('{"content": ["said by the model"]}', "did not match", "schema_mismatch"),+ ],)-def test_parse_failures_carry_the_billed_usage(content: str, match: str):+def test_parse_failures_carry_the_billed_usage(content: str, match: str, code: str):+ """And a code, while neither the code nor the message quotes the answer,+ which a dream pass's record would keep (``forget_copies.py``)."""with pytest.raises(InferenceError, match=match) as exc_info:parse_structured_output(content, _Fact, _USAGE)assert exc_info.value.usage == _USAGE+ assert exc_info.value.code == code+ assert "only" not in str(exc_info.value)+ assert "said by the model" not in str(exc_info.value)# ---------------------------------------------------------------------------diff --git a/autogpt_platform/backend/backend/copilot/dream/orchestrator.py b/autogpt_platform/backend/backend/copilot/dream/orchestrator.pyindex 602d2531ca..402630c1f2 100644--- a/autogpt_platform/backend/backend/copilot/dream/orchestrator.py+++ b/autogpt_platform/backend/backend/copilot/dream/orchestrator.py@@ -52,6 +52,8 @@ from backend.util.feature_flag import Flag, is_feature_enabledfrom .apply import apply_operations, drain_status_from_statsfrom .batch_handoff import submit_dream_pass_batchfrom .billing import PhaseChargeError, check_dream_budget, record_phase_cost+from .cancel import stopped_error+from .citations import fact_scopesfrom .clamp import clamp_pass_operationsfrom .fetch import (DreamInput,@@ -66,8 +68,10 @@ from .locks import (LOCK_CHECK_TIMEOUT_SECONDS,DreamLockHandle,DreamLockHeld,+ DreamLockLostError,dream_lock,)+from .pass_errors import pass_errorfrom .pass_record import DreamTriggerfrom .pass_run import DreamPassRun, PassEndedfrom .phase_jobs import phase_job@@ -434,7 +438,12 @@ async def _execute_dream_pass_async(run = DreamPassRun.begin(user_id, await _route_for(user_id, config), force=force)try:scope = MemoryScope.build(user_id, expert_id)- await start_pass(run, scope, trigger=trigger)+ if not await start_pass(run, scope, trigger=trigger):+ # A hard forget finds only passes whose rows it can read: one+ # whose row may not be there must not read the graph.+ result = run.failure(pass_error("pass", "record_unconfirmed"))+ await record_sync_outcome(result)+ return resultasync with dream_lock(scope, token=run.lease_token) as handle:run.hold(handle)return await _run_locked(@@ -444,7 +453,7 @@ async def _execute_dream_pass_async(result = run.skipped("lock_held")except Exception as exc: # pragma: no cover — last-resort guardlogger.exception("Dream pass crashed for user %s: %s", user_id[:12], exc)- result = run.failure(str(exc))+ result = run.failure(pass_error("pass", "crashed"))await record_sync_outcome(result)return result@@ -496,9 +505,17 @@ async def _run_locked(result = await _dream(run, scope, lock_handle, config=config, status_id=status_id)+ except DreamLockLostError:+ logger.exception("Dream pass lost its lock for user %s", run.user_id[:12])+ result = run.failure(pass_error("apply", "lock_lost"))+ except PhaseChargeError:+ logger.exception(+ "Dream pass could not be charged for user %s", run.user_id[:12]+ )+ result = run.failure(pass_error("billing", "charge_failed"))except Exception as exc:logger.exception("Dream pass crashed for user %s: %s", run.user_id[:12], exc)- result = run.failure(str(exc))+ result = run.failure(pass_error("pass", "crashed"))await record_sync_outcome(result)return result@@ -558,7 +575,9 @@ async def _gather(budget_ok, budget_skip = await check_dream_budget(run.user_id, config=config)if not budget_ok:if budget_skip == "rate_limit_unavailable":- raise PassEnded(run.failure(f"billing: {budget_skip}"))+ raise PassEnded(+ run.failure(pass_error("billing", "rate_limit_unavailable"))+ )raise PassEnded(run.skipped(budget_skip or "insufficient_credits"))input_bundle = await gather_dream_input(scope)if not input_bundle.episodes and not input_bundle.facts:@@ -567,7 +586,12 @@ async def _gather(raise PassEnded(run.skipped("no_input"))if await _nothing_new_since_last_pass(run, scope, input_bundle, status_id):raise PassEnded(run.skipped("no_new_activity"))- await record_gathered(run.pass_id, input_bundle)+ if not await record_gathered(run.pass_id, input_bundle):+ # What the pass read must be on its row before any model call: a+ # hard forget marks a closed pass by it. A row that refused it was+ # closed by a stop, whose error the pass ends with.+ stopped = await stopped_error(run.pass_id)+ raise PassEnded(run.failure(stopped or pass_error("gather", "record_failed")))return input_bundle@@ -637,7 +661,8 @@ async def _phase(try:output, usage = await call()except InferenceError as exc:- raise PassEnded(run.failure(f"{phase}: {exc}")) from exc+ logger.warning("Dream pass %s: %s failed: %s", run.pass_id, phase, exc)+ raise PassEnded(run.failure(pass_error(phase, exc.code))) from excexcept PhaseChargeError as exc:run.phases.append(phase_usage(phase, exc.usage))raise@@ -663,6 +688,7 @@ async def _apply(ops,known_fact_uuids=input_bundle.known_fact_uuids,known_episode_uuids=input_bundle.known_episode_uuids,+ fact_scopes=fact_scopes(input_bundle),lock_handle=lock_handle,lease=lease,)@@ -701,6 +727,9 @@ def _applied_result(proposal_count=_as_int("proposal_count"),demotion_count=_as_int("demotion_count"),entity_invalidation_count=_as_int("entity_invalidation_count"),+ provenance_pending=_as_int("provenance_pending"),+ uncited_writes_dropped=_as_int("uncited_writes_dropped"),+ cross_scope_citations_dropped=_as_int("cross_scope_citations_dropped"),protected_demotions=_as_int("protected_demotions"),indeterminate_demotion_writes=_as_int("indeterminate_demotion_writes"),# Only apply's own False marks the count unconfirmed.diff --git a/autogpt_platform/backend/backend/copilot/dream/orchestrator_stop_test.py b/autogpt_platform/backend/backend/copilot/dream/orchestrator_stop_test.pyindex a3d863b2d6..7de1ef60ed 100644--- a/autogpt_platform/backend/backend/copilot/dream/orchestrator_stop_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/orchestrator_stop_test.py@@ -194,9 +194,7 @@ class TestTheGuard:row = fake_dream_db.rows[other]assert row["status"] is DreamPassStatus.EXPIREDassert row["cancel_generation"] == 1- assert row["error"] == (- f"forced by an admin-triggered dream pass {result.pass_id}"- )+ assert row["error"] == "expired: forced"assert fake_dream_db.rows[result.pass_id]["status"] is (DreamPassStatus.COMPLETE)@@ -266,7 +264,7 @@ class TestTheStopChecks:result = await orchestrator_mod.execute_dream_pass("u")- assert result.error == "cancelled: testing"+ assert result.error == "cancelled: by_admin"assert result.usage is not Noneassert [p.phase for p in result.usage.phases] == billedassert phases.await_count == len(billed)@@ -276,7 +274,7 @@ class TestTheStopChecks:row = fake_dream_db.rows[result.pass_id]assert (row["status"], row["error"], row["cancel_generation"]) == (DreamPassStatus.CANCELLED,- "testing",+ "cancelled: by_admin",1,)assert DreamPassStatus.ERRORED not in fake_dream_db.statuses(result.pass_id)@@ -286,7 +284,9 @@ class TestTheStopChecks:):async def expire() -> None:[pass_id] = list(fake_dream_db.rows)- await write_stop(pass_id, expired("forced", not_updated_since=None))+ await write_stop(+ pass_id, expired("expired: forced", not_updated_since=None)+ )_phases(mocker, 1, expire)diff --git a/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py b/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.pyindex 139f978e0a..50172486ff 100644--- a/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py@@ -51,6 +51,7 @@ from . import orchestrator as orchestrator_modfrom .apply import INGESTION_DRAIN_TIMEOUT_SECONDS, LOCK_DRAIN_RENEWAL_SECONDSfrom .batch_callbacks import handle_dream_batch_resultfrom .batch_submit import persist_input_bundle+from .citations import fact_scopesfrom .fetch import DreamInput, EpisodeRow, FactRowfrom .locks import (BATCH_LOCK_TTL_SECONDS,@@ -679,6 +680,7 @@ async def test_clamps_oversized_sanitizer_output(mocker):*,known_fact_uuids=None,known_episode_uuids=None,+ fact_scopes=None,lock_handle=None,lease=None,):@@ -749,6 +751,7 @@ async def test_demotions_capped_at_five_percent_of_active_facts(mocker):*,known_fact_uuids=None,known_episode_uuids=None,+ fact_scopes=None,lock_handle=None,lease=None,):@@ -864,6 +867,7 @@ async def test_sync_path_filters_hallucinated_demotion_before_cap(mocker):*,known_fact_uuids=None,known_episode_uuids=None,+ fact_scopes=None,lock_handle=None,lease=None,):@@ -954,6 +958,10 @@ async def test_sync_path_passes_known_fact_uuids_to_apply(mocker):apply_mock.await_args.kwargs["known_episode_uuids"]== input_bundle.known_episode_uuids)+ # Each fact's scope, which a write's fact citations must share+ # (``citations.py``).+ assert apply_mock.await_args.kwargs["fact_scopes"] == fact_scopes(input_bundle)+ assert apply_mock.await_args.kwargs["fact_scopes"]@pytest.mark.asyncio@@ -1146,9 +1154,7 @@ async def test_partial_failure_still_charges_completed_phases(result = await orchestrator_mod.execute_dream_pass("u")- assert result.error is not None- assert result.error.startswith(f"{failed_phase}:")- assert error in result.error+ assert result.error == f"{failed_phase}: inference_failed"assert charge_spy.await_count == completedassert [c.args[0].job.phase for c in charge_spy.await_args_list] == (phases[:completed]@@ -1189,7 +1195,11 @@ async def test_a_failed_phase_preserves_trace_usage_without_charging(mocker):mocker.patch.object(orchestrator_mod,"structured_complete",- AsyncMock(side_effect=InferenceError("LLM JSON did not match", billed)),+ AsyncMock(+ side_effect=InferenceError(+ "LLM JSON did not match", billed, code="schema_mismatch"+ )+ ),)mocker.patch.object(orchestrator_mod, "record_phase_cost", billing_mod.record_phase_cost@@ -1203,8 +1213,7 @@ async def test_a_failed_phase_preserves_trace_usage_without_charging(mocker):result = await orchestrator_mod.execute_dream_pass("u", expert_id="expert-1")- assert result.error is not None- assert result.error.startswith("consolidate: LLM JSON did not match")+ assert result.error == "consolidate: schema_mismatch", "a code, never the answer"apply_mock.assert_not_awaited()persist.assert_not_awaited()(seen,) = traced@@ -1226,7 +1235,11 @@ async def test_a_phase_that_got_no_answer_records_nothing(mocker):mocker.patch.object(orchestrator_mod,"structured_complete",- AsyncMock(side_effect=InferenceError("LLM call failed: TimeoutError: ")),+ AsyncMock(+ side_effect=InferenceError(+ "LLM call failed: TimeoutError: ", code="call_failed"+ )+ ),)mocker.patch.object(orchestrator_mod, "record_phase_cost", billing_mod.record_phase_cost@@ -1237,8 +1250,7 @@ async def test_a_phase_that_got_no_answer_records_nothing(mocker):result = await orchestrator_mod.execute_dream_pass("u")- assert result.error is not None- assert result.error.startswith("consolidate: LLM call failed")+ assert result.error == "consolidate: call_failed"persist.assert_not_awaited()assert result.usage is not None and result.usage.phases == []@@ -2029,6 +2041,30 @@ class TestNearDuplicateWriteDedup:# Survivor keeps its own uuids first, then the absorbed extras.assert kept[0].source_episode_uuids == ["ep-2", "ep-3", "ep-1"]+ def test_survivor_absorbs_dropped_writes_fact_citations(self):+ """A forget of a fact only the dropped duplicate cited must still+ reach the survivor (``graphiti/recall_cascade.py``)."""+ writes = [+ ConsolidatedFact(+ content="Nick uses Terminus on his iPhone for CLI work",+ confidence=0.6,+ source_fact_uuids=["f-1"],+ ),+ ConsolidatedFact(+ content=(+ "Nick uses Terminus on his iPhone for CLI work and wants "+ "it to display more ASCII characters"+ ),+ confidence=0.7,+ source_episode_uuids=["ep-3"],+ source_fact_uuids=["f-2"],+ ),+ ]+ kept, dropped = dedup_mod.dedupe_near_duplicate_writes(writes)+ assert dropped == 1+ assert kept[0].source_fact_uuids == ["f-2", "f-1"]+ assert kept[0].source_episode_uuids == ["ep-3"]+def test_word_order_permutation_is_not_merged(self):writes = [ConsolidatedFact(content="Alice introduced Bob to Carol", confidence=0.7),@@ -2138,7 +2174,7 @@ async def test_batch_handoff_revokes_batch_when_lock_extend_fails(mocker):remove_pending.assert_awaited_once_with("batch-xyz")cleanup.assert_awaited_once_with(run.pass_id)handle.disown.assert_not_called()- assert result.error and "lock lost" in result.error+ assert result.error == "submit: lock_lost"# ---------------------------------------------------------------------------@@ -2292,7 +2328,7 @@ async def test_a_phase_failure_closes_the_row_errored_at_that_phase(result = await orchestrator_mod.execute_dream_pass("u")- assert result.error == "recombine: provider down"+ assert result.error == "recombine: inference_failed"apply_mock.assert_not_awaited()assert fake_dream_db.statuses(result.pass_id) == [DreamPassStatus.RUNNING,@@ -2311,9 +2347,13 @@ async def test_a_phase_failure_closes_the_row_errored_at_that_phase(@pytest.mark.asyncio-async def test_a_store_outage_never_fails_the_pass(mocker, fake_dream_db, caplog):+async def test_a_pass_whose_insert_is_not_confirmed_never_reads_the_graph(+ mocker, fake_dream_db, caplog+):+ """A hard forget marks only passes whose rows it can read: a pass whose+ insert failed or ran out of time stops before it gathers."""fake_dream_db.fail = True- mocker.patch.object(+ gather = mocker.patch.object(orchestrator_mod, "gather_dream_input", AsyncMock(return_value=_build_input()))apply_mock = _stub_three_phases_and_apply(mocker)@@ -2321,12 +2361,52 @@ async def test_a_store_outage_never_fails_the_pass(mocker, fake_dream_db, caplogwith caplog.at_level(logging.WARNING):result = await orchestrator_mod.execute_dream_pass("u")- assert result.error is None and result.skipped is False- assert result.dream_session_id == "s"- apply_mock.assert_awaited_once()+ assert result.error == "pass: record_unconfirmed" and result.skipped is False+ gather.assert_not_awaited()+ apply_mock.assert_not_awaited()assert fake_dream_db.writes == []assert "could not insert its record" in caplog.text- assert "could not record the outcome" in caplog.text++[email protected]("landed", [False, None], ids=["refused", "failed"])+async def test_a_pass_whose_read_set_is_not_on_its_row_makes_no_model_call(+ mocker, fake_dream_db, landed+):+ """What the pass read goes on its row with the gather, before the first+ model call: a write the row refused (closed or gone, with no stop to+ report) or that failed ends the pass there."""+ mocker.patch.object(+ orchestrator_mod, "gather_dream_input", AsyncMock(return_value=_build_input())+ )+ mocker.patch.object(+ orchestrator_mod, "record_gathered", AsyncMock(return_value=landed)+ )+ apply_mock = _stub_three_phases_and_apply(mocker)+ consolidate = mocker.patch.object(orchestrator_mod, "_run_consolidate", AsyncMock())++ result = await orchestrator_mod.execute_dream_pass("u")++ assert result.error == "gather: record_failed"+ consolidate.assert_not_awaited()+ apply_mock.assert_not_awaited()+++async def test_the_gather_records_every_fact_and_episode_it_read(mocker, fake_dream_db):+ bundle = _build_input(episodes=2, facts=2).model_copy(+ update={"known_fact_uuids": {"f0", "f1", "f9"}}+ )+ mocker.patch.object(+ orchestrator_mod, "gather_dream_input", AsyncMock(return_value=bundle)+ )+ _stub_three_phases_and_apply(mocker)++ result = await orchestrator_mod.execute_dream_pass("u")++ assert result.error is None+ row = fake_dream_db.rows[result.pass_id]+ assert row["read_uuids"] == ["e0", "e1", "f0", "f1", "f9"]def _batch_route(mocker, *, lock_extends: bool) -> MagicMock:@@ -2412,7 +2492,7 @@ async def test_a_batch_handoff_that_lost_the_lock_is_recorded_errored(result = await orchestrator_mod.execute_dream_pass("u")- assert result.error and "lock lost" in result.error+ assert result.error == "submit: lock_lost"assert fake_dream_db.statuses(result.pass_id) == [DreamPassStatus.RUNNING,DreamPassStatus.ERRORED,@@ -2457,12 +2537,15 @@ def _three_billed_phases(mocker) -> None:@pytest.mark.asyncio@pytest.mark.parametrize(- "failure",- [RuntimeError("apply exploded"), DreamLockLostError("u")],+ "failure, error",+ [+ (RuntimeError("apply exploded"), "pass: crashed"),+ (DreamLockLostError("u"), "apply: lock_lost"),+ ],ids=["apply_crash", "lock_lost"],)async def test_a_failure_after_the_phases_keeps_all_three_phases_usage(- mocker, fake_dream_db, failure+ mocker, fake_dream_db, failure, error):mocker.patch.object(orchestrator_mod, "gather_dream_input", AsyncMock(return_value=_build_input())@@ -2474,7 +2557,7 @@ async def test_a_failure_after_the_phases_keeps_all_three_phases_usage(result = await orchestrator_mod.execute_dream_pass("u")- assert result.error == str(failure)+ assert result.error == error, "no exception text is recorded"assert result.usage is not Noneassert [p.phase for p in result.usage.phases] == ["consolidate",@@ -2560,8 +2643,8 @@ def _fail_the_charge(mocker, phase: str, failure: str) -> AsyncMock:@pytest.mark.parametrize("answer, failure, error",[- ("parsed", "trial_cost", "trial cost database unavailable"),- ("parsed", "cost_log", "cost log scheduling failed after the charge"),+ ("parsed", "trial_cost", "billing: charge_failed"),+ ("parsed", "cost_log", "billing: charge_failed"),],ids=["trial_cost", "cost_log"],)@@ -2620,7 +2703,7 @@ async def test_failed_inference_is_absorbed_in_ledger_and_durable_pass(result = await orchestrator_mod.execute_dream_pass("u")completed = _PHASES.index(phase)- assert result.error == f"{phase}: answer did not parse"+ assert result.error == f"{phase}: inference_failed"assert charges.await_count == completedapply_mock.assert_not_awaited()assert result.usage is not None@@ -2787,13 +2870,12 @@ async def test_a_stalled_record_store_costs_each_write_only_its_deadline(result = await asyncio.wait_for(orchestrator_mod.execute_dream_pass("u"), 10)- assert result.error is None and result.dream_session_id == "s"- apply_mock.assert_awaited_once()- # The insert, the guard's read of the scope's open passes, six updates,- # the four stop checks and the four lease renewals (three phases, apply),- # each abandoned at the deadline; neither the guard nor a check nor a- # renewal held the pass on a store that did not answer.- assert (stalled_dream_db.started, stalled_dream_db.cancelled) == (16, 16)+ assert result.error == "pass: record_unconfirmed"+ apply_mock.assert_not_awaited()+ # The insert, then the outcome that would close a row it left behind,+ # each abandoned at the deadline: a pass whose row the store cannot+ # confirm never reads the graph, and is not held by the store.+ assert (stalled_dream_db.started, stalled_dream_db.cancelled) == (2, 2)@pytest.mark.asyncio@@ -2820,7 +2902,8 @@ async def test_the_nightly_fan_out_reaches_ratification_past_a_stalled_store(result = await asyncio.wait_for(nightly_batch_mod.run_nightly_batch_submit("u"), 10)ratification.assert_awaited_once_with("u", expert_id=None)- assert result.dream is not None and result.dream.error is None+ assert result.dream is not None+ assert result.dream.error == "pass: record_unconfirmed"@pytest.mark.asynciodiff --git a/autogpt_platform/backend/backend/copilot/dream/pass_errors.py b/autogpt_platform/backend/backend/copilot/dream/pass_errors.pynew file mode 100644index 0000000000..3f3b0ddd81--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/pass_errors.py@@ -0,0 +1,85 @@+"""The errors a dream pass records: ``<stage>: <code>``, both from the fixed+sets below, so no recorded error holds the model's output, the pass's input+or an exception's text (those go to the log). The store records no other+error (``data/dream_pass_update.transition_args`` turns any other into+``pass: unrecorded``), and a hard forget keeps an error only when it is one+of these (``dream_error_blanked``, in the migration that added the mark).+"""++from typing import Literal, get_args++from backend.copilot.inference.context import InferenceErrorCode++# Where the pass was: the pass as a whole, its billing check, one of its+# steps (the row's phases), the batch route's handoff and its callbacks'+# payload, or a stop from outside (a cancel, an expiry).+PassErrorStage = Literal[+ "pass",+ "billing",+ "gather",+ "consolidate",+ "recombine",+ "sanitize",+ "apply",+ "done",+ "submit",+ "batch",+ "cancelled",+ "expired",+]++# What went wrong: a phase's answer (``InferenceError.code``), the+# provider's, the infrastructure, the pass's lock or record, a batch+# payload, or why a stop from outside ended it.+PassErrorCode = Literal[+ InferenceErrorCode,+ "provider_error",+ "no_rows",+ "missing_content",+ "crashed",+ "import_failed",+ "no_api_key",+ "submit_failed",+ "input_persist_failed",+ "gate_unavailable",+ "rate_limit_unavailable",+ "charge_failed",+ "record_unconfirmed",+ "record_failed",+ "unrecorded",+ "not_on_record",+ "lock_lost",+ "lease_unrenewed",+ "row_refused",+ "payload_incomplete",+ "usage_attribution_unavailable",+ "unknown_phase",+ "input_missing",+ "scope_mismatch",+ "already_closed",+ "duplicate_delivery",+ "by_admin",+ "by_wipe",+ "forced",+ "lease_lapsed",+ "stalled",+ "reaped",+ "stopped",+]++PASS_ERROR_STAGES: tuple[PassErrorStage, ...] = get_args(PassErrorStage)+PASS_ERROR_CODES: tuple[PassErrorCode, ...] = get_args(PassErrorCode)++# What the store records in place of an error that is not one of these.+UNRECORDED = "pass: unrecorded"+++def pass_error(stage: PassErrorStage, code: PassErrorCode) -> str:+ """The error a pass records: ``<stage>: <code>``."""+ return f"{stage}: {code}"+++def is_pass_error(error: str) -> bool:+ """Whether *error* is ``<stage>: <code>`` from the fixed sets."""+ stage, separator, code = error.partition(": ")+ return bool(separator) and stage in PASS_ERROR_STAGES and code in PASS_ERROR_CODESdiff --git a/autogpt_platform/backend/backend/copilot/dream/pass_record.py b/autogpt_platform/backend/backend/copilot/dream/pass_record.pyindex d843804d7c..261de43462 100644--- a/autogpt_platform/backend/backend/copilot/dream/pass_record.py+++ b/autogpt_platform/backend/backend/copilot/dream/pass_record.py@@ -44,6 +44,7 @@ from backend.data.dream_pass_models import (from .fetch import DreamInputfrom .locks import DEFAULT_LOCK_TTL_SECONDS+from .pass_errors import is_pass_error, pass_errorfrom .routing import ExecutionPathfrom .schemas import DreamOperations, DreamPassResult, DreamPassUsage, DreamPhase@@ -119,11 +120,25 @@ def lease(lease_token: str, ttl_seconds: int) -> DreamPassUpdate:def gathered(input_bundle: DreamInput) -> DreamPassUpdate:- """The input is in: the window it covers; consolidate is next."""+ """The input is in: the window it covers and every fact and episode it+ read, which the row keeps for a hard forget; consolidate is next."""return DreamPassUpdate(phase=DreamPassPhase.CONSOLIDATE,window_start=input_bundle.window_start,window_end=input_bundle.window_end,+ read_uuids=read_uuids(input_bundle),+ )+++def read_uuids(input_bundle: DreamInput) -> list[str]:+ """Every fact and episode *input_bundle* holds, sorted."""+ return sorted(+ {+ *input_bundle.known_fact_uuids,+ *input_bundle.known_episode_uuids,+ *(fact.uuid for fact in input_bundle.facts),+ *(episode.uuid for episode in input_bundle.episodes),+ })@@ -316,12 +331,16 @@ def cleanup_finished() -> DreamPassUpdate:def stop_error(row: DreamPassRecord) -> str | None:"""Why the row says its pass was stopped from outside, as the pass's- error (``cancelled: <reason>``, ``expired: <reason>``); ``None`` while its+ error: the stop's own (``cancelled: by_admin``, ``expired: forced``,+ ``consolidate: reaped``), else ``<status>: stopped``; ``None`` while itscancel generation is still the one a new row starts at."""if row.cancel_generation == INITIAL_CANCEL_GENERATION:return None- how = row.status.value.lower()- return f"{how}: {row.error}" if row.error else how+ if row.error is not None and is_pass_error(row.error):+ return row.error+ if row.status == DreamPassStatus.CANCELLED:+ return pass_error("cancelled", "stopped")+ return pass_error("expired", "stopped")def dream_pass_result_from_row(row: DreamPassRecord) -> DreamPassResult:@@ -345,6 +364,9 @@ def dream_pass_result_from_row(row: DreamPassRecord) -> DreamPassResult:dropped_forgotten=applied.dropped_forgotten,pending_writes=applied.pending_writes,failed_writes=applied.failed_writes,+ provenance_pending=applied.provenance_pending,+ uncited_writes_dropped=applied.uncited_writes_dropped,+ cross_scope_citations_dropped=applied.cross_scope_citations_dropped,protected_demotions=applied.protected_demotions,indeterminate_demotion_writes=applied.indeterminate_demotion_writes,demotion_accounting_complete=applied.demotion_accounting_complete,@@ -368,6 +390,9 @@ def _applied(result: DreamPassResult) -> DreamPassApplied:dropped_forgotten=result.dropped_forgotten,pending_writes=result.pending_writes,failed_writes=result.failed_writes,+ provenance_pending=result.provenance_pending,+ uncited_writes_dropped=result.uncited_writes_dropped,+ cross_scope_citations_dropped=result.cross_scope_citations_dropped,protected_demotions=result.protected_demotions,indeterminate_demotion_writes=result.indeterminate_demotion_writes,demotion_accounting_complete=result.demotion_accounting_complete,diff --git a/autogpt_platform/backend/backend/copilot/dream/prompts.py b/autogpt_platform/backend/backend/copilot/dream/prompts.pyindex bf583a8911..c1305c32ee 100644--- a/autogpt_platform/backend/backend/copilot/dream/prompts.py+++ b/autogpt_platform/backend/backend/copilot/dream/prompts.py@@ -124,14 +124,21 @@ CONSOLIDATE_SYSTEM = ("a fact only if it is new or meaningfully refines an existing one.\n"" * Group by scope — never merge a fact in 'project:foo' with one in ""'real:global'.\n"- " * Carry forward the source episode uuids exactly. Never make uuids up.\n"+ " * Every fact MUST cite what it was drawn from: the uuids of the recent "+ "episodes in source_episode_uuids and of the active facts in "+ "source_fact_uuids, copied exactly from the lists below. Cite every "+ "source you used, and at least one; a fact citing nothing listed below "+ "is dropped. Never make uuids up. Cite only active facts of the fact's "+ "own scope (they are listed by scope): a fact citing an active fact of "+ "another scope is dropped. Recent episodes can be cited whatever the "+ "fact's scope: a chat turn can hold facts of any scope.\n"" * Confidence is your own — 0.0 means 'I am unsure', 1.0 means ""'this is restated verbatim across multiple sources'.\n"f" * Output AT MOST {MAX_WRITES_PER_PASS} consolidated facts. Better ""to consolidate hard than to flood the recombination step.\n\n""JSON SCHEMA (your response MUST match this shape):\n"'{ "facts": [ { "content": str, "scope": str, "confidence": float, '- '"source_episode_uuids": [str, ...] } ] }\n\n'+ '"source_episode_uuids": [str, ...], "source_fact_uuids": [str, ...] } ] }\n\n''If you have nothing to emit, return `{"facts": []}` — still JSON, '"still starting with `{`.")@@ -169,14 +176,18 @@ RECOMBINE_SYSTEM = ("ROLE:\n""You are the recombination step of a sleep-time memory pass. The ""previous step consolidated the user's recent facts. Your job is to "- "look for *novel connections* — weak-link findings, implications "- "across scopes, useful rules the user has implicitly demonstrated.\n\n"+ "look for *novel connections* — weak-link findings, implications, "+ "useful rules the user has implicitly demonstrated — each within one "+ "scope: a finding stays in the scope of the facts it cites.\n\n""RULES:\n"- " * Every proposal must cite at least one source_fact_uuid or "- "source_episode_uuid from the consolidation step's input. Proposals "- "with no citation will be rejected by the sanitizer.\n"- " * Stay in scope — proposed findings live in the same scope as "- "their evidence.\n"+ " * Every proposal must cite what it rests on: the uuids of the active "+ "facts (source_fact_uuids) and recent episodes (source_episode_uuids) "+ "listed below, copied exactly, and at least one. A consolidated fact has "+ "no uuid of its own: cite the uuids it cites. A proposal citing nothing "+ "listed below is dropped.\n"+ " * Stay in scope — a finding lives in the scope of the active facts "+ "it cites, and one citing an active fact of another scope is dropped. "+ "Recent episodes can be cited whatever the finding's scope.\n"' * Do not propose first-person-as-user statements ("I think X"). '"All proposals must describe the user's world, not the assistant's.\n"f" * Output AT MOST {MAX_PROPOSALS_PER_PASS} proposals. Quality > quantity.\n"@@ -265,7 +276,9 @@ SANITIZE_SYSTEM = ("fact uuids. Do not invent uuids.\n"" * Entity invalidations require an entity_uuid present in the ""active-facts source/target set. Single-hop demotion only.\n"- " * Reject any proposal that has zero source citations.\n"+ " * Reject any write or proposal that has zero source citations "+ "(source_fact_uuids and source_episode_uuids both empty). Keep the "+ "citations of everything you let through exactly as given.\n"" * Reject any proposal whose content starts with first-person "'("I think", "I believe", "In my opinion") — those are not user '"memories.\n"diff --git a/autogpt_platform/backend/backend/copilot/dream/prompts_test.py b/autogpt_platform/backend/backend/copilot/dream/prompts_test.pyindex dde6fc41b7..b4ef2604e8 100644--- a/autogpt_platform/backend/backend/copilot/dream/prompts_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/prompts_test.py@@ -87,6 +87,27 @@ def test_consolidate_prompt_has_system_and_user_messages():assert "[scope=project:x]" in user_body+def test_the_scope_rule_binds_the_fact_citations_only():+ """Facts are listed by scope and one citing a fact of another scope is+ dropped; a chat turn can hold facts of any scope, so episodes carry no+ scope and are cited from any."""+ bundle = _build_bundle()++ system, user = build_consolidate_prompt(bundle)+ recombine = build_recombine_prompt(bundle, "{}")[0]["content"]++ assert "- uuid=ep-1 valid_at=2026-05-10" in user["content"]+ assert "a fact citing an active fact of another scope is dropped" in (+ system["content"]+ )+ assert "Recent episodes can be cited whatever the fact's scope" in (+ system["content"]+ )+ assert "across scopes" not in recombine+ assert "a finding stays in the scope of the facts it cites" in recombine+ assert "Recent episodes can be cited whatever the finding's scope" in recombine++def test_consolidate_prompt_instructs_merge_and_no_restate():"""#13387: the consolidate prompt must tell the model to mergeparaphrases into one statement and not re-emit existing active facts."""diff --git a/autogpt_platform/backend/backend/copilot/dream/reaper.py b/autogpt_platform/backend/backend/copilot/dream/reaper.pyindex 5ff9db2adc..8f6eca580e 100644--- a/autogpt_platform/backend/backend/copilot/dream/reaper.py+++ b/autogpt_platform/backend/backend/copilot/dream/reaper.py@@ -88,6 +88,12 @@ a pass's landed phases are still there to charge behind about seventy runs'worth of cleanups that keep failing. One whose state lapsed all the same isreported as lost usage, an error (``cleanup.py``).+With ``SWEEP_RESERVE_SECONDS`` of the budget left after the rows, a run also+sweeps the memory graphs where a dream write's derivation record failed+(``graphiti/provenance_pending.py``): it completes their citation markers+under each graph's write lock, so a dream fact's provenance is complete even+before the next forget there, which reconciles on its own anyway.+One line per row (INFO when it expired, was cleaned or moved; WARNING when itfailed, is to be retried, or the budget left it), then one with the countsper outcome.@@ -101,10 +107,11 @@ from collections.abc import Awaitablefrom datetime import datetime, timedelta, timezonefrom typing import Any, Literal, cast-from prisma.enums import DreamPassRoute, DreamPassStatus+from prisma.enums import DreamPassPhase, DreamPassRoutefrom pydantic import BaseModelfrom backend.copilot.config import ChatConfig+from backend.copilot.graphiti.provenance_pending import sweep_pendingfrom backend.copilot.graphiti.scope import MemoryScopefrom backend.data import redis_clientfrom backend.data.dream_pass_models import OPEN_STATUSES, DreamPassRecord@@ -116,6 +123,7 @@ from .locks import (LOCK_CHECK_TIMEOUT_SECONDS,release_dream_lock,)+from .pass_errors import PassErrorStage, pass_errorfrom .pass_record import reapedfrom .provider_batch import PROVIDER_BATCH_WINDOW_SECONDSfrom .schemas import DreamPhase@@ -151,6 +159,8 @@ _SCAN_KEYS: dict[ScanKind, str] = {SCAN_TTL_SECONDS = 24 * 60 * 60# Giving back the reaper's hold on a scope, carved out of the budget.RELEASE_TIMEOUT_SECONDS = 2.0+# The budget a run needs left after its rows to sweep pending dream records.+SWEEP_RESERVE_SECONDS = 20.0ReapOutcome = Literal["expired", "cleaned", "moved", "retry", "failed", "out_of_budget"]# The outcomes of a row whose cleanup finished, its mark still to clear.@@ -158,10 +168,12 @@ _FINISHED: frozenset[ReapOutcome] = frozenset({"expired", "cleaned"})class ReaperRun(BaseModel):- """What one run did: how many rows it listed, and each one's outcome."""+ """What one run did: how many rows it listed, and each one's outcome;+ and how many dream records it completed in the graphs it swept."""listed: intoutcomes: dict[ReapOutcome, int]+ reconciled: int = 0async def reap_expired_passes(*, now: datetime | None = None) -> ReaperRun:@@ -171,13 +183,16 @@ async def reap_expired_passes(*, now: datetime | None = None) -> ReaperRun:lapsed: list[DreamPassRecord] = []pending: list[DreamPassRecord] = []done: dict[str, ReapOutcome] = {}+ reconciled = 0try:async with asyncio.timeout_at(deadline):lapsed, pending = await _list(now or datetime.now(timezone.utc))await _reap_both(lapsed, pending, done, deadline)+ if loop.time() < deadline - SWEEP_RESERVE_SECONDS:+ reconciled = (await sweep_pending()).completedexcept Exception:logger.warning("Dream pass reaper: the run stopped short", exc_info=True)- return _summary([*lapsed, *pending], done)+ return _summary([*lapsed, *pending], done, reconciled)async def _list(now: datetime) -> tuple[list[DreamPassRecord], list[DreamPassRecord]]:@@ -360,7 +375,9 @@ async def _attempt(done.setdefault(row.id, "out_of_budget")-def _summary(rows: list[DreamPassRecord], done: dict[str, ReapOutcome]) -> ReaperRun:+def _summary(+ rows: list[DreamPassRecord], done: dict[str, ReapOutcome], reconciled: int+) -> ReaperRun:"""Log each row the budget left and the counts per outcome."""for row in rows:if row.id not in done:@@ -370,8 +387,11 @@ def _summary(rows: list[DreamPassRecord], done: dict[str, ReapOutcome]) -> Reape)outcomes: Counter[ReapOutcome] = Counter(done.values())outcomes["out_of_budget"] += len(rows) - len(done)- run = ReaperRun(listed=len(rows), outcomes=dict(+outcomes))- logger.info(f"Dream pass reaper: {run.listed} listed; {_counts(run)}")+ run = ReaperRun(listed=len(rows), outcomes=dict(+outcomes), reconciled=reconciled)+ logger.info(+ f"Dream pass reaper: {run.listed} listed; {_counts(run)}; "+ f"{run.reconciled} dream record(s) reconciled"+ )return run@@ -408,16 +428,18 @@ async def _close("""Expire *row*, marked for cleanup, then clean up after its pass;``moved`` when the row was written since it was read, and nothing istouched."""- closing = reaped(_error(row), not_updated_since=row.updated_at)- if not await write_stop(row.id, closing):+ error = _error(row)+ if not await write_stop(row.id, reaped(error, not_updated_since=row.updated_at)):logger.info(f"Dream pass reaper: pass {row.id} moved since it was read; left")return "moved"# A held scope may still be the dead pass's: released only if it is.cleanup = await _clean_up(row, scope, release=not took_scope)lock = "held by the reaper" if took_scope else "released if the dead pass's"- return _logged(- row, cleanup, "expired", f"expired ({_error(row)})", f"; scope lock {lock}"+ what = (+ f"expired as {error} ({row.status.value.lower()}, lease until "+ f"{row.lease_expires_at}, last written {row.updated_at})")+ return _logged(row, cleanup, "expired", what, f"; scope lock {lock}")async def _finish_cleanup(row: DreamPassRecord, scope: MemoryScope) -> ReapOutcome:@@ -571,16 +593,20 @@ def _batch_to_stop(row: DreamPassRecord) -> str | None:return row.provider_batch_id+# The step a row's phase names, as its error records it.+_PHASE_STAGES: dict[DreamPassPhase, PassErrorStage] = {+ DreamPassPhase.GATHER: "gather",+ DreamPassPhase.CONSOLIDATE: "consolidate",+ DreamPassPhase.RECOMBINE: "recombine",+ DreamPassPhase.SANITIZE: "sanitize",+ DreamPassPhase.APPLY: "apply",+ DreamPassPhase.DONE: "done",+}++def _error(row: DreamPassRecord) -> str:- """The closed row's error, naming the phase its pass died at."""- phase = row.phase.value.lower()- lapsed = row.lease_expires_at.isoformat() if row.lease_expires_at else "unknown"- if row.status == DreamPassStatus.APPLYING:- return (- f"{phase}: the lease lapsed at {lapsed} while applying; closed by the "- "reaper without applying again, and what it wrote may have landed"- )- return f"{phase}: the lease lapsed at {lapsed}; closed by the reaper"+ """The closed row's error, naming the step its pass died at."""+ return pass_error(_PHASE_STAGES[row.phase], "reaped")def _counts(run: ReaperRun) -> str:diff --git a/autogpt_platform/backend/backend/copilot/dream/reaper_sweep_test.py b/autogpt_platform/backend/backend/copilot/dream/reaper_sweep_test.pynew file mode 100644index 0000000000..f6f6a273b7--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/dream/reaper_sweep_test.py@@ -0,0 +1,52 @@+"""The dream pass reaper also sweeps the memory graphs where a dream write's+derivation record failed (``graphiti/provenance_pending.py``), with what its+budget leaves after the rows. The sweep itself is pinned in+``graphiti/recall_reconcile_test.py`` and run live in+``graphiti/recall_provenance_integration_test.py``.+"""++from unittest.mock import AsyncMock++import pytest++from backend.copilot.graphiti.provenance_pending import Swept++from . import reaper as reaper_mod+from .reaper import reap_expired_passes+++def sweep(mocker) -> AsyncMock:+ return mocker.patch.object(+ reaper_mod, "sweep_pending", AsyncMock(return_value=Swept(completed=2))+ )+++async def test_a_run_sweeps_the_pending_dream_records(+ fake_dream_db, fake_dream_redis, sweep: AsyncMock+) -> None:+ run = await reap_expired_passes()++ sweep.assert_awaited_once()+ assert (run.listed, run.reconciled) == (0, 2)+++async def test_a_run_short_of_budget_leaves_the_sweep_to_the_next(+ fake_dream_db, fake_dream_redis, sweep: AsyncMock, mocker+) -> None:+ mocker.patch.object(reaper_mod, "REAPER_BUDGET_SECONDS", 10.0)++ run = await reap_expired_passes()++ sweep.assert_not_awaited()+ assert run.reconciled == 0+++async def test_a_sweep_that_raises_does_not_fail_the_run(+ fake_dream_db, fake_dream_redis, sweep: AsyncMock+) -> None:+ sweep.side_effect = RuntimeError("down")++ run = await reap_expired_passes()++ assert (run.listed, run.reconciled) == (0, 0)diff --git a/autogpt_platform/backend/backend/copilot/dream/reaper_test.py b/autogpt_platform/backend/backend/copilot/dream/reaper_test.pyindex d119f3ff98..f6d1ff7149 100644--- a/autogpt_platform/backend/backend/copilot/dream/reaper_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/reaper_test.py@@ -77,8 +77,7 @@ class TestAPassThatOutlivedItsLease:DreamPassStatus.EXPIRED,1,)- assert row["error"].startswith("recombine: the lease lapsed at ")- assert row["error"].endswith("; closed by the reaper")+ assert row["error"] == "recombine: reaped"assert (row["lease_token"], row["lease_expires_at"]) == (None, None)assert (row["input_bundle"], row["cleanup_pending_at"]) == (None, None)provider_cancel.assert_awaited_once()@@ -87,7 +86,10 @@ class TestAPassThatOutlivedItsLease:assert state_key("p1") not in fake_dream_redis.hashesassert input_bundle_key("p1") not in fake_dream_redis.storeassert _LOCK_KEY not in fake_dream_redis.store- assert "Dream pass reaper: pass p1 expired (recombine:" in caplog.text+ assert "Dream pass reaper: pass p1 expired as recombine: reaped (" in (+ caplog.text+ )+ assert ", lease until " in caplog.textassert "Dream pass reaper: 1 listed; expired=1" in caplog.textasync def test_within_its_grace_is_left_open(self, fake_dream_db, provider_cancel):@@ -227,10 +229,7 @@ class TestAPassThatOutlivedItsLease:assert run.outcomes == {"expired": 1}row = fake_dream_db.rows["p1"]assert row["status"] is DreamPassStatus.EXPIRED- assert row["error"].startswith("apply: the lease lapsed at ")- assert "while applying; closed by the reaper without applying again" in (- row["error"]- )+ assert row["error"] == "apply: reaped"apply.assert_not_awaited()assert _charged(charges) == ["consolidate"]diff --git a/autogpt_platform/backend/backend/copilot/dream/recall_guard_routes_test.py b/autogpt_platform/backend/backend/copilot/dream/recall_guard_routes_test.pyindex d068e39a61..e0358d7665 100644--- a/autogpt_platform/backend/backend/copilot/dream/recall_guard_routes_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/recall_guard_routes_test.py@@ -188,7 +188,7 @@ def graph(mocker) -> SimpleNamespace:database = MagicMock()database.create_chat_session = AsyncMock()database.update_chat_session_title = AsyncMock()- database.add_chat_message = AsyncMock()+ database.add_dream_narrative = AsyncMock()mocker.patch("backend.data.db_accessors.chat_db", return_value=database)mocker.patch("backend.data.db_accessors.orgs_db",diff --git a/autogpt_platform/backend/backend/copilot/dream/schemas.py b/autogpt_platform/backend/backend/copilot/dream/schemas.pyindex 67eaa54bda..668de2d4b8 100644--- a/autogpt_platform/backend/backend/copilot/dream/schemas.py+++ b/autogpt_platform/backend/backend/copilot/dream/schemas.py@@ -61,8 +61,10 @@ class IngestionDrainStatus(str, Enum):class ConsolidatedFact(BaseModel):"""A cluster of related facts merged into a single canonical statement.- Phase 1 output. Provenance always points back to the source episodes- so apply.py can record where the consolidation came from.+ Phase 1 output. It cites the episodes and active facts it was drawn from,+ so apply can record where the consolidation came from and a forget of one+ of them reaches it (``dream/citations.py``); one citing nothing the pass+ read is dropped unwritten."""content: str = Field(description="Canonical statement of the consolidated fact.")@@ -79,6 +81,13 @@ class ConsolidatedFact(BaseModel):default_factory=list,description="UUIDs of the :Episodic nodes the fact was consolidated from.",)+ source_fact_uuids: list[str] = Field(+ default_factory=list,+ description=(+ "UUIDs of the active facts (:RELATES_TO edges) the fact was "+ "consolidated from."+ ),+ )class ConsolidationOutput(BaseModel):@@ -194,7 +203,12 @@ class DreamOperations(BaseModel):apply.py:* ≤ ``max_demotions_per_pass`` demotions per pass (runaway-demotionmitigation per spec §3 / TODO P0.3b).- * Scope match enforced — proposals cannot cross scopes.+ * Scope match enforced — a write or proposal citing a fact of+ another scope (a fact's scope, else ``real:global``) is dropped+ whole, as is one citing nothing the pass read+ (``dream/citations.py``); episode citations are not scoped. A+ source in the same scope that the write does not truly rest on is+ not caught.* Empty ``writes`` and ``proposals`` is fine; a pass can be no-op."""@@ -368,8 +382,22 @@ class DreamPassResult(BaseModel):# lock, or being written. Every one on the batch path, which does not# wait for them.pending_writes: int = 0- # The ones whose write raised: dropped, logged, not retried.+ # Of those writes and proposals, the ones the worker did not make: their+ # citation marker could not be written (the write fails closed) or the+ # graph write raised; only those failed before the pass was reported.failed_writes: int = 0+ # Of those made, the ones whose derivation record failed after the write:+ # their citation marker stays in the graph until it is reconciled, before+ # the next forget there or by the dream reaper+ # (``graphiti/recall_reconcile.py``); same reporting window.+ provenance_pending: int = 0+ # Writes and proposals dropped before they were queued because they cited+ # nothing the pass read, or a fact of another scope than their own+ # (``dream/citations.py``); not counted in ``consolidated_count`` or+ # ``proposal_count``.+ uncited_writes_dropped: int = 0+ # Their citations of a fact of another scope; each write is counted above.+ cross_scope_citations_dropped: int = 0# Distinct facts the recall guard kept live through the pass# (``recall_guard.py``): an acknowledged write spared them, and the# pass's final read, after every acknowledged write, found them live. Adiff --git a/autogpt_platform/backend/backend/copilot/dream/store.py b/autogpt_platform/backend/backend/copilot/dream/store.pyindex 96e2e7f910..0f7cf025b7 100644--- a/autogpt_platform/backend/backend/copilot/dream/store.py+++ b/autogpt_platform/backend/backend/copilot/dream/store.py@@ -3,7 +3,7 @@Both routes write here, through ``db_accessors.dream_db()``: the Prisma modulewhere it is connected, the DatabaseManager RPC in the scheduler and the batchexecutor. The sync orchestrator records the start with its lease, the-gathered window, each phase's output, each renewal of the lease, the+gathered window and what it read, each phase's output, each renewal of the lease, theoperations it is about to apply and how the pass ended. The batch pathrecords the submit, each phase that lands and the lease it renewed, each nextbatch, the apply and the end, with what the landed phases used; its end@@ -15,7 +15,10 @@ the database) is ``backend/data/dream_pass_update.py``.A record write is best-effort and bounded. Each one gets``RECORD_WRITE_TIMEOUT_SECONDS``, whatever the client does inside it(connecting, retrying): a write that fails or runs out of time is logged at-warning and dropped, and the pass goes on. So a stalled DatabaseManager costs+warning and dropped, and the pass goes on, but for the two a hard forget+depends on: the insert, and the gather with what the pass read+(``start_pass``, ``record_gathered``), without which the pass stops before+its first model call. So a stalled DatabaseManager costsa pass at most that deadline per write, and the nightly fan-out and the batchwalker keep moving.@@ -76,11 +79,14 @@ RECORD_WRITE_TIMEOUT_SECONDS = 10.0async def start_pass(run: DreamPassRun, scope: MemoryScope, *, trigger: DreamTrigger-) -> None:+) -> bool:"""Insert the run's row: running, gathering its input, holding the leaseof the lock it is about to take, and the usage attribution the pass runsunder (#15208, captured as it started, ``orchestrator``'s- ``attributed_usage``), which every charge of its phases then goes by."""+ ``attributed_usage``), which every charge of its phases then goes by.+ Whether the insert is confirmed: one that failed or ran out of time may+ never land, or land later, so the pass must not gather (a hard forget+ marks only passes it can find)."""try:draft = new_pass(run.pass_id,@@ -94,9 +100,12 @@ async def start_pass(await _bounded(dream_db().create_dream_pass(draft))except Exception:logger.warning(- f"Dream pass {run.pass_id}: could not insert its record; the pass goes on",+ f"Dream pass {run.pass_id}: could not insert its record; the pass "+ "stops before it gathers",exc_info=True,)+ return False+ return Trueasync def read_cost_context(pass_id: str) -> TrialCostContext | None:@@ -125,8 +134,11 @@ async def record_lease(pass_id: str, lease_token: str, ttl_seconds: int) -> boolreturn await _write(pass_id, "the lease", lambda: lease(lease_token, ttl_seconds))-async def record_gathered(pass_id: str, input_bundle: DreamInput) -> None:- await _write(pass_id, "the gathered window", lambda: gathered(input_bundle))+async def record_gathered(pass_id: str, input_bundle: DreamInput) -> bool | None:+ """The gathered window and every fact and episode read, before any model+ call; whether it landed (see ``_write``): a pass whose read set is not on+ its row must not go on, or a hard forget could miss it once closed."""+ return await _write(pass_id, "the gathered window", lambda: gathered(input_bundle))async def record_phase_output(@@ -300,6 +312,20 @@ async def read_user_passes()+async def erase_passes(+ scope: MemoryScope, *, reached: list[str], named: list[str]+) -> list[str]:+ """Mark, for a hard forget, the passes of *scope* it may have reached,+ blanking those closed, and keep only the codes of the scope's errors+ (``forget_copies.py``); the ids marked. Raises when the write fails or+ runs out of time."""+ return await _bounded(+ dream_db().erase_dream_passes(+ scope.owner_user_id, scope.scope_key, reached=reached, named=named+ )+ )++async def delete_old_passes(created_before: datetime, *, limit: int, timeout: float) -> int:@@ -336,7 +362,7 @@ async def _write(written = await _bounded(dream_db().update_dream_pass(pass_id, build()))except Exception:logger.warning(- f"Dream pass {pass_id}: could not record {step}; the pass goes on",+ f"Dream pass {pass_id}: could not record {step}",exc_info=True,)return Nonediff --git a/autogpt_platform/backend/backend/copilot/dream/store_deadline_test.py b/autogpt_platform/backend/backend/copilot/dream/store_deadline_test.pyindex 90c6cb7f9a..cf56fac710 100644--- a/autogpt_platform/backend/backend/copilot/dream/store_deadline_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/store_deadline_test.py@@ -109,7 +109,7 @@ async def test_a_stalled_insert_costs_the_deadline(deadline, rpc_over, caplog):started = loop.time()with caplog.at_level(logging.WARNING, logger=store.logger.name):- await store.start_pass(+ confirmed = await store.start_pass(DreamPassRun.begin("u1", "sync_baseline"),MemoryScope.for_user("u1"),trigger="cron",@@ -117,6 +117,7 @@ async def test_a_stalled_insert_costs_the_deadline(deadline, rpc_over, caplog):assert loop.time() - started < _BOUNDassert hanging.cancelled == 1+ assert confirmed is False, "an insert that may land later is not confirmed"assert "could not insert its record" in caplog.textdiff --git a/autogpt_platform/backend/backend/copilot/dream/store_test.py b/autogpt_platform/backend/backend/copilot/dream/store_test.pyindex 2df3a03abe..6893d642e1 100644--- a/autogpt_platform/backend/backend/copilot/dream/store_test.py+++ b/autogpt_platform/backend/backend/copilot/dream/store_test.py@@ -144,7 +144,7 @@ class TestWrites:):scope = MemoryScope.for_expert("u1", "expert-1")- await store.start_pass(_run(route), scope, trigger=trigger)+ assert await store.start_pass(_run(route), scope, trigger=trigger)db.create_dream_pass.assert_awaited_once_with(DreamPassDraft(@@ -162,10 +162,17 @@ class TestWrites:))- async def test_gathered_records_the_window_and_moves_to_consolidate(self, db):- bundle = _bundle()+ async def test_gathered_records_the_window_and_what_was_read(self, db):+ """Every fact and episode the bundle holds, sorted: a hard forget+ marks the pass by them once it has closed."""+ bundle = _bundle().model_copy(+ update={+ "known_fact_uuids": {"f2", "f1"},+ "known_episode_uuids": {"e1"},+ }+ )- await store.record_gathered("p1", bundle)+ assert await store.record_gathered("p1", bundle)assert _update(db) == ("p1",@@ -173,9 +180,19 @@ class TestWrites:phase=DreamPassPhase.CONSOLIDATE,window_start=bundle.window_start,window_end=bundle.window_end,+ read_uuids=["e1", "f1", "f2"],),)+ @pytest.mark.parametrize("written, said", [(False, False), (None, None)])+ async def test_gathered_says_whether_it_landed(self, db, written, said):+ if written is None:+ db.update_dream_pass.side_effect = ConnectionError("db down")+ else:+ db.update_dream_pass.return_value = written++ assert await store.record_gathered("p1", _bundle()) is said+@pytest.mark.parametrize("phase, output, next_step",[@@ -382,15 +399,16 @@ class TestAFailedWriteNeverFailsThePass:for r in caplog.records)- async def test_an_insert_that_raises_is_logged_at_warning(self, db, caplog):+ async def test_an_insert_that_raises_is_unconfirmed_and_logged(self, db, caplog):db.create_dream_pass.side_effect = ConnectionError("db down")with caplog.at_level(logging.WARNING, logger=store.logger.name):- await store.start_pass(+ confirmed = await store.start_pass(_run("sync_baseline"), MemoryScope.for_user("u1"), trigger="cron")- assert "could not insert" in caplog.text+ assert confirmed is False+ assert "could not insert its record; the pass stops" in caplog.textasync def test_an_unreachable_accessor_is_logged_at_warning(self, mocker, caplog):mocker.patch.object(store, "dream_db", side_effect=RuntimeError("no rpc"))@@ -612,7 +630,12 @@ class TestReadSide:async def test_a_sync_outcome_reads_back_as_the_result_it_recorded(self, db):"""What the orchestrator returned is what the eval driver reads."""result = _complete_result(- dropped_forgotten=2, pending_writes=3, failed_writes=4+ dropped_forgotten=2,+ pending_writes=3,+ failed_writes=4,+ uncited_writes_dropped=2,+ cross_scope_citations_dropped=3,+ provenance_pending=5,)await store.record_sync_outcome(result)_, update = _update(db)diff --git a/autogpt_platform/backend/backend/copilot/graphiti/AGENTS.md b/autogpt_platform/backend/backend/copilot/graphiti/AGENTS.mdindex 3bbbff9f9b..ba14967d31 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/AGENTS.md+++ b/autogpt_platform/backend/backend/copilot/graphiti/AGENTS.md@@ -23,13 +23,14 @@ Runtime prompt instructions live in `prompting.py:get_graphiti_supplement()`.- Keep warm-context and tool-driven recall resilient: failures should degrade gracefully rather than break chat execution.- Warm context is shown, never stored. Each turn's `<temporal_context>` block (the first turn's `context.fetch_warm_context` and every follow-up's refresh, `context_refresh.py`) goes into that turn's model input only, stamped with the per-process mark in `context_marker.py`. It is never written into the stored user message (`inject_user_context` takes no memory) nor into the transcript either engine records, and the SDK engine scrubs it, by that mark, from the CLI session file before upload; a block that cannot be marked is not injected. So a later turn sees memory only through its own read: a qualifying follow-up (three signal units; on the SDK engine also any turn whose query build compacted the history) gets a fresh block, a trivial one ("ok, do it") gets none, and every retry reads memory again, a retry of a first turn included. That is the trade-off: turn 2 onward no longer sees turn 1's block through the history. Before this, the SDK engine stored the first turn's block in the session's first message, wrapped in `<memory_context>` (from #12790). One matcher, `copilot/legacy_first_turn_memory.py`, proves before anything removes it that a block is one a renderer could have written, where the platform put it: the wrapper as written, around a body (`copilot/legacy_first_turn_memory_body.py`) whose fact lines all end in one renderer's validity or retirement stamp, whose episode lines each open on a `[created_at]` and hold a body no longer than the renderer's cut, and whose every time is `str()` of a real datetime; anything else, a block a user wrote included, is left as it is. Text alone cannot prove who wrote it: a byte-for-byte copy of such a block at that position, in an imported row or one that bypassed the inbound sanitizer (which removes the tags from what a user sends), is removed too. Model input never reads a proved block, whatever the storage holds: every reader that turns stored messages into model input or a tool's output (history rebuilt from the database on both engines, the transcript seeded from it, a baseline turn's current message when the turn stored none, the session title, the action supervisor's prompt, `get_sub_session_result`'s progress, the dream's session bodies) reads the session's first message through `without_stored_first_turn_memory`. `copilot/first_turn_memory_backfill.py` removes the block from the stored rows (dry run by default), the storage cleanup. A CLI session file uploaded for such a session recorded it too, and every restore goes through `download_transcript` (`copilot/legacy_session_file.py`): the block is stripped from the file's first user entry, and a file that holds it where it cannot be stripped (behind a pending message folded into the first query, or in a `<conversation_history>` rebuilt from the database) is not resumed from at all, so the turn rebuilds from the database and its upload replaces the file. The same text a user typed or pasted anywhere else is theirs, and neither stripped nor a reason to drop the file. The system prompt tells the model how to read the marked block: the in-code fallback in `copilot/service.py` does, and the Langfuse prompt production reads (`ChatConfig.langfuse_prompt_name`) has to say the same.- Recall goes through `recall.py` (what may be read back) and `recall_render.py` (how it is written out). Read facts with `search_facts` and episodes with `recent_episodes`; what the assistant is shown is read again by uuid as the last graph read before rendering (`search_facts` ends with `live_now`, and warm context and `memory_search`, which hold both lists while the slower read finishes, call `recall_recheck.recheck`). Ingestion hands graphiti `recall_ingest.previous_episode_uuids`. Code with its own Cypher uses the predicates there: `live_fact_predicate` for facts (the settings and admin fact lists, the dream's demotion writers), `forgotten_facts_clause` with `recallable_episode_predicate` for episodes (the dream gather and its session filter, the forget's redaction).- - A fact is live while `expired_at` and `forgotten_at` are unset and `status` is `active`, `tentative` or unset. It is forgotten once a forget has stamped `forgotten_at`. The marker is written only by a forget (`recall_forget.py`, and the legacy-forget backfill for older forgets), and no ingestion, dream demotion or other status change can make a forgotten fact look remembered, nor a remembered one look forgotten: a `status` of `retracted` or an `expiration_reason` of `user_signal` is no forget without the marker, since graphiti's attribute extraction fills both from an episode's text and a dream demotion picks its reason. The extraction schema (`types.MemoryFact`) leaves both out, and an extracted `retracted` is put back to `active` as the episode is written (`ingest._undo_extracted_retractions`). Nor can extraction write the marker: graphiti-core 0.30.2 saves every key its attribute model answers, so `recall_ingest.ForgetAwareLLMClient` keeps only what the edge or entity type declares (`types.extracted_attributes`), and no extraction model declares the marker or the audit copies (`types.POLICY_EDGE_FIELDS`). Forgets made before the marker are recognised by the pre-policy shape (`expired_at` set with no `invalid_at` and no reason). `migrations/backfill_legacy_forgets.py` restamps that shape, after which its clause can go. It restamps each graph holding the graph's write lock (below), so an ingestion running meanwhile cannot save an older copy of an edge over the restamp; a graph whose lock another writer keeps past 60 s, or that cannot be locked because Redis is unreachable, is skipped with nothing written and counted busy, a graph whose backfill raises is counted failed, and either makes the script exit 1 so it is run again. Drop the clause only after an `--apply` run that exits 0 everywhere (a dry run exits 0 too, having written nothing). An episode is recallable while it has no `redacted_at` and none of its `entity_edges` is forgotten.- - What a forget guarantees: the fact and every episode it came from are hidden from warm context (a session's first turn and every follow-up turn's refresh, `context.py` and `context_refresh.py`; no turn stores its block, `context_marker.py`), `memory_search`, normal `memory_forget_search` calls (without `include_forgotten`), the settings fact list and counts, the dream's input (its episodes and facts, and every chat session a hidden episode came from, left out whole) and ingestion's extraction context (the earlier episodes graphiti shows its prompts). A read already under way when the forget answers does not show it either, once its last check has run: right before rendering, warm context and `memory_search` check every fact and episode they are about to show again, by uuid, in one statement (`recall_recheck.recheck`), and `memory_forget_search` its facts (`search_facts` ends with `live_now`). Every item shown passed that check, its own last graph read. A forget writes each fact's marker before anything else, so a check that began after the marker committed drops the fact and every episode citing it; a response whose check began earlier can still carry them while it is delivered (rendering, scheduling, network), and no later read will. The sentence also leaves what graphiti keeps for its own prompts (`recall_hide.py`). The edge's `fact` and `name` read `[forgotten]` (`recall.FORGOTTEN_FACT`), the originals moved to `fact_redacted` / `name_redacted`; no edge type has that name, so graphiti never runs its attribute prompt, which lists an edge's stored properties, on a forgotten edge. Every entity the fact joins, and every entity an episode citing it mentions, loses its `summary` and every property but its identity (uuid, name, group_id, labels, created_at, name_embedding): typed attributes such as `Person.role` and any `attributes` map go. The `summary` of each community one of them belongs to is blanked too. They are blanked, not rewritten: graphiti extracts them again from the next episode that mentions the entity, and the weekly rebuild restores the communities.- - What it does not: admin audit surfaces (status `any`, the graph view) keep showing retracted facts, with the text from `fact_redacted`, and redacted episodes, on purpose, and show a tombstone's title and `hard_deleted_at` (never its content). `memory_forget_search` with `include_forgotten` returns only redacted placeholders and metadata for forgotten facts, so a soft forget can be made permanent with a hard one, but never what they said: each is `[forgotten]` with its uuid and when it was stored and forgotten (each read as a timestamp and written out anew, else `unknown`), without its text, audit copy, scope or valid dates. `recall.search_forgotten_facts` finds them by the embedding a forget keeps, ranking the forgotten facts alone, so no live fact takes their slots. Not scrubbed: community names, the fact's embedding vector, and a redacted episode's name and source description. A hard forget keeps, redacted, the text of an episode another edge still cites, and a tombstone keeps its name (for a stored memory, the model's short title). Two windows have no bound. When Redis cannot be reached, forgets and ingestion go ahead without the write lock (below); and a holder that loses its lease (its renewals fail, or its event loop stalls past the five-minute expiry) keeps writing while another writer takes the lock, since the renewal only warns and nothing fences the graph write. Either way an ingestion that read a fact before a forget of it can save its older copy over the forget. On a soft-forgotten fact the driver saves nothing but its episode list (`fact_attributes.py`): the fact stays forgotten, scrubbed and out of recall, and a later statement of it is a new fact. But graphiti's `add_episode` saves the fact's entities whole, so they get back the summaries and attributes the forget blanked, and a hard forget deletes the edge, so the save writes the fact again, live, with its sentence. Either stays until the fact is forgotten again: Redis coming back repairs nothing. A marker older builds left on unscrubbed text shows up in `migrations/backfill_legacy_forgets.py --scan-markers`, and forgetting that fact again scrubs it. And order is by when an episode is written, not when it was said, by design: an episode still queued when a forget lands is written after it, and if it states the fact again it teaches it again, as a message sent after the forget would. The weekly community rebuild does not take the lock either, so one running while a forget lands can write a community summary from entity summaries the forget then blanks. Nor does the settings page's erase-all, which deletes the whole graph: an ingestion running meanwhile can save into the emptied graph what it read before the erase. And a forget does not reach what a chat session already holds besides warm context, a boundary proposed as a known limit that still needs the owner's acceptance: the assistant's own earlier replies and tool results (a `memory_search` result, say) in a session that saw the fact keep it, and so do a compaction summary written from a turn's input, its block included (the CLI's own, or the history compression's), Langfuse traces of the turns that sent it, and, within one turn, the block that turn already sent its provider, which the model reads until the turn ends whatever a `memory_forget` in it does, and a building-mode restart after such a forget: it resumes the native CLI session, which still holds the turn's query and its block until the upload scrub runs at the end of the turn.- - Every forget goes through `recall_forget.retract`, which holds the graph's write lock (`scope_lock.py`) for the whole forget: if an ingestion holds it for all of `FORGET_LOCK_WAIT_SECONDS` (20 s), every uuid fails as `busy` and nothing is written; the chat tool waits five seconds and tries once more, and the settings page answers 409. Soft stamps `forgotten_at`, sets `status='retracted'` and `expiration_reason`, keeps an earlier `expired_at`, leaves `invalid_at` alone, then scrubs the sentence, entities and communities and stamps `redacted_at` on every episode citing the fact (`recall_hide.py`). An audit copy, once written, is never replaced by the placeholder. Hard does all of that first, then (`recall_orphans.py`) turns each episode no other edge of any status cites into a tombstone (`content` emptied, `hard_deleted_at` and `redacted_at` set, its name, source description and envelope `provenance` kept, so the dream still leaves its chat session out), and last, in one query per edge, deletes the edge, drops the tombstones' mentions and deletes each entity left with no fact and no mention. Every step before the delete is idempotent and the delete finds the edge again, so repeating a forget whose clean-up failed finishes it. A step that fails is reported per edge as `cleanup_error`; recall hides the text regardless. A tombstone is never recalled, counted on the settings page or used as extraction context.+ - A fact is live while `expired_at` and `forgotten_at` are unset and `status` is `active`, `tentative` or unset. It is forgotten once a forget has stamped `forgotten_at`. The marker is written only by a forget (`recall_forget.py` and the cascade it runs on what the dream derived from the fact, `recall_cascade.py`; the legacy-forget backfill and the derivation backfill's cascade for older forgets), and no ingestion, dream demotion or other status change can make a forgotten fact look remembered, nor a remembered one look forgotten: a `status` of `retracted` or an `expiration_reason` of `user_signal` is no forget without the marker, since graphiti's attribute extraction fills both from an episode's text and a dream demotion picks its reason. The extraction schema (`types.MemoryFact`) leaves both out, and an extracted `retracted` is put back to `active` as the episode is written (`ingest._undo_extracted_retractions`). Nor can extraction write the marker: graphiti-core 0.30.2 saves every key its attribute model answers, so `recall_ingest.ForgetAwareLLMClient` keeps only what the edge or entity type declares (`types.extracted_attributes`), and no extraction model declares the marker or the audit copies (`types.POLICY_EDGE_FIELDS`). Forgets made before the marker are recognised by the pre-policy shape (`expired_at` set with no `invalid_at` and no reason). `migrations/backfill_legacy_forgets.py` restamps that shape, after which its clause can go. It restamps each graph holding the graph's write lock (below), so an ingestion running meanwhile cannot save an older copy of an edge over the restamp; a graph whose lock another writer keeps past 60 s, or that cannot be locked because Redis is unreachable, is skipped with nothing written and counted busy, a graph whose backfill raises is counted failed, and either makes the script exit 1 so it is run again. Drop the clause only after an `--apply` run that exits 0 everywhere (a dry run exits 0 too, having written nothing). An episode is recallable while it has no `redacted_at`, none of its `entity_edges` is forgotten and it is not `write_pending` (a dream write's episode graphiti has not saved yet, below). `recall.forgotten_fact_predicate` is never null (it only asks whether properties are set), so `NOT` it is the not-forgotten test on any edge, a live one included.+ - What a forget guarantees: the fact and every episode it came from are hidden from warm context (a session's first turn and every follow-up turn's refresh, `context.py` and `context_refresh.py`; no turn stores its block, `context_marker.py`), `memory_search`, normal `memory_forget_search` calls (without `include_forgotten`), the settings fact list and counts, the dream's input (its episodes and facts, and every chat session a hidden episode came from, left out whole) and ingestion's extraction context (the earlier episodes graphiti shows its prompts). So is every fact the dream derived from it, and the dream text that derived it, level by level (the cascade, below). A read already under way when the forget answers does not show it either, once its last check has run: right before rendering, warm context and `memory_search` check every fact and episode they are about to show again, by uuid, in one statement (`recall_recheck.recheck`), and `memory_forget_search` its facts (`search_facts` ends with `live_now`). Every item shown passed that check, its own last graph read. A forget writes each fact's marker before anything else, so a check that began after the marker committed drops the fact and every episode citing it; a response whose check began earlier can still carry them while it is delivered (rendering, scheduling, network), and no later read will. The sentence also leaves what graphiti keeps for its own prompts (`recall_hide.py`). The edge's `fact` and `name` read `[forgotten]` (`recall.FORGOTTEN_FACT`), the originals moved to `fact_redacted` / `name_redacted`; no edge type has that name, so graphiti never runs its attribute prompt, which lists an edge's stored properties, on a forgotten edge. Every entity the fact joins, and every entity an episode citing it mentions, loses its `summary` and every property but its identity (uuid, name, group_id, labels, created_at, name_embedding): typed attributes such as `Person.role` and any `attributes` map go. The `summary` of each community one of them belongs to is blanked too. They are blanked, not rewritten: graphiti extracts them again from the next episode that mentions the entity, and the weekly rebuild restores the communities.+ - What it does not: admin audit surfaces (status `any`, the graph view) keep showing retracted facts, with the text from `fact_redacted` (blank for a derived fact a hard forget erased), and redacted episodes, on purpose, and show a tombstone's title and `hard_deleted_at` (never its content). `memory_forget_search` with `include_forgotten` returns only redacted placeholders and metadata for forgotten facts, so a soft forget can be made permanent with a hard one, but never what they said: each is `[forgotten]` with its uuid and when it was stored and forgotten (each read as a timestamp and written out anew, else `unknown`), without its text, audit copy, scope or valid dates. `recall.search_forgotten_facts` finds them by the embedding a forget keeps, ranking the forgotten facts alone, so no live fact takes their slots. Not scrubbed: community names, the fact's embedding vector (a hard forget removes a derived fact's), and a redacted episode's name and source description (a hard forget trims a dream episode's to its kind). A hard forget keeps, redacted, the text of an episode another edge still cites, and a tombstone keeps its name (for a stored memory, the model's short title). Two windows have no bound. When Redis cannot be reached, forgets and ingestion go ahead without the write lock (below); and a holder that loses its lease (its renewals fail, or its event loop stalls past the five-minute expiry) keeps writing while another writer takes the lock, since the renewal only warns and nothing fences the graph write. Either way an ingestion that read a fact before a forget of it can save its older copy over the forget. On a soft-forgotten fact the driver saves nothing but its episode list (`fact_attributes.py`): the fact stays forgotten, scrubbed and out of recall, and a later statement of it is a new fact. But graphiti's `add_episode` saves the fact's entities whole, so they get back the summaries and attributes the forget blanked, and a hard forget deletes the edge, so the save writes the fact again, live, with its sentence. Either stays until the fact is forgotten again: Redis coming back repairs nothing. A marker older builds left on unscrubbed text shows up in `migrations/backfill_legacy_forgets.py --scan-markers`, and forgetting that fact again scrubs it. And order is by when an episode is written, not when it was said, by design: an episode still queued when a forget lands is written after it, and if it states the fact again it teaches it again, as a message sent after the forget would. The weekly community rebuild does not take the lock either, so one running while a forget lands can write a community summary from entity summaries the forget then blanks. Nor does the settings page's erase-all, which deletes the whole graph: an ingestion running meanwhile can save into the emptied graph what it read before the erase. And a forget does not reach what a chat session already holds besides warm context, a boundary proposed as a known limit that still needs the owner's acceptance: the assistant's own earlier replies and tool results (a `memory_search` result, say) in a session that saw the fact keep it, and so do a compaction summary written from a turn's input, its block included (the CLI's own, or the history compression's), Langfuse traces of the turns that sent it, and, within one turn, the block that turn already sent its provider, which the model reads until the turn ends whatever a `memory_forget` in it does, and a building-mode restart after such a forget: it resumes the native CLI session, which still holds the turn's query and its block until the upload scrub runs at the end of the turn.+ - Every forget goes through `recall_forget.retract`, which holds the graph's write lock (`scope_lock.py`) for the whole forget: if an ingestion holds it for all of `FORGET_LOCK_WAIT_SECONDS` (20 s), every uuid fails as `busy` and nothing is written; the chat tool waits five seconds and tries once more, and the settings page answers 409. It first completes the dream records still pending in the graph (`recall_reconcile.py`, below). Soft stamps `forgotten_at`, sets `status='retracted'` and `expiration_reason`, keeps an earlier `expired_at`, leaves `invalid_at` alone, then scrubs the sentence, entities and communities and stamps `redacted_at` on every episode citing the fact (`recall_hide.py`); then, still under the lock, it retracts what the dream derived from the forgotten facts (`recall_cascade.py`, below). An audit copy, once written, is never replaced by the placeholder. Hard does all of that first, stamping `hard_forgotten_at` on each fact as it retracts it (so a dream write settling before the purge erases, below), the cascade too (softly, erasing the derived text it reaches, below), then (`recall_orphans.py`) turns each episode no other edge of any status cites into a tombstone (`content` emptied, `hard_deleted_at` and `redacted_at` set, its name, source description and envelope `provenance` kept, so the dream still leaves its chat session out), and last, in one query per edge, deletes the edge, drops the tombstones' mentions and deletes each entity left with no fact and no mention. Every step before the delete is idempotent and the delete finds the edge again, so repeating a forget whose clean-up failed finishes it, a purged fact's cascade included (below). A step that fails is reported per edge as `cleanup_error`; recall hides the text regardless. A tombstone is never recalled, counted on the settings page or used as extraction context.- While Redis answers and the holder's lease holds, a forget and an ingestion never overlap. graphiti's `add_episode` saves the edges and entities it read when it began, so a forget landing mid-episode would be written over. The ingestion worker therefore holds the graph's write lock around `add_episode` and the dream metadata stamp after it (`ingest._write_locked`), waiting up to `INGEST_LOCK_WAIT_SECONDS` (60 s) for the writer holding it (a forget, or an ingestion in another process) to finish; if it cannot, the episode goes to the back of the queue after a pause (one second, doubling with each wait up to 30 s) and waits again, as often as it takes: contention never drops an episode while the process runs, and every third wait a warning names the episode still unwritten. The queue lives in the process: what it holds when the process stops is lost. Waiters take turns: a writer that finds the lock held claims the next turn (the lock's key plus `:next`, renewed at every poll, lapsing two seconds after its waiter stops), and while it holds that claim nobody else takes the lock, the writer that just let it go included, so a worker writing a backlog hands the lock over between two episodes. The lock (`scope_lock.py`) is one Redis key per graph (`MemoryScope.redis_key("write_lock")`), set NX with a token and a five-minute expiry that the holder renews, and released by the dream lock's compare-and-delete script, so a holder whose key expired never frees a newer holder's. When Redis cannot be reached both writers go ahead without it and log a warning (the unbounded windows above). graphiti also resolves each new statement against every edge between the same entities, forgotten ones included: if its model named a forgotten edge a duplicate, graphiti would append the episode to it, hiding the statement under the forget, and a contradiction would stamp its `invalid_at`. So every graphiti client is built with `recall_ingest.ForgetAwareLLMClient` (`client._build_graphiti`), which keeps a forgotten edge (its text is `[forgotten]`) out of graphiti's edge-dedup answers: a statement said again after a forget is saved by graphiti as a new live edge in the same `add_episode`, one merged into a live fact stays merged, and graphiti's model-decided dedup never rewrites a forgotten edge. There is no second extraction and nothing to repair afterwards. One path runs before the model and so past the guard: graphiti's exact-text match reuses an edge whose text equals the new statement's (lower-cased, whitespace collapsed), so a statement that reads `[forgotten]` appends its episode's uuid to the forgotten edge's `episodes`. The edge keeps its marker and audit copies and stays out of recall, and the episode, citing it, is hidden. The guard reads graphiti's prompt, so it fails closed: a candidate printed with the placeholder is forgotten wherever it appears, in the two candidate lists or not, and when the prompt shows the placeholder, anything short of both lists, once each and readable, makes the answer name no edge at all, so the statement is saved as new (at worst a duplicate of a live fact). A prompt whose lists it cannot read is logged at error once per process for each shape, so a graphiti upgrade that changes the prompt shows up before a forgotten edge meets it.- - The dream keeps its own Cypher. Its gather and session filter (`dream/fetch.py`, `dream/hidden_sessions.py`) build on those predicates, and so does its fact gather (`live_fact_predicate`). Its writers only write over live facts: its demotions (`supersede_unless_recalled`) and single-hop neighbour invalidation (`invalidate_entity_direct_neighbors`), both in `guarded_writes.py`, match `live_fact_predicate`, and ratification (`dream/ratification.py`) supersedes with `supersede_unless_recalled` and `expected_status='tentative'` and promotes only a still-tentative, unexpired, unforgotten edge, so none of them can overwrite a forget, and none writes a new fact. Its consolidated facts and proposals are new statements, written minutes or hours after its read, so each goes to the ingestion worker with what it cites (`recall_citations.Citations`; one citing nothing is taken to rest on everything the pass read, and its statement is compared, lower-cased and whitespace-collapsed, with the sentences forgotten facts keep). Under the write lock, right before `add_episode`, a write resting on a fact forgotten or deleted, or on an episode no longer recallable, since the pass read the graph is dropped unwritten, logged at info, and counted in the pass's `dropped_forgotten` (it counts the drops the worker made before apply returned, which is all of them on a drained pass; the batch path does not wait, so its count covers only the drops made before it returned). The pass counts a write consolidated, or a proposal proposed, only once graphiti has saved it: those still queued or being written when apply returns are reported as `pending_writes` (every one on the batch path), and those whose write raised as `failed_writes`. The check trusts the model's citations: a write that cites one source but restates another, forgotten, in other words is not recognised, and an uncited write drawn only from a chat session whose episode and fact the pass did not read (beyond its 50 newest episodes and 500 facts) is recognised only by an exact restatement of a softly forgotten sentence.+ - The dream keeps its own Cypher. Its gather and session filter (`dream/fetch.py`, `dream/hidden_sessions.py`) build on those predicates, and so does its fact gather (`live_fact_predicate`). Its writers only write over live facts: its demotions (`supersede_unless_recalled`) and single-hop neighbour invalidation (`invalidate_entity_direct_neighbors`), both in `guarded_writes.py`, match `live_fact_predicate`, and ratification (`dream/ratification.py`) supersedes with `supersede_unless_recalled` and `expected_status='tentative'` and promotes only a still-tentative, unexpired, unforgotten edge, so none of them can overwrite a forget, and none writes a new fact. Its consolidated facts and proposals are new statements, written minutes or hours after its read, so each goes to the ingestion worker with what it cites (`recall_citations.Citations`). Every one must cite the facts and episodes it was drawn from: apply keeps a citation only when it names something the pass read, filed under the kind the pass read it as, and drops before it is queued a write left citing nothing or citing a fact of another scope than its own (a fact's `scope`, else `real:global`: the prompts' "group by scope" rule, enforced), counting it in `uncited_writes_dropped` and each such citation in `cross_scope_citations_dropped` (`dream/citations.py`; both counts through the apply stats, `DreamPassResult`, the DreamPass record and the job status). The whole write goes, not only its citations of another scope: trimmed, it could restate the other fact and escape that fact's forget. Episode citations are not scoped: a chat turn can hold facts of any scope, and the cascade reaches a fact consolidated from one through `derived_from_episodes` when a forget hides that turn. Under the write lock, right before `add_episode`, a write resting on a forget made since the pass read the graph is dropped unwritten: one citing a fact forgotten or deleted, an episode no longer recallable, or a derived fact no longer live (superseded, contradicted) that rests on one of those. The last is found walking up the records (`derived_from_facts`, `derived_from_episodes`) from each cited derived fact no longer live, as far as derived facts no longer live go, the cascade's walk-through rule upward (`recall_sources.ancestry`): the cascade walks down through such a fact to what rests on it, and the write would rest on it the same way, carrying that content. A live fact, a user's fact (a source without a record states it) or a recallable episode ends a branch clean. The walk is bounded like the cascade, at most `CASCADE_MAX_ROUNDS` (10) levels up and `CASCADE_MAX_ITEMS` (500) facts read, and a walk stopped by either with facts left to read drops the write too (it fails closed). A dropped write is logged at info, and counted in the pass's `dropped_forgotten` (through the same places as `uncited_writes_dropped`; a record written before it reads 0; it counts the drops the worker made before apply returned, which is all of them on a drained pass; the batch path does not wait, so its count covers only the drops made before it returned). The pass counts a write consolidated, or a proposal proposed, only once graphiti has saved it: those still queued or being written when apply returns are reported as `pending_writes` (every one on the batch path), and those not made, their citation marker unwritable or their write raising, as `failed_writes`. The check trusts the model's citations: a write that cites one source but restates another, forgotten, in other words is not recognised, and no check can tell whether a source in the write's own scope truly supports it. (The check still compares the statement of a write citing nothing with the sentences forgotten facts keep, as defence in depth: apply no longer sends one.)- graphiti's extraction seeds a new fact's attributes and never changes a stored fact's (`fact_attributes.py`). graphiti-core 0.30.2 saves a fact with `SET e = $edge_data` (`SET r = edge` in `add_episode`'s bulk save), which removes every property its map leaves out, and when its model merges a statement into a stored fact the map holds only what its attribute model answered, or nothing for an edge whose name is no edge type: every stamp the platform wrote went, and the model's answer set the status afresh. So the driver runs both saves as `+=`, matched by their exact text (graphiti's own fields are always in its map, nulls included, and `+=` removes a property set to null, so they are saved as before; a forgotten fact takes only its episode list, above), and `recall_ingest.ForgetAwareLLMClient` answers no attributes, without asking the model, for a fact whose attribute prompt shows stored ones; a prompt it cannot read counts as a stored fact's and is logged at error once per reason. A fact saved with no attributes at all reads as new. A user's chat turn then promotes each tentative fact it states, new or merged into, to `active` with `ratified_at` (`ingest._promote_restated`), under the same lock: tentative is probation for what the system inferred, and the user saying it ends it. Only the user's message episode carries the `_restatement` sidecar that asks for it (`enqueue_conversation_turn`), so the assistant's finding lane, `memory_store` calls and dream writes promote nothing, and only facts whose `episodes` include the turn's are promoted, so those graphiti invalidated stay as they were. Saying a fact moves no recall stamp or hit count.+ - A forget reaches what the dream derived from what it forgot. Before the ingestion worker makes a dream write (`marked_write.py`), holding the write lock, it marks the write's complete citations in the same graph (`recall_derivation.mark`: a `DreamCitations` node, `pending`, holding the uuid the write's episode is given, the dream episode's name for people reading the graph (never used to find it), `derived_from_facts`, `derived_from_episodes`, the graph's `group_id` and when it was written; uuids and names only, never text), and a marker that cannot be written drops the write (it fails closed, counted in `failed_writes`), so a dream fact's provenance is in the graph before the fact is. The uuid is drawn before the marker and handed to graphiti's `add_episode`: graphiti-core 0.30.2 loads the episode a uuid names instead of creating one, so the worker first places it under that uuid, `write_pending` (hidden from every read by `recall.recallable_episode_predicate`), and graphiti's own save of the episode, a `MERGE` on the uuid that replaces every property, clears it. The write is checked against forgets only once it is marked, and one the check drops has its marker withdrawn. Right after the write, under the same lock, the worker records the write's citations as `derived_from_facts` / `derived_from_episodes` (`recall_derivation.record`, finding the episode by uuid in its graph), settles it (`recall_landing.settle`, below) and only then deletes the marker. A record that fails leaves the marker, notes the graph in the Redis set `graphiti:provenance_pending` and counts the write in `provenance_pending` (both counts go through the apply stats, `DreamPassResult`, the DreamPass record, both routes and the job status); an `add_episode` that raised marks the marker `aborted` and notes the graph. Every forget completes the graph's markers before anything else (`recall_reconcile.reconcile`, landed ones first, at most 200 a call): one whose write landed (graphiti saved its episode and every fact the episode lists) is recorded and settled as its writer would have, then deleted; an aborted one is completed if graphiti saved its episode, even in part, and otherwise deleted with the placed episode, by a statement that first confirms no saved episode has its uuid; a pending one that has not landed waits, and for a day (`MARKER_EXPIRY_SECONDS`) a forget of anything it cites (a root, a derived fact the cascade retracted or walked through, an episode it hid) reports `cleanup_error` (`recall_forget_writes.py`), since where the lock does not hold the write can still land after the forget; past the day it is marked `expired`, holds nothing up and is kept, for no marker is deleted for its age alone. A forget whose reconcile fails or leaves a landed marker reports `cleanup_error` too, so forgetting again finishes it. Where the lock does not hold, a forget can also land between a write's check and its record, where its cascade cannot reach the write's facts: so whoever completes a marker, its writer or a reconcile, settles the write first. `recall_landing.settle` looks up every source the write cites and, for each one a forget reached (a fact forgotten, retracted by a cascade or gone; an episode hidden or gone), runs a forget's cascade from it under the root the user forgot, retracting the write's facts and hiding its episode, erasing for what a hard forget reached (a purged fact, a fact a hard forget is still purging, which it stamps `hard_forgotten_at` as it retracts it, a derived fact its cascade erased, an emptied episode). A cited derived fact no longer live that rests on such a source (the same walk up as the check) is cascaded from itself under that source's root: the cascade leaves it as it is and retracts what rests on it, the write's facts among them; one whose walk stopped at its bound is cascaded from under its own name (fail closed). The writer settles from the citations it holds, so a write whose marker is gone (resolved by an operator, or completed by a reconcile that ran beside it) is settled all the same when it lands. A late write is thus retracted a few statements after graphiti saved it: a read in those statements can see it live, and a writer that dies there leaves its marker for the next reconcile (the forget's retry, which its `cleanup_error` asks for, the reaper, the backfill or the operator command) to complete. `poetry run python -m backend.copilot.graphiti.migrations.dream_markers` lists every marker (its state, age and episode uuid, whether graphiti saved the episode and whether the write landed); `--resolve <uuid>` or `--resolve-expired` with `--apply` completes one whose episode was saved and deletes any other with its placed episode, under the graph's lock; only this command resolves an expired marker. It refuses a pending marker younger than `MARKER_EXPIRY_SECONDS`, whose writer may still be writing, unless `--force`, and says what `--force` would do: resolve a pending marker only once its writer is gone. Each dream reaper run sweeps up to 20 noted graphs with what its budget leaves (`provenance_pending.py`, keeping a graph noted while a write in it could still land), and the backfill reconciles each graph it writes. The record lands on the dream's episode, which graphiti never saves again, so it is the lasting record and marks the episode as the dream's; and on every fact the write produced or merged into whose source episodes (`episodes`) all carry a record, as the union of theirs. A fact the write alone produced gets the write's citations; a user's fact a dream write merged into gets none, since it has a source no forget of the dream's citations reaches. A statement merged into a fact leaves its attributes as they were (`fact_attributes.py`, above), its record among them: a later dream write merging into a derived fact whose sources are all the dream's rebuilds the union from the episodes' records, and a user's statement merging into one leaves the record in place, but the fact then names an episode with no record, so the cascade passes it over (below), and extraction can write no record (`types.POLICY_EDGE_FIELDS`). After a forget has retracted its facts and hidden their episodes, still under the lock, `recall_cascade.cascade` retracts every live fact whose record names a forgotten fact or a hidden episode and that no user's episode states (every episode it names carries a record), hides every dream episode whose record does (redacted, the entities it mentions blanked), then repeats over what that retracted and hid until a round finds nothing new. Any one forgotten source is enough. A derived fact no longer live (superseded, contradicted, forgotten already) is left as it is but walked through; one a user's own episode also states is passed over with what rests on it. A derived fact is retracted as a soft forget retracts one (`forgotten_at`, `status='retracted'`, its sentence moved to its audit copy, its entities and communities blanked, the episodes citing it redacted), with the `expiration_reason` `derived_from_forgotten:<uuid>` naming the forgotten fact it descends from. `ForgetResult.derived` lists them, the chat tool's reply counts them (`derived_uuids`) and the settings route returns `derived_forgotten`. A hard forget cascades too, softly (retraction takes the derived facts out of every read, and deleting them would purge whatever the model's citations name, which can be more than a fact rests on, with no way back), and erases the text it reaches (`recall_erase.py`): every derived fact it retracts or walks through reads `[forgotten]` in `fact` and `name` with both audit copies blank and its `fact_embedding` removed (FalkorDB's vector similarity gives an edge without one no score, so graphiti's searches and dedup pass it by), and every dream episode it hides loses its body and the rationale and citations in its `source_description`, which keeps the kind (`dream-pass proposal`). A user's episode it hides keeps its text: only an episode with a dream's record is the dream's. Last, after the graph's lock is released, it blanks the dream's copies of that text in Postgres (`dream/forget_copies.py`). It marks with `erasedAt`, never cleared, every pass of the scope it reached, in one statement over all of the scope's passes, under its own statement timeout just under the store's deadline: one that read a fact or episode it reached (`DreamPass.readUuids`, every one the pass gathered, recorded before its first model call and never cleared; a pass whose row or read set the store does not confirm stops before it reads the graph or calls a model), one whose read set is unknown and so may have read anything (it reads `*`, `UNKNOWN_READS`: every row that existed when the column was added does, and so does a row a release without the column writes during a rollout; this release's create states its read set, empty until the pass gathers), one still open, one closed with its cleanup pending (a stop from outside closes the row while the pass may still be applying), and one whose dream write it erased (`ForgetResult.dream_writes`, the episodes' names naming their pass: a pass does not read its own writes). A marked pass that is closed has every sentence (`content`, `rationale`, `reason`, `summary_for_user`) in its phase outputs and operations blanked there and then; an open one keeps them until it closes, and the statement that closes it, however it closes (its own end, a cancel, an expiry, the reaper's close, the cleanup's end), blanks them (`data/dream_pass_update.py`), the reaper's expiry being the backstop for a pass that never ends. The narrative each pass marked (now or by an earlier forget, so repeating a forget whose narrative blank failed finishes it) or named opened its dream session with is blanked, and so is each whose session's copy of its pass's read set holds a reached uuid, its pass's row aged out or not, and each of the forget's scope (the session's `expertId`) whose session keeps no read set (written before sessions kept one) or one holding `*` (the session's other messages are the user's chat with it, which no forget reaches). A narrative goes only into its pass's own dream session and only while the pass's row is there, carrying the read set into the session's metadata, and one written after the mark is written blank, in one statement holding the pass's row (`copilot/db.add_dream_narrative`); apply also writes it blank when one of the pass's writes was dropped for resting on a forget. Every error a pass records is `<stage>: <code>` from fixed sets (`dream/pass_errors.py`; the store records `pass: unrecorded` for any other), and every pass of the scope keeps its error only when it is one. The uuids, scopes, counts and usage stay. A failure of the blanking, its timeout included, is a `cleanup_error` on each fact forgotten, and forgetting again finishes it. One forget retires at most 500 derived facts and dream episodes (`CASCADE_MAX_ITEMS`) over at most 10 rounds (`CASCADE_MAX_ROUNDS`); one stopped by a bound or a failed step reports `cleanup_error` on each fact it forgot, and forgetting them again resumes from the facts already retracted for them (their reason names them). A hard forget whose cascade stops short still purges the facts it forgot, since the user's own text goes whatever happens to the rest. Forgetting a purged uuid again goes on with its cascade from whatever still names it (a derivation record, a pending marker, an earlier cascade's reason, or the `redacted_for` a forget stamps on each episode it hides, which the purge leaves where `entity_edges` loses the fact), erasing, and says so (`ForgetResult.resumed`, the chat tool's `resumed_uuids`, the settings route's `resumed`); a uuid nothing names stays a no match. Every reason keeps naming the root the user forgot. Retracted derived facts fail the live-fact test like any forgotten fact, a dream write citing one (or a hidden dream episode) is dropped by `recall_citations`, and the recall stamp never stamps them. Not reached: a citation the model left out; a dream write whose writer dies between graphiti's save and its settle stays live until the next reconcile in its graph, which only a forget's retry is sure to prompt while its marker is young; one whose marker an operator resolved and whose writer then dies before its record is not reached at all; a dream write merged into a user's fact (its dream text is still hidden); a user's fact a derived fact's contradiction expired (graphiti's invalidation is not undone); after a hard forget, the entities the derived facts join (their names and name embeddings kept, as a soft forget keeps them), and the text a dream pass still running keeps in its row until it ends (above); an older dream write whose citation is a fact a hard forget purged before the backfill ran (a uuid the graph lacks cannot be told from a forged one, so it is rejected; one citing the user's emptied chat turn is reached through that hidden episode); a forged citation shaped exactly as the dream wrote one, naming a real source in the right scope; and dream writes made before the records until the backfill has run. `poetry run python -m backend.copilot.graphiti.migrations.backfill_derivations` records those from each dream episode's `source_description` (at most five uuids of each kind: a consolidation listed its episodes, a proposal its facts). The model's rationale shared that text and its `;` delimiter, so it could forge a citation: the backfill reads a description only in the shapes the dream wrote, its keys ending it, treats one naming a key twice or anywhere before its end as ambiguous (it attributes nothing), and keeps only uuids the graph has, a fact only in the dream episode's own scope (`migrations/legacy_citations.py`); an older write citing a fact of another scope cannot be dropped, so it keeps its other citations and could escape that fact's forget. It reports the ambiguous descriptions, the citations it rejected and the dream facts it could not attribute. A new dream write's description lists no citation (the marker and the records carry them), and the `;` of its rationale is written as `,`. It is a dry run unless `--apply`, which writes each graph under its write lock in batches of 500 and exits 1 while any graph was busy or failed; `--cascade-existing-forgets` also runs the cascade from every root the graph's forgets left (`migrations/backfill_cascade.py`): the facts a user forgot that are still there, softly (one a cascade retracted is picked up from its root); the facts a hard forget purged that something still names, erasing; and the hidden episodes a record names, erasing from those a hard forget emptied, each naming the root it was hidden for.- A recall leaves usage on the facts it returned (`recall_stamp.py`): warm context (`dream.ratification.try_ratify_on_hit`) and `memory_search` (`recall_stamp.record_recall`) stamp each live fact they returned, one batched write per call, with `recall_count`, `last_recalled_at` and `prev_recalled_at`; a recall within 24 hours of the last one is the same use and is not counted. A forgotten or expired fact is never stamped. The stamp takes no write lock: it writes only those three properties, never the envelope, so it cannot undo a forget, and an ingestion that saves an older copy of the edge over it loses one recall and nothing else; a statement merged into the fact keeps them (above). Recall history is only ever evidence that a memory is relied on. Each of the dream's destructive writes tests it in its own statement (`recall_stamp.spared_by_recall`): a live fact last recalled within `Config.dream_demotion_protect_days` (30 by default; 0 turns this guard off) is left alone unless the reason is the user's retraction or a contradiction citing another fact the pass read, and the statement reports what it changed and what it spared, in one row however many facts it reached (`guarded_writes.py`). `dream/demotions.py` counts in `protected_demotions` the distinct facts an acknowledged write spared that one read, made after every acknowledged write, finds live. The count is a snapshot at that read: a later forget or pass can retire a counted fact. A write that raised may have committed, never arrived, or still be queued on FalkorDB (the read, a `GRAPH.RO_QUERY`, does not wait for queued writes), so it is counted in `indeterminate_demotion_writes` rather than as a failure, and `demotion_accounting_complete` is True only when the read answered and no write's outcome is unknown; otherwise the count is provisional. Ratification follows the same rule: a promotion or supersession whose write raised leaves the sweep's counts and the warm-context hit hook's count provisional (`RatificationResult.accounting_complete`, `HitRatification.accounting_complete`), while a per-edge error whose outcome is known does not. Entity invalidation has no degree cap: a hub is invalidated whole, behind the `dream-pass-invalidate-entity` flag, as before recall stamps. The ratification sweep's supersession of an unratified proposal carries the same guard with no override, so a proposal recalled within the window stays tentative even when its Redis hit count was lost (counted in `RatificationResult.protected_count`). Nothing is read beforehand to decide, and usage never changes which operations a pass attempts, so a pass never changes more facts with usage data than without. The sanitize prompt separately asks the model not to demote a recalled fact for staleness at all. No recalls means nothing, and retrieval order ignores the stamps. An edge written before the stamps reads as never recalled; nothing is backfilled.- A live fact whose valid time has ended (a past `invalid_at`, never expired) is still recalled, as history: it renders as `fact (valid: X — Y)`. Only a retired fact gets a retirement label, and "present" appears only while no end is known.diff --git a/autogpt_platform/backend/backend/copilot/graphiti/backfill_derivations_integration_test.py b/autogpt_platform/backend/backend/copilot/graphiti/backfill_derivations_integration_test.pynew file mode 100644index 0000000000..45c267e915--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/backfill_derivations_integration_test.py@@ -0,0 +1,188 @@+"""The derivation backfill on a live FalkorDB, on dream facts written the way+they were before derivation records: no record anywhere, and the citations+only in each dream episode's description, in its old form (a consolidation+listed its episodes, a proposal its facts).++It records them so a later forget reaches them, counts the one it cannot+attribute (a proposal that cited only an episode), and, asked to, cascades+from a forget made before it ran. The script's unit tests are+``migrations/backfill_derivations_test.py``.++Run with FalkorDB reachable (see ``conftest.py``)::++ poetry run pytest -m integration backend/copilot/graphiti/backfill_derivations_integration_test.py+"""++from collections.abc import AsyncIterator+from unittest.mock import AsyncMock++import pytest+import pytest_asyncio+from pytest_mock import MockerFixture++from backend.copilot.dream import apply++from .falkordb_driver import AutoGPTFalkorDriver+from .migrations.backfill_derivations import backfill_graph+from .recall_cascade_fixtures import Bakery, build_bakery, derivation+from .recall_forget import retract+from .recall_integration_fixtures import (+ edge_row,+ live_facts,+ patch_recall_boundaries,+ stop_ingestion_workers,+)++[email protected](autouse=True)+def boundaries(mocker, scope_graph, stub_graphiti_client):+ patch_recall_boundaries(mocker, scope_graph[0], stub_graphiti_client)+++@pytest_asyncio.fixture(loop_scope="function")+async def dream_apply(mocker: MockerFixture) -> AsyncIterator[None]:+ mocker.patch.object(apply, "_create_dream_session", AsyncMock(return_value="s"))+ mocker.patch.object(apply, "_write_dream_summary_message", AsyncMock())+ mocker.patch(+ "backend.copilot.dream.registry.ensure_dream_system_scheduled",+ AsyncMock(return_value=None),+ )+ yield+ await stop_ingestion_workers()+++async def _as_before_records(driver: AutoGPTFalkorDriver, bakery: Bakery) -> None:+ """Take every record away and give each dream episode the description+ the dream wrote before records existed."""+ await driver.execute_query(+ "MATCH (ep:Episodic) "+ "SET ep.derived_from_facts = NULL, ep.derived_from_episodes = NULL"+ )+ await driver.execute_query(+ "MATCH ()-[e:RELATES_TO]->() "+ "SET e.derived_from_facts = NULL, e.derived_from_episodes = NULL"+ )+ described = {+ bakery.supplies_episode: (+ f"dream-pass consolidation; src_episodes={bakery.said}"+ ),+ bakery.boule_flour_episode: (+ "dream-pass proposal; rationale=the bakery's flour; "+ f"src_facts={bakery.supplies},{bakery.boule}"+ ),+ bakery.weekly_episode: "dream-pass proposal; rationale=the bakery's flour",+ }+ for uuid, description in described.items():+ await driver.execute_query(+ "MATCH (ep:Episodic {uuid: $uuid}) SET ep.source_description = $d",+ uuid=uuid,+ d=description,+ )+++async def test_a_dry_run_counts_and_writes_nothing(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ driver, scope = scope_graph+ bakery = await build_bakery(driver, scope, stub_graphiti_client)+ await _as_before_records(driver, bakery)++ found = await backfill_graph(driver, apply=False, cascade_forgets=True)++ assert (found.episodes, found.facts, found.unattributed) == (3, 3, 1)+ assert (found.roots, found.derived) == (0, 0)+ assert (await derivation(driver, bakery.supplies))["facts"] is None+ assert (await derivation(driver, bakery.supplies_episode))["facts"] is None+++async def test_apply_records_them_so_a_later_forget_reaches_them(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ driver, scope = scope_graph+ bakery = await build_bakery(driver, scope, stub_graphiti_client)+ await _as_before_records(driver, bakery)++ found = await backfill_graph(driver, apply=True, cascade_forgets=False)+ again = await backfill_graph(driver, apply=True, cascade_forgets=False)++ assert (found.episodes, found.facts, found.unattributed) == (3, 3, 1)+ assert (again.episodes, again.facts, again.busy) == (0, 0, 0), "idempotent"+ supplies = await derivation(driver, bakery.supplies)+ assert (supplies["facts"], supplies["episodes"]) == ([], [bakery.said])+ boule_flour = await derivation(driver, bakery.boule_flour)+ assert boule_flour["facts"] == [bakery.supplies, bakery.boule]+ result = await retract(scope, [bakery.flour])+ assert sorted(result.derived) == sorted([bakery.supplies, bakery.boule_flour])+ assert bakery.weekly in await live_facts(driver), "cited an episode only"+++async def test_it_cascades_from_a_forget_made_before_it_ran(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ driver, scope = scope_graph+ bakery = await build_bakery(driver, scope, stub_graphiti_client)+ await _as_before_records(driver, bakery)+ earlier = await retract(scope, [bakery.flour])+ assert earlier.derived == [], "nothing recorded to follow yet"++ found = await backfill_graph(driver, apply=True, cascade_forgets=True)++ # The forgotten fact, and the chat turn it hid, which a record now names.+ assert (found.roots, found.derived, found.failed) == (2, 2, 0)+ live = set(await live_facts(driver))+ assert live == {bakery.boule, bakery.cafe, bakery.weekly}+ for uuid in (bakery.supplies, bakery.boule_flour):+ row = await edge_row(driver, uuid)+ assert row["reason"] == f"derived_from_forgotten:{bakery.flour}"+++async def test_a_forged_or_ambiguous_description_attributes_nothing(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ """Codex's probe on older dream text: a rationale that forges a citation+ of a uuid the graph lacks, one naming a key twice, and a project-scoped+ write citing the global cafe fact. The parser that took the last key+ linked each to what it named; now none is attributed, and forgetting the+ cafe retracts nothing."""+ driver, scope = scope_graph+ bakery = await build_bakery(driver, scope, stub_graphiti_client)+ await _as_before_records(driver, bakery)+ ghost = "11111111-2222-3333-4444-555555555555"+ forged = {+ bakery.supplies_episode: (+ f"dream-pass proposal; rationale=ordinary; src_facts={ghost}"+ ),+ bakery.boule_flour_episode: (+ "dream-pass proposal; rationale=ordinary; "+ f"src_facts={bakery.cafe}; src_facts={bakery.supplies}"+ ),+ bakery.weekly_episode: (+ f"dream-pass proposal; rationale=weekly; src_facts={bakery.cafe}"+ ),+ }+ for uuid, description in forged.items():+ await driver.execute_query(+ "MATCH (ep:Episodic {uuid: $uuid}) SET ep.source_description = $d",+ uuid=uuid,+ d=description,+ )+ await driver.execute_query(+ "MATCH (ep:Episodic {uuid: $uuid}) SET ep.content = $content",+ uuid=bakery.weekly_episode,+ content='{"content": "orders weekly", "scope": "project:ordering"}',+ )++ found = await backfill_graph(driver, apply=True, cascade_forgets=False)++ assert (found.ambiguous, found.rejected) == (1, 2)+ for uuid in (bakery.supplies, bakery.boule_flour, bakery.weekly):+ assert (await derivation(driver, uuid))["facts"] == [], uuid+ assert (await retract(scope, [bakery.cafe])).derived == []diff --git a/autogpt_platform/backend/backend/copilot/graphiti/conftest.py b/autogpt_platform/backend/backend/copilot/graphiti/conftest.pyindex f46d4c7640..fd219b56b6 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/conftest.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/conftest.py@@ -36,6 +36,7 @@ from collections.abc import Awaitable, Iterablefrom datetime import datetime, timezonefrom pathlib import Pathfrom typing import AsyncIterator, cast+from unittest.mock import AsyncMockimport pytestimport pytest_asyncio@@ -49,7 +50,7 @@ from pydantic import BaseModelfrom pytest_mock import MockerFixturefrom redis.asyncio import Redis-from . import scope_lock+from . import recall_forget, scope_lockfrom .client import _build_graphitifrom .config import graphiti_configfrom .falkordb_driver import AutoGPTFalkorDriver@@ -150,6 +151,16 @@ def lock_redis(mocker: MockerFixture) -> FakeRedis:return redis[email protected](autouse=True)+def dream_copies(mocker: MockerFixture) -> AsyncMock:+ """No forget here reaches Postgres: a hard forget's blanking of the+ dream's copies (``dream/forget_copies.py``) does nothing and marks no+ pass. Tests of that step read or set this mock."""+ blank = mocker.AsyncMock(return_value=[])+ mocker.patch.object(recall_forget, "blank_dream_copies", blank)+ return blank++@pytest_asyncio.fixture(loop_scope="function")async def live_lock(falkordb_available: bool, mocker: MockerFixturediff --git a/autogpt_platform/backend/backend/copilot/graphiti/dream_ratification_integration_test.py b/autogpt_platform/backend/backend/copilot/graphiti/dream_ratification_integration_test.pyindex 949fe54835..cb58d08e05 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/dream_ratification_integration_test.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/dream_ratification_integration_test.py@@ -51,6 +51,7 @@ from . import ingest as ingest_modfrom .client import derive_group_idfrom .config import graphiti_configfrom .falkordb_driver import AutoGPTFalkorDriver+from .recall_citations import Citationsfrom .recall_stamp import stamp_recalls, stamp_timefrom .scope import MemoryScope@@ -261,6 +262,9 @@ async def _ingest_dream_proposal(user_id: str) -> None:PROPOSAL,session_id=SESSION_ID,completion=completion,+ # What it cites is beside the point here; the citation checks and the+ # cascade are ``recall_cascade_integration_test.py``.+ citations=Citations(),)assert queued, "proposal was dropped before reaching the ingestion queue"completion.register()diff --git a/autogpt_platform/backend/backend/copilot/graphiti/fact_attributes_integration_test.py b/autogpt_platform/backend/backend/copilot/graphiti/fact_attributes_integration_test.pyindex 92dac2823c..c7a7426629 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/fact_attributes_integration_test.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/fact_attributes_integration_test.py@@ -56,8 +56,9 @@ _PARAPHRASE: Fact = ("Alice", "Atlas", "Alice is working on Atlas")_TENTATIVE = {"status": "tentative", "scope": "real:global"}# What the platform writes on a live fact outside graphiti's extraction: the-# recall stamps, the ratification and web-verification stamps, and the-# dream's envelope metadata (``ingest._stamp_edge_metadata``).+# recall stamps, the ratification and web-verification stamps, the dream's+# envelope metadata (``ingest._stamp_edge_metadata``) and its derivation+# record (``recall_derivation.record``).STAMPS: dict[str, Any] = {"recall_count": 3,"last_recalled_at": "2026-05-01T00:00:00.000000+00:00",@@ -69,6 +70,8 @@ STAMPS: dict[str, Any] = {"confidence": 0.9,"provenance": "dream:pass-1","scope": "project:atlas",+ "derived_from_facts": ["fact-cited"],+ "derived_from_episodes": ["episode-cited"],}diff --git a/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver.py b/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver.pyindex 065375f6fe..9543788d1f 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver.py@@ -411,7 +411,17 @@ def open_driver((``falkordb_connect.DeferredFalkorDB``), soit can be opened anywhere, the event loop included."""- return AutoGPTFalkorDriver(database=scope.group_id, build_indices=build_indices)+ return open_graph_driver(scope.group_id, build_indices=build_indices)+++def open_graph_driver(+ database: str, *, build_indices: bool = False+) -> AutoGPTFalkorDriver:+ """Open a Cypher driver on the graph named ``database``; the caller must+ close it. For code that knows a graph only by its name (an expert+ graph's name is a digest no scope can be rebuilt from): a migration, or+ a sweep over graphs noted in Redis. Opening one creates no graph."""+ return AutoGPTFalkorDriver(database=database, build_indices=build_indices)async def connect_driver(diff --git a/autogpt_platform/backend/backend/copilot/graphiti/ingest.py b/autogpt_platform/backend/backend/copilot/graphiti/ingest.pyindex 8eda2a61da..0893fe4b7b 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/ingest.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/ingest.py@@ -19,6 +19,7 @@ from backend.copilot.dream.ratification import _promote_if_tentativefrom backend.copilot.dream.registry import ensure_scope_scheduledfrom backend.util.background import spawn_background_task+from . import marked_writefrom .client import ensure_indices_once, get_graphiti_clientfrom .memory_model import MemoryEnvelope, MemoryKind, MemoryStatus, SourceKindfrom .recall_citations import Citations, rests_on_a_forget@@ -112,6 +113,7 @@ class IngestionCompletion:"written","dropped_forgotten","failed",+ "provenance_pending",)def __init__(self) -> None:@@ -124,8 +126,13 @@ class IngestionCompletion:# Episodes the worker dropped, unwritten, for resting on a forget# made since they were queued (``recall_citations.py``).self.dropped_forgotten = 0- # Episodes whose write raised: dropped, logged, not retried.+ # Episodes the worker did not write: a dream write whose citation+ # marker could not be written (it fails closed), a graph write that+ # raised, a payload for another graph.self.failed = 0+ # Dream writes made whose derivation record failed after the write:+ # their marker stays until it is reconciled (``recall_reconcile.py``).+ self.provenance_pending = 0@propertydef registered(self) -> int:@@ -403,26 +410,52 @@ async def _write_locked("""Write one episode holding the graph's write lock (``scope_lock.py``),so no forget lands between what graphiti reads and what it saves overit; False, writing nothing, when another writer kept it for the wait.- A dream write resting on a forget is dropped under the same lock, and a- user's chat turn promotes the tentative facts it states under it- (``_promote_restated``)."""+ A dream write has its citations marked first, failing closed; one+ resting on a forget is then dropped under the same lock, its marker+ withdrawn, as is the marker of one whose check raised; one that is+ written is recorded after it+ (``marked_write.py``), so a forget waiting for the lock finds them. A+ user's chat turn promotes the tentative facts it states under the same+ lock (``_promote_restated``)."""wait = INGEST_LOCK_WAIT_SECONDSasync with graph_write_lock(group_id, wait_seconds=wait) as lock:if lock is LockState.BUSY:return Falseclient = await get_graphiti_client(group_id)- if await _dropped_as_forgotten(client, payload, completion):- return True+ # Sidecars (not add_episode kwargs): the citations and the edge+ # metadata come only with a dream write, the restatement only with a+ # chat turn's user-message episode.+ citations: Citations | None = payload.pop("_citations", None)+ edge_metadata = payload.pop("_edge_metadata", None)+ restatement = payload.pop("_restatement", False)# This is the write path, so materializing the graph is intended# here — unlike driver construction, which must never create one.# Once per group per loop.await ensure_indices_once(group_id, client)- # ``_edge_metadata`` is a sidecar (not an add_episode kwarg) — pop it- # before the **payload spread. Present only for dream writes.- edge_metadata = payload.pop("_edge_metadata", None)- # Sidecar, set only on a chat turn's user-message episode.- restatement = payload.pop("_restatement", False)- result = await _add_episode(client, group_id, payload)+ marked = await marked_write.marked(+ client, group_id, payload, citations, completion+ )+ if citations is not None and marked is None:+ return True+ try:+ dropped = await _dropped_as_forgotten(+ client, payload, citations, completion+ )+ except Exception:+ # Nothing is placed yet. Left pending, the marker would hold up+ # the forgets of what it cites until it expired.+ await marked_write.withdrawn(client.driver, marked)+ raise+ if dropped:+ await marked_write.withdrawn(client.driver, marked)+ return True+ episode = marked.episode if marked is not None else None+ result = await marked_write.added(+ _add_episode(client, group_id, payload, episode),+ client.driver,+ group_id,+ marked,+ )await _undo_extracted_retractions(client, result)if restatement:await _promote_restated(client, result)@@ -432,11 +465,20 @@ async def _write_locked(# ``_stamp_edge_metadata`` for the dedup-safety invariant).if edge_metadata:await _stamp_edge_metadata(client, group_id, result, edge_metadata, user_id)+ if citations is not None and marked is not None:+ await marked_write.recorded(+ client, group_id, marked, result, citations, completion+ )if completion is not None:completion.written += 1return True+def _count_failed(completion: IngestionCompletion | None) -> None:+ if completion is not None:+ completion.failed += 1++async def _promote_restated(client: Graphiti, result: AddEpisodeResults) -> None:"""Promote each tentative fact a user's chat turn states, new or mergedinto, to ``active`` with a ``ratified_at`` stamp (the ratification@@ -490,11 +532,12 @@ SET e.status = $activeasync def _dropped_as_forgotten(client: Graphiti,payload: dict[str, Any],+ citations: Citations | None,completion: IngestionCompletion | None,) -> bool:"""True, counting it on ``completion``, when ``payload`` is a dream writethat rests on a forget made since the dream read the graph."""- reason = await rests_on_a_forget(client.driver, payload.pop("_citations", None))+ reason = await rests_on_a_forget(client.driver, citations)if reason is None:return Falselogger.info(f"Dropped dream write {payload.get('name')!r}: it {reason}")@@ -555,13 +598,11 @@ def _note_lock_wait(payload: dict[str, Any], waits: int) -> None:)-def _count_failed(completion: IngestionCompletion | None) -> None:- if completion is not None:- completion.failed += 1--async def _add_episode(- client: Graphiti, group_id: str, payload: dict[str, Any]+ client: Graphiti,+ group_id: str,+ payload: dict[str, Any],+ episode: str | None = None,) -> AddEpisodeResults:"""graphiti's ``add_episode`` with our types, under the recall policy.@@ -577,12 +618,19 @@ async def _add_episode(found them. (graphiti's exact-text match, which runs before the model,can still list a new episode on a forgotten edge whose ``[forgotten]``text the statement repeats word for word.)++ A marked dream write's ``episode`` uuid is its marker's: the episode is+ placed under it first (``marked_write.placed``), after the earlier+ episodes are read, and graphiti writes over it."""previous = await previous_episode_uuids(client.driver, group_id, payload["reference_time"], payload["source"])+ if episode is not None:+ await marked_write.placed(client.driver, group_id, payload, episode)return await client.add_episode(**payload,+ uuid=episode,previous_episode_uuids=previous,entity_types=ENTITY_TYPES,edge_types=EDGE_TYPES,@@ -714,7 +762,10 @@ async def enqueue_episode(citations: What a dream write rests on. The worker checks them underthe graph's write lock right before writing and drops theepisode, counting it on ``completion``, when a forget made since- the dream read the graph reached them (``recall_citations.py``).+ the dream read the graph reached them (``recall_citations.py``);+ once it is written, the worker records them on the episode and+ on the facts only dream episodes state (``recall_derivation.py``),+ so a later forget of one retracts them (``recall_cascade.py``).``None`` for chat / memory-store writes.Returns ``True`` if the episode was queued, ``False`` if it was dropped.diff --git a/autogpt_platform/backend/backend/copilot/graphiti/marked_write.py b/autogpt_platform/backend/backend/copilot/graphiti/marked_write.pynew file mode 100644index 0000000000..bea43a7d27--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/marked_write.py@@ -0,0 +1,167 @@+"""A dream write between its citation marker and its derivation record: the+ingestion worker's side of ``recall_derivation.py``.++``ingest._write_locked`` holds the graph's write lock for the whole write+and calls these in turn. ``marked`` draws the uuid the write's episode will+have and writes the write's citation marker under it before anything else;+when the marker cannot be written the write is not made and counts as+failed (it fails closed). A write the check against forgets then drops, or+whose check raises, has its marker taken back (``withdrawn``). ``placed`` puts the episode in the+graph under that uuid, ``write_pending`` (hidden from recall), and+graphiti's ``add_episode`` is handed the uuid: graphiti-core 0.30.2 loads+the episode it names (``EpisodicNode.get_by_uuid``) instead of creating+one, extracts from it, and saves it again with its facts (a ``MERGE`` on+the uuid that replaces every property), so ``write_pending`` goes when+graphiti saves the episode. ``added`` runs the write; one that raised marks+its marker ``aborted`` and notes the graph for the reaper+(``provenance_pending.py``), as the episode may have landed in part.+``recorded`` records what the write was derived from and settles it+(``recall_derivation.record``); a record that fails leaves the marker,+notes the graph and counts as ``provenance_pending``. The counts land on+the pass's ``ingest.IngestionCompletion``.+"""++import logging+import uuid as uuidlib+from collections.abc import Awaitable+from datetime import datetime, timezone+from typing import Any, Protocol++from graphiti_core import Graphiti+from graphiti_core.driver.driver import GraphDriver+from graphiti_core.graphiti import AddEpisodeResults+from pydantic import BaseModel++from .provenance_pending import note_pending+from .recall_citations import Citations+from .recall_derivation import abort, mark, record, withdraw++logger = logging.getLogger(__name__)+++class Outcomes(Protocol):+ """The counters a dream write's outcome lands on+ (``ingest.IngestionCompletion``)."""++ failed: int+ provenance_pending: int+++class Marked(BaseModel):+ """A dream write's marker and the uuid its episode is written under."""++ marker: str+ episode: str+++async def marked(+ client: Graphiti,+ group_id: str,+ payload: dict[str, Any],+ citations: Citations | None,+ outcomes: Outcomes | None,+) -> Marked | None:+ """The citation marker of a dream write, written before the write under+ the episode uuid drawn for it; None for any other write, and for a dream+ write whose marker could not be written, which is then not made and+ counts as failed."""+ if citations is None:+ return None+ episode = str(uuidlib.uuid4())+ try:+ marker = await mark(+ client.driver, group_id, episode, str(payload["name"]), citations+ )+ except Exception:+ logger.warning(+ f"Dropped dream write {payload.get('name')!r}: its citation marker "+ f"could not be written in graph {group_id[:20]}",+ exc_info=True,+ )+ if outcomes is not None:+ outcomes.failed += 1+ return None+ return Marked(marker=marker, episode=episode)+++async def withdrawn(driver: GraphDriver, marked: Marked | None) -> None:+ """Take back the marker of a dream write dropped before it was made."""+ if marked is not None:+ await withdraw(driver, marked.marker)+++async def placed(+ driver: GraphDriver, group_id: str, payload: dict[str, Any], episode: str+) -> None:+ """Put the write's episode in the graph under ``episode``, as graphiti+ saves one and ``write_pending``, for graphiti's ``add_episode`` to load+ and save over."""+ await driver.execute_query(+ PLACE_EPISODE_QUERY,+ uuid=episode,+ name=str(payload["name"]),+ group_id=group_id,+ source_description=payload["source_description"],+ source=payload["source"].value,+ content=payload["episode_body"],+ created_at=datetime.now(timezone.utc),+ valid_at=payload["reference_time"],+ )+++async def added(+ write: Awaitable[AddEpisodeResults],+ driver: GraphDriver,+ group_id: str,+ marked: Marked | None,+) -> AddEpisodeResults:+ """The graph ``write``, awaited; a marked one that raised may have landed+ in part, so its marker is marked aborted and its graph noted for the+ reaper to reconcile."""+ try:+ return await write+ except Exception:+ if marked is not None:+ await abort(driver, marked.marker)+ await note_pending(group_id)+ raise+++async def recorded(+ client: Graphiti,+ group_id: str,+ marked: Marked,+ result: AddEpisodeResults,+ citations: Citations,+ outcomes: Outcomes | None,+) -> None:+ """Record what the dream write just made was derived from and settle it;+ a record that failed leaves its marker, is noted for the reaper and+ counts as ``provenance_pending``."""+ episode = result.episode.uuid+ touched = [edge.uuid for edge in result.edges if episode in edge.episodes]+ if await record(+ client.driver, group_id, marked.marker, episode, touched, citations+ ):+ return+ await note_pending(group_id)+ if outcomes is not None:+ outcomes.provenance_pending += 1+++# The properties graphiti saves an episode with (graphiti-core 0.30.2), and+# ``write_pending``, which graphiti's own save of the episode replaces.+PLACE_EPISODE_QUERY = """+CREATE (:Episodic {+ uuid: $uuid,+ name: $name,+ group_id: $group_id,+ source_description: $source_description,+ source: $source,+ content: $content,+ entity_edges: [],+ created_at: $created_at,+ valid_at: $valid_at,+ write_pending: true+})+"""diff --git a/autogpt_platform/backend/backend/copilot/graphiti/memory_model.py b/autogpt_platform/backend/backend/copilot/graphiti/memory_model.pyindex dee4fccb51..3a92420f07 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/memory_model.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/memory_model.py@@ -145,6 +145,21 @@ class MemoryForgetFailure(BaseModel):),)+ @classmethod+ def derived_left(cls, uuid: str) -> "MemoryForgetFailure":+ """The fact is forgotten, but the forget stopped at its bound before+ retracting everything the dream derived from it+ (``recall_cascade.py``); forgetting it again continues."""+ return cls(+ uuid=uuid,+ code=MemoryForgetFailureCode.CLEANUP_ERROR,+ reason=(+ "Forgotten and no longer recalled, but more facts derived from "+ "it remain than one forget retracts. Forget it again to "+ "continue."+ ),+ )+def _describe(exc: Exception) -> str:if isinstance(exc, (OSError, RedisConnectionError, RedisTimeoutError)):@@ -160,16 +175,28 @@ class ForgetResult(BaseModel):``failures`` holds one entry per requested uuid that was not, in theshape ``memory_forget_confirm`` reports, plus a ``cleanup_error`` for anedge whose clean-up failed, which is in ``deleted`` too when its own write- landed. The episode and entity lists record the clean-up done: a hard- forget empties an episode nothing else cites into a tombstone rather than- deleting it, so the chat session it came from stays known.+ landed. ``derived`` lists the facts the dream derived from them that the+ forget retracted with them (``recall_cascade.py``), a soft retraction in+ either mode, a hard one erasing their text (``recall_erase.py``), and+ ``passed`` the derived facts no longer live it walked through. The+ episode and entity lists record the clean-up done: a hard forget empties+ an episode nothing else cites into a tombstone rather than deleting it,+ so the chat session it came from stays known. ``resumed`` lists the+ requested uuids that were already gone, purged by a hard forget, whose+ cascade this forget went on with, erasing (``recall_forget.py``)."""deleted: list[str] = Field(default_factory=list)failures: list[MemoryForgetFailure] = Field(default_factory=list)+ derived: list[str] = Field(default_factory=list)+ passed: list[str] = Field(default_factory=list)+ resumed: list[str] = Field(default_factory=list)redacted_episodes: list[str] = Field(default_factory=list)tombstoned_episodes: list[str] = Field(default_factory=list)deleted_entities: list[str] = Field(default_factory=list)+ # The names of the dream write episodes a hard forget erased, each naming+ # its pass (``dream/forget_copies.py`` blanks that pass's copies).+ dream_writes: list[str] = Field(default_factory=list)def envelope_provenance(content: str | None) -> str | None:diff --git a/autogpt_platform/backend/backend/copilot/graphiti/migrations/backfill_cascade.py b/autogpt_platform/backend/backend/copilot/graphiti/migrations/backfill_cascade.pynew file mode 100644index 0000000000..de1ca907a7--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/migrations/backfill_cascade.py@@ -0,0 +1,207 @@+"""The derivation backfill's cascade from the forgets already made+(``--cascade-existing-forgets``, ``backfill_derivations.py``).++It finds every root a forget left in the graph and runs a forget's cascade+from it (``recall_cascade.py``):++- every fact still in the graph that is forgotten, softly;+- every fact that is gone but that something still names as what it rests+ on or was hidden for: a derivation record (a fact's or an episode's), a+ pending citation marker, an episode's ``redacted_for``, or an earlier+ cascade's reason. A hard forget purged it, perhaps before its cascade ran+ or finished, so the cascade from it erases (``recall_erase.py``);+- every hidden episode a record names as a source, so what rests on an+ episode a forget hid is reached even when the forgotten fact is gone and+ nothing links the two any more (a forget made before ``redacted_for``). It+ names the root it was hidden for when it remembers one, else itself; one+ a hard forget emptied (``hard_deleted_at``) is erased from.++The hard roots cascade first, then the soft. A fact both reach is erased+either way: a hard walk erases the derived facts it only walks through. A+dry run counts the roots and writes nothing.+"""++from datetime import datetime, timezone+from typing import Any++from pydantic import BaseModel, Field++from backend.copilot.graphiti.falkordb_driver import AutoGPTFalkorDriver+from backend.copilot.graphiti.memory_model import ForgetResult+from backend.copilot.graphiti.recall import (+ forgotten_fact_predicate,+ forgotten_facts_clause,+ recallable_episode_predicate,+)+from backend.copilot.graphiti.recall_cascade import cascade+from backend.copilot.graphiti.recall_cascade_walk import DERIVED_FROM_FORGOTTEN+from backend.copilot.graphiti.recall_derivation import MARKER_LABEL++from .backfill_pages import BATCH_SIZE, pages, rows++_PREFIX = f"{DERIVED_FROM_FORGOTTEN}:"+++class Roots(BaseModel):+ """The roots of one graph's forgets: facts and hidden episodes (each to+ the root it names), those to erase from (``hard``) and the rest."""++ soft: list[str] = Field(default_factory=list)+ hard: list[str] = Field(default_factory=list)+ soft_seeds: dict[str, str] = Field(default_factory=dict)+ hard_seeds: dict[str, str] = Field(default_factory=dict)++ def count(self) -> int:+ return sum(+ len(part)+ for part in (self.soft, self.hard, self.soft_seeds, self.hard_seeds)+ )+++class Cascaded(BaseModel):+ """What the cascade found and did: its ``roots``, the facts it retracted+ (``derived``), and whether a cascade stopped short (``failed``)."""++ roots: int = 0+ derived: int = 0+ failed: bool = False+++async def cascade_existing_forgets(+ driver: AutoGPTFalkorDriver, *, apply: bool+) -> Cascaded:+ """Cascade from every root the graph's forgets left, hard ones first,+ erasing; count them only on a dry run."""+ roots = await find_roots(driver)+ done = Cascaded(roots=roots.count())+ if not (apply and done.roots):+ return done+ result = ForgetResult()+ now = datetime.now(timezone.utc).isoformat()+ graph = driver.graph_name+ for facts, seeds, erase in (+ (roots.hard, roots.hard_seeds, True),+ (roots.soft, roots.soft_seeds, False),+ ):+ if facts or seeds:+ await cascade(driver, graph, facts, now, result, erase=erase, seeds=seeds)+ done.derived = len(result.derived)+ done.failed = bool(result.failures)+ return done+++async def find_roots(driver: AutoGPTFalkorDriver) -> Roots:+ """Every fact a user forgot that is still there (one a cascade retracted+ is picked up from its root, whose name its reason keeps), every gone+ fact still named, and every hidden episode a record names, as+ ``Roots``."""+ named_facts: dict[str, None] = {}+ named_episodes: dict[str, None] = {}+ rooted: dict[str, str] = {}+ for row in [+ *await pages(driver, FACT_NAMES_QUERY),+ *await pages(driver, EPISODE_NAMES_QUERY),+ *rows(await driver.execute_query(MARKER_NAMES_QUERY)),+ ]:+ named_facts |= dict.fromkeys([*row["facts"], *row["hidden_for"]])+ named_episodes |= dict.fromkeys(row["episodes"])+ if (row["reason"] or "").startswith(_PREFIX):+ rooted[row["uuid"]] = row["reason"].removeprefix(_PREFIX)+ named_facts[rooted[row["uuid"]]] = None+ forgotten = await pages(driver, FORGOTTEN_FACTS_QUERY)+ gone = await _gone(driver, list(named_facts))+ hidden = await _hidden(driver, list(named_episodes))+ seeds = {row["uuid"]: (_name(row, rooted), row["hard"]) for row in hidden}+ return Roots(+ soft=[row["uuid"] for row in forgotten if row["uuid"] not in rooted],+ hard=gone,+ soft_seeds={uuid: name for uuid, (name, hard) in seeds.items() if not hard},+ hard_seeds={uuid: name for uuid, (name, hard) in seeds.items() if hard},+ )+++async def _gone(driver: AutoGPTFalkorDriver, uuids: list[str]) -> list[str]:+ """Those of ``uuids`` no fact in the graph has."""+ found: set[str] = set()+ for start in range(0, len(uuids), BATCH_SIZE):+ chunk = uuids[start : start + BATCH_SIZE]+ result = await driver.execute_query(EXISTING_FACTS_QUERY, uuids=chunk)+ found |= {row["uuid"] for row in rows(result)}+ return [uuid for uuid in uuids if uuid not in found]+++async def _hidden(driver: AutoGPTFalkorDriver, uuids: list[str]) -> list[dict]:+ """Those of ``uuids`` that are episodes the recall policy hides."""+ hidden: list[dict[str, Any]] = []+ for start in range(0, len(uuids), BATCH_SIZE):+ chunk = uuids[start : start + BATCH_SIZE]+ result = await driver.execute_query(HIDDEN_EPISODES_QUERY, uuids=chunk)+ hidden.extend(rows(result))+ return hidden+++def _name(row: dict[str, Any], rooted: dict[str, str]) -> str:+ """The root a hidden episode's retractions name: that of the first+ forgotten fact it remembers it was hidden for (the fact itself, or the+ root a cascade that retracted it named), else the episode itself."""+ hidden_for: list[str] = row["hidden_for"] or []+ if not hidden_for:+ return str(row["uuid"])+ return rooted.get(hidden_for[0], hidden_for[0])+++FORGOTTEN_FACTS_QUERY = f"""+MATCH ()-[e:RELATES_TO]->()+WHERE e.uuid > $after AND {forgotten_fact_predicate("e")}+RETURN e.uuid AS uuid+ORDER BY uuid+LIMIT $limit+"""++# Every fact's record and an earlier cascade's reason.+FACT_NAMES_QUERY = f"""+MATCH ()-[e:RELATES_TO]->()+WHERE e.uuid > $after+ AND (e.derived_from_facts IS NOT NULL+ OR e.expiration_reason STARTS WITH '{_PREFIX}')+RETURN e.uuid AS uuid, coalesce(e.derived_from_facts, []) AS facts,+ coalesce(e.derived_from_episodes, []) AS episodes, [] AS hidden_for,+ e.expiration_reason AS reason+ORDER BY uuid+LIMIT $limit+"""++# Every episode's record and what it was hidden for.+EPISODE_NAMES_QUERY = """+MATCH (ep:Episodic)+WHERE ep.uuid > $after+ AND (ep.derived_from_facts IS NOT NULL OR ep.redacted_for IS NOT NULL)+RETURN ep.uuid AS uuid, coalesce(ep.derived_from_facts, []) AS facts,+ coalesce(ep.derived_from_episodes, []) AS episodes,+ coalesce(ep.redacted_for, []) AS hidden_for, NULL AS reason+ORDER BY uuid+LIMIT $limit+"""++MARKER_NAMES_QUERY = f"""+MATCH (m:{MARKER_LABEL})+RETURN coalesce(m.derived_from_facts, []) AS facts,+ coalesce(m.derived_from_episodes, []) AS episodes, [] AS hidden_for,+ NULL AS reason+"""++EXISTING_FACTS_QUERY = """+MATCH ()-[e:RELATES_TO]->()+WHERE e.uuid IN $uuids+RETURN e.uuid AS uuid+"""++HIDDEN_EPISODES_QUERY = (+ forgotten_facts_clause()+ + f"""+MATCH (ep:Episodic)+WHERE ep.uuid IN $uuids AND NOT ({recallable_episode_predicate("ep")})+RETURN ep.uuid AS uuid, ep.hard_deleted_at IS NOT NULL AS hard,+ coalesce(ep.redacted_for, []) AS hidden_for+"""+)diff --git a/autogpt_platform/backend/backend/copilot/graphiti/migrations/backfill_cascade_test.py b/autogpt_platform/backend/backend/copilot/graphiti/migrations/backfill_cascade_test.pynew file mode 100644index 0000000000..800b3dd7d5--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/migrations/backfill_cascade_test.py@@ -0,0 +1,204 @@+"""Unit tests for the derivation backfill's cascade from the forgets already+made (``backfill_cascade.py``): the roots it finds (forgotten facts, facts a+hard forget purged that something still names, hidden episodes a record+names) and the cascades it runs from them, hard ones first and erasing. The+live run is ``graphiti/recall_cascade_resume_integration_test.py``.+"""++from typing import Any+from unittest.mock import AsyncMock, patch++import pytest++from backend.copilot.graphiti.memory_model import MemoryForgetFailure++from . import backfill_cascade+from . import backfill_derivations as backfill+from .backfill_fake import BackfillGraph++_PREFIX = "derived_from_forgotten:"+++class _Graph:+ """What the root reads see: each query's rows, paged by uuid where the+ query pages."""++ graph_name = "user_a"++ def __init__(self) -> None:+ self.paged: dict[str, list[dict[str, Any]]] = {+ backfill_cascade.FORGOTTEN_FACTS_QUERY: [+ {"uuid": "d2"},+ {"uuid": "f-soft"},+ ],+ backfill_cascade.FACT_NAMES_QUERY: [+ _named("d1", facts=["f-soft", "f-gone"], episodes=["chat", "ep-d2"]),+ _named("d2", reason=f"{_PREFIX}f-purged"),+ _named("d3", reason="stale_fact"),+ ],+ backfill_cascade.EPISODE_NAMES_QUERY: [+ _named("ep-dream", facts=["f-gone"], episodes=["said"]),+ _named("said", hidden_for=["f-purged"]),+ ],+ }+ self.markers = [_named("m1", facts=["f-pending"])]+ self.existing = {"f-soft"}+ self.hidden = [+ {"uuid": "said", "hard": True, "hidden_for": ["f-purged"]},+ {"uuid": "chat", "hard": False, "hidden_for": []},+ {"uuid": "ep-d2", "hard": False, "hidden_for": ["d2"]},+ ]++ async def execute_query(self, query: str, **params: Any):+ if query in self.paged:+ rows = [r for r in self.paged[query] if r["uuid"] > params["after"]]+ return rows[: params["limit"]], [], None+ if query == backfill_cascade.MARKER_NAMES_QUERY:+ return self.markers, [], None+ if query == backfill_cascade.EXISTING_FACTS_QUERY:+ found = [{"uuid": u} for u in params["uuids"] if u in self.existing]+ return found, [], None+ assert query == backfill_cascade.HIDDEN_EPISODES_QUERY+ return [r for r in self.hidden if r["uuid"] in params["uuids"]], [], None+++def _named(+ uuid: str,+ *,+ facts: list[str] | None = None,+ episodes: list[str] | None = None,+ hidden_for: list[str] | None = None,+ reason: str | None = None,+) -> dict[str, Any]:+ return {+ "uuid": uuid,+ "facts": facts or [],+ "episodes": episodes or [],+ "hidden_for": hidden_for or [],+ "reason": reason,+ }+++class TestFindRoots:+ @pytest.mark.asyncio+ async def test_finds_every_root_the_graphs_forgets_left(self) -> None:+ """A fact a user forgot, still there, is soft (``d2``, which a cascade+ retracted for ``f-purged``, is picked up from that root); a gone+ fact a record, a marker, a reason or an episode's ``redacted_for``+ names was purged by a hard forget; a hidden episode a record names+ is a seed, erased from when a hard forget emptied it, naming the+ root of what it was hidden for, else itself."""+ roots = await backfill_cascade.find_roots(_Graph())++ assert roots.soft == ["f-soft"]+ assert sorted(roots.hard) == ["f-gone", "f-pending", "f-purged"]+ assert roots.hard_seeds == {"said": "f-purged"}+ assert roots.soft_seeds == {"chat": "chat", "ep-d2": "f-purged"}+ assert roots.count() == 7++ @pytest.mark.asyncio+ async def test_a_graph_no_forget_touched_has_no_root(self) -> None:+ graph = _Graph()+ graph.paged = {query: [] for query in graph.paged}+ graph.markers = []++ roots = await backfill_cascade.find_roots(graph)++ assert roots.count() == 0++ def test_the_reads_write_nothing(self) -> None:+ for query in (+ backfill_cascade.FORGOTTEN_FACTS_QUERY,+ backfill_cascade.FACT_NAMES_QUERY,+ backfill_cascade.EPISODE_NAMES_QUERY,+ backfill_cascade.MARKER_NAMES_QUERY,+ backfill_cascade.EXISTING_FACTS_QUERY,+ backfill_cascade.HIDDEN_EPISODES_QUERY,+ ):+ assert "SET" not in query and "DELETE" not in query+++class TestCascadeExistingForgets:+ @pytest.mark.asyncio+ async def test_cascades_from_the_hard_roots_first_erasing(self) -> None:+ calls: list[tuple[list[str], dict[str, str], bool]] = []++ async def cascade(driver, group_id, roots, now, result, *, erase, seeds):+ calls.append((sorted(roots), seeds, erase))+ result.derived.append(f"d-{len(calls)}")++ with patch.object(backfill_cascade, "cascade", cascade):+ done = await backfill_cascade.cascade_existing_forgets(_Graph(), apply=True)++ assert calls == [+ (["f-gone", "f-pending", "f-purged"], {"said": "f-purged"}, True),+ (["f-soft"], {"chat": "chat", "ep-d2": "f-purged"}, False),+ ]+ assert done == backfill_cascade.Cascaded(roots=7, derived=2, failed=False)++ @pytest.mark.asyncio+ async def test_a_dry_run_counts_and_cascades_nothing(self) -> None:+ cascade = AsyncMock()++ with patch.object(backfill_cascade, "cascade", cascade):+ done = await backfill_cascade.cascade_existing_forgets(+ _Graph(), apply=False+ )++ cascade.assert_not_awaited()+ assert done.roots == 7+++class TestThroughTheBackfill:+ @pytest.mark.asyncio+ async def test_a_dry_run_counts_the_forgets_it_would_cascade_from(self) -> None:+ driver = BackfillGraph([], [], forgotten=["x1", "x2"])+ cascade = AsyncMock()++ with patch.object(backfill_cascade, "cascade", cascade):+ found = await backfill.backfill_graph(+ driver, apply=False, cascade_forgets=True+ )++ assert found.roots == 2+ cascade.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_apply_cascades_from_every_forget_after_stamping(self) -> None:+ driver = BackfillGraph([], [], forgotten=["x1", "x2"])++ async def retract_two(+ driver, group_id, roots, now, result, *, erase, seeds+ ) -> None:+ assert (group_id, roots, erase, seeds) == (+ "user_a",+ ["x1", "x2"],+ False,+ {},+ )+ result.derived.extend(["d1", "d2"])++ with patch.object(+ backfill_cascade, "cascade", AsyncMock(side_effect=retract_two)+ ):+ found = await backfill.backfill_graph(+ driver, apply=True, cascade_forgets=True+ )++ assert (found.roots, found.derived, found.failed) == (2, 2, 0)++ @pytest.mark.asyncio+ async def test_a_cascade_that_stopped_short_fails_the_graph(self) -> None:+ driver = BackfillGraph([], [], forgotten=["x1"])++ async def stop_short(driver, group_id, roots, now, result, **_: object):+ result.failures.append(MemoryForgetFailure.derived_left("x1"))++ with patch.object(+ backfill_cascade, "cascade", AsyncMock(side_effect=stop_short)+ ):+ found = await backfill.backfill_graph(+ driver, apply=True, cascade_forgets=True+ )++ assert found.failed == 1diff --git a/autogpt_platform/backend/backend/copilot/graphiti/migrations/backfill_derivations.py b/autogpt_platform/backend/backend/copilot/graphiti/migrations/backfill_derivations.pynew file mode 100644index 0000000000..2d00bbb162--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/migrations/backfill_derivations.py@@ -0,0 +1,297 @@+"""Record what dream facts written before derivation records rest on.++The ingestion worker records a dream write's citations on its episode and on+the facts only dream episodes state (``recall_derivation.py``); a forget's+cascade follows them (``recall_cascade.py``). An older dream write has them+only in its episode's ``source_description``, the first five of each kind+(a consolidation listed episodes, a proposal facts). This reads them back,+only in the shapes the dream wrote and only uuids the graph has, facts in+the episode's own scope (``legacy_citations.py``: a model's rationale could+forge a citation): every dream episode with no record (named ``dream_...``+or described ``dream-pass...``) gets one from its description, empty when+it lists nothing or its shape is ambiguous; then every fact with no record+whose source episodes all have one gets their union, as ingestion stamps+it. It reports the ambiguous descriptions, the citations it rejected, and+the dream facts it could not attribute (stamped with nothing cited, so no+forget reaches them); a citation past the first five is lost too, and so is+one of a fact a hard forget purged before this ran. It does not cascade on+its own: ``--cascade-existing-forgets`` also runs a forget's cascade from+every root the graph's forgets left, the facts a hard forget purged and the+episodes a forget hid included (``backfill_cascade.py``), which a dry run+only counts.++Dry run by default. ``--apply`` writes, each graph holding its write lock+(``scope_lock.py``) from its first read to its last write, in batches of+``BATCH_SIZE``, after completing the dream records a failed write left+pending in it (``recall_reconcile.py``). A graph locked past+``BACKFILL_LOCK_WAIT_SECONDS``, or not lockable because Redis is+unreachable, is skipped unwritten and counted busy; one whose backfill+raises, or whose cascade stops short, is counted failed. Either makes the+script exit 1: run it again. Every write is idempotent.++Usage:++ poetry run python -m \\+ backend.copilot.graphiti.migrations.backfill_derivations \\+ [--apply] [--graph <name>] [--cascade-existing-forgets]+"""++import argparse+import asyncio+import logging+import sys+from typing import Any++from pydantic import BaseModel++from backend.copilot.dream.citations import envelope_scope+from backend.copilot.graphiti.falkordb_driver import (+ AutoGPTFalkorDriver,+ open_graph_driver,+)+from backend.copilot.graphiti.graphs import list_graph_names+from backend.copilot.graphiti.recall_reconcile import reconcile+from backend.copilot.graphiti.scope_lock import LockState, graph_write_lock++from .backfill_cascade import cascade_existing_forgets+from .backfill_legacy_forgets import MEMORY_GRAPH_PREFIXES+from .backfill_pages import pages, write_rows+from .legacy_citations import LegacyCitations, checked, described_citations++logger = logging.getLogger(__name__)++BACKFILL_LOCK_WAIT_SECONDS = 60+++class Derivations(BaseModel):+ """What the backfill found (with ``--apply``, wrote): pending dream+ records completed (``reconciled``), dream ``episodes`` given a record,+ of their descriptions those ``ambiguous``, the citations ``rejected``+ (not in the graph in the episode's scope), ``facts`` stamped, of them+ ``unattributed`` with nothing cited; the ``roots`` a cascade started+ from (forgotten or purged facts, hidden episodes) and the facts it+ retracted (``derived``); and the graphs to run again, ``busy`` and+ ``failed``."""++ reconciled: int = 0+ episodes: int = 0+ ambiguous: int = 0+ rejected: int = 0+ facts: int = 0+ unattributed: int = 0+ roots: int = 0+ derived: int = 0+ busy: int = 0+ failed: int = 0++ def plus(self, other: "Derivations") -> "Derivations":+ theirs = other.model_dump()+ return Derivations(**{k: v + theirs[k] for k, v in self.model_dump().items()})+++async def backfill_graph(+ driver: AutoGPTFalkorDriver, *, apply: bool, cascade_forgets: bool+) -> Derivations:+ """Record one graph's derivations; with ``apply``, write them holding the+ graph's write lock, and cascade from its forgets if asked."""+ if not apply:+ return await _derive(driver, apply=False, cascade_forgets=cascade_forgets)+ graph, wait = driver.graph_name, BACKFILL_LOCK_WAIT_SECONDS+ async with graph_write_lock(graph, wait_seconds=wait) as lock:+ if lock is not LockState.HELD:+ logger.warning(f"Skipped graph {graph[:20]}: write lock {lock.value}")+ return Derivations(busy=1)+ pending = await reconcile(driver, graph)+ found = await _derive(driver, apply=True, cascade_forgets=cascade_forgets)+ found.reconciled = pending.completed+ if pending.incomplete():+ found.failed = 1+ return found+++async def _derive(+ driver: AutoGPTFalkorDriver, *, apply: bool, cascade_forgets: bool+) -> Derivations:+ """Both steps, reading the whole graph before writing anything."""+ records: dict[str, _Record] = {}+ described: dict[str, LegacyCitations] = {}+ scopes: dict[str, str] = {}+ for row in await pages(driver, DREAM_EPISODES_QUERY):+ if row["facts"] is None:+ described[row["uuid"]] = described_citations(row["description"])+ scopes[row["uuid"]] = envelope_scope(row["content"])+ else:+ records[row["uuid"]] = _Record(+ uuid=row["uuid"], facts=row["facts"], episodes=row["episodes"] or []+ )+ verified = await checked(driver, described, scopes)+ new = [+ _Record(uuid=uuid, facts=cited.facts, episodes=cited.episodes)+ for uuid, cited in verified.cited.items()+ ]+ records |= {record.uuid: record for record in new}+ stamps = [+ stamp+ for row in await pages(driver, UNSTAMPED_FACTS_QUERY)+ if (stamp := _union(row["uuid"], row["episodes"], records)) is not None+ ]+ found = Derivations(+ episodes=len(new),+ ambiguous=sum(1 for cited in described.values() if cited.ambiguous),+ rejected=verified.rejected,+ facts=len(stamps),+ unattributed=sum(1 for stamp in stamps if not (stamp.facts or stamp.episodes)),+ )+ if apply:+ await write_rows(driver, RECORD_EPISODES_QUERY, _dumped(new))+ await write_rows(driver, STAMP_FACTS_QUERY, _dumped(stamps))+ if cascade_forgets:+ cascaded = await cascade_existing_forgets(driver, apply=apply)+ found.roots, found.derived = cascaded.roots, cascaded.derived+ found.failed = int(cascaded.failed)+ return found+++class _Record(BaseModel):+ """A dream episode's record, or the stamp a fact gets from its sources'."""++ uuid: str+ facts: list[str]+ episodes: list[str]+++def _union(+ uuid: str, sources: list[str], records: dict[str, _Record]+) -> _Record | None:+ """Fact ``uuid``'s stamp when every episode it names has a record, as+ ``recall_derivation.STAMP_FACTS_QUERY`` builds it; else None."""+ named = list(dict.fromkeys(sources))+ if not named or any(source not in records for source in named):+ return None+ facts = [cited for source in named for cited in records[source].facts]+ episodes = [cited for source in named for cited in records[source].episodes]+ return _Record(+ uuid=uuid,+ facts=list(dict.fromkeys(facts)),+ episodes=list(dict.fromkeys(episodes)),+ )+++def _dumped(records: list[_Record]) -> list[dict[str, Any]]:+ return [record.model_dump() for record in records]+++async def backfill_all_graphs(+ *, apply: bool, cascade_forgets: bool, graph: str | None = None+) -> Derivations:+ """Backfill every memory graph on the server, or just ``graph``; a graph+ that is busy or fails is logged, skipped and counted."""+ names = [graph] if graph else await _memory_graph_names()+ totals = Derivations()+ for name in names:+ driver = open_graph_driver(name)+ try:+ found = await backfill_graph(+ driver, apply=apply, cascade_forgets=cascade_forgets+ )+ except Exception:+ logger.warning(f"Backfill failed for graph {name[:20]}", exc_info=True)+ totals.failed += 1+ continue+ finally:+ await driver.close()+ totals = totals.plus(found)+ return totals+++async def _memory_graph_names() -> list[str]:+ names = await list_graph_names()+ return sorted(name for name in names if name.startswith(MEMORY_GRAPH_PREFIXES))+++# Reads page by uuid, so a dry run never writes (nor creates) a graph.+DREAM_EPISODES_QUERY = """+MATCH (ep:Episodic)+WHERE ep.uuid > $after+ AND ep.write_pending IS NULL+ AND (ep.derived_from_facts IS NOT NULL+ OR ep.name STARTS WITH 'dream_'+ OR ep.source_description STARTS WITH 'dream-pass')+RETURN ep.uuid AS uuid, ep.source_description AS description,+ ep.content AS content,+ ep.derived_from_facts AS facts, ep.derived_from_episodes AS episodes+ORDER BY uuid+LIMIT $limit+"""++# As ingestion stamps: never a fact a forget reached.+UNSTAMPED_FACTS_QUERY = """+MATCH ()-[e:RELATES_TO]->()+WHERE e.uuid > $after+ AND e.derived_from_facts IS NULL+ AND e.forgotten_at IS NULL+RETURN e.uuid AS uuid, coalesce(e.episodes, []) AS episodes+ORDER BY uuid+LIMIT $limit+"""++# ``IS NULL`` keeps a record already written, the ingestion worker's+# complete one included.+RECORD_EPISODES_QUERY = """+UNWIND $rows AS row+MATCH (ep:Episodic {uuid: row.uuid})+WHERE ep.derived_from_facts IS NULL+SET ep.derived_from_facts = row.facts,+ ep.derived_from_episodes = row.episodes+"""++STAMP_FACTS_QUERY = """+UNWIND $rows AS row+MATCH ()-[e:RELATES_TO {uuid: row.uuid}]->()+WHERE e.derived_from_facts IS NULL+SET e.derived_from_facts = row.facts,+ e.derived_from_episodes = row.episodes+"""+++async def main(args: argparse.Namespace) -> int:+ logging.basicConfig(+ level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s"+ )+ totals = await backfill_all_graphs(+ apply=args.apply,+ cascade_forgets=args.cascade_existing_forgets,+ graph=args.graph,+ )+ verb = "recorded" if args.apply else "would record (dry run)"+ print(+ f"completed {totals.reconciled} pending dream records; "+ f"{verb} {totals.episodes} dream episodes and {totals.facts} facts; "+ f"{totals.unattributed} dream facts cite nothing to attribute; "+ f"{totals.ambiguous} descriptions ambiguous, {totals.rejected} "+ "citations not in the graph, or of a fact of another scope"+ )+ if args.cascade_existing_forgets:+ done = f"retracted {totals.derived} derived facts" if args.apply else "not run"+ print(f"cascade from {totals.roots} roots forgets left: {done}")+ if not (totals.busy or totals.failed):+ return 0+ print(f"skipped {totals.busy} busy and {totals.failed} failed graphs: run again")+ return 1+++def parser() -> argparse.ArgumentParser:+ """The command line: a dry run unless ``--apply``."""+ cli = argparse.ArgumentParser(description=__doc__)+ cli.add_argument("--apply", action="store_true", help="Write, under each lock.")+ cli.add_argument("--graph", help="Backfill one graph instead of all of them.")+ cli.add_argument(+ "--cascade-existing-forgets",+ action="store_true",+ help="Also retract what the dream derived from facts already forgotten.",+ )+ return cli+++if __name__ == "__main__":+ sys.exit(asyncio.run(main(parser().parse_args())))diff --git a/autogpt_platform/backend/backend/copilot/graphiti/migrations/backfill_derivations_test.py b/autogpt_platform/backend/backend/copilot/graphiti/migrations/backfill_derivations_test.pynew file mode 100644index 0000000000..50ef0f13ee--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/migrations/backfill_derivations_test.py@@ -0,0 +1,179 @@+"""Unit tests for the derivation backfill: what it would write on a dry run+and writes with ``--apply``, under the graph's write lock, and its command+line. How it reads a description is in ``legacy_citations_test.py``, its+cascade in ``backfill_cascade_test.py``; the live run is in+``backfill_derivations_integration_test.py``.+"""++from typing import Any+from unittest.mock import AsyncMock, patch++import pytest++from backend.copilot.graphiti.recall_fake_redis import FakeRedis+from backend.copilot.graphiti.scope import write_lock_key++from . import backfill_derivations as backfill+from . import backfill_pages+from .backfill_fake import BackfillGraph, older_dreams++_LOCK = write_lock_key("user_a")+++class TestBackfillGraph:+ @pytest.mark.asyncio+ async def test_a_dry_run_counts_and_writes_nothing(self) -> None:+ driver = older_dreams()++ found = await backfill.backfill_graph(+ driver, apply=False, cascade_forgets=False+ )++ assert (found.episodes, found.facts, found.unattributed) == (3, 3, 1)+ assert driver.writes == []++ @pytest.mark.asyncio+ async def test_apply_records_the_episodes_then_stamps_the_facts(self) -> None:+ """A fact a user's own turn also states is left unstamped; one whose+ dream episodes cite nothing is stamped empty and reported."""+ driver = older_dreams()++ await backfill.backfill_graph(driver, apply=True, cascade_forgets=False)++ assert driver.reconciled, "pending dream records are completed first"+ (records_query, records), (stamps_query, stamps) = driver.writes+ assert records_query == backfill.RECORD_EPISODES_QUERY+ assert records == [+ {"uuid": "d1", "facts": [], "episodes": ["e0", "e9"]},+ {"uuid": "d2", "facts": ["f1"], "episodes": []},+ {"uuid": "d4", "facts": [], "episodes": []},+ ]+ assert stamps_query == backfill.STAMP_FACTS_QUERY+ assert stamps == [+ {"uuid": "c1", "facts": [], "episodes": ["e0", "e9"]},+ {"uuid": "c2", "facts": ["f1", "f2"], "episodes": []},+ {"uuid": "c4", "facts": [], "episodes": []},+ ]++ @pytest.mark.asyncio+ async def test_reads_and_writes_in_batches(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ monkeypatch.setattr(backfill_pages, "BATCH_SIZE", 2)+ driver = older_dreams()++ await backfill.backfill_graph(driver, apply=True, cascade_forgets=False)++ assert [len(rows) for _, rows in driver.writes] == [2, 1, 2, 1]++ def test_a_record_already_written_is_never_replaced(self) -> None:+ assert "WHERE ep.derived_from_facts IS NULL" in backfill.RECORD_EPISODES_QUERY+ assert "WHERE e.derived_from_facts IS NULL" in backfill.STAMP_FACTS_QUERY+ assert "SET" not in backfill.DREAM_EPISODES_QUERY+ assert "SET" not in backfill.UNSTAMPED_FACTS_QUERY+++class TestTheWriteLock:+ @pytest.mark.asyncio+ async def test_apply_holds_the_graphs_write_lock_throughout(+ self, lock_redis: FakeRedis+ ) -> None:+ driver = older_dreams()+ held: list[bool] = []+ query = driver.execute_query++ async def locked(cypher: str, **params: Any):+ held.append(_LOCK in lock_redis.values)+ return await query(cypher, **params)++ with patch.object(driver, "execute_query", locked):+ await backfill.backfill_graph(driver, apply=True, cascade_forgets=False)++ assert held and all(held)+ assert _LOCK not in lock_redis.values, "released afterwards"++ @pytest.mark.asyncio+ async def test_a_graph_another_writer_keeps_locked_is_skipped_as_busy(+ self, lock_redis: FakeRedis, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ lock_redis.values[_LOCK] = "an ingestion's token"+ monkeypatch.setattr(backfill, "BACKFILL_LOCK_WAIT_SECONDS", 0)+ driver = older_dreams()++ found = await backfill.backfill_graph(driver, apply=True, cascade_forgets=True)++ assert (found.busy, found.episodes, driver.writes) == (1, 0, [])+++class TestBackfillAllGraphs:+ @pytest.mark.asyncio+ async def test_walks_every_memory_graph_and_counts_a_failing_one(self) -> None:+ broken = BackfillGraph([], [])+ broken.answers = {}+ drivers = {+ "user_a": older_dreams(),+ "expert_b": older_dreams(),+ "user_broken": broken,+ }++ with (+ patch.object(+ backfill,+ "list_graph_names",+ AsyncMock(return_value=[*drivers, "other", "default_db"]),+ ),+ patch.object(+ backfill, "open_graph_driver", side_effect=drivers.__getitem__+ ),+ ):+ totals = await backfill.backfill_all_graphs(+ apply=False, cascade_forgets=False+ )++ assert (totals.episodes, totals.facts, totals.failed) == (6, 6, 1)+ for driver in drivers.values():+ driver.close.assert_awaited_once()+++class TestMain:+ def test_the_command_line_is_a_dry_run_unless_told_otherwise(self) -> None:+ args = backfill.parser().parse_args([])+ assert (args.apply, args.graph, args.cascade_existing_forgets) == (+ False,+ None,+ False,+ )+ args = backfill.parser().parse_args(+ ["--apply", "--graph", "user_a", "--cascade-existing-forgets"]+ )+ assert (args.apply, args.graph, args.cascade_existing_forgets) == (+ True,+ "user_a",+ True,+ )++ @pytest.mark.parametrize(+ "totals, code",+ [+ (backfill.Derivations(facts=2), 0),+ (backfill.Derivations(facts=2, busy=1), 1),+ (backfill.Derivations(failed=1), 1),+ ],+ ids=["all done", "a graph busy", "a graph failed"],+ )+ @pytest.mark.asyncio+ async def test_exits_non_zero_while_any_graph_is_left_to_run_again(+ self,+ totals: backfill.Derivations,+ code: int,+ capsys: pytest.CaptureFixture[str],+ ) -> None:+ args = backfill.parser().parse_args(["--apply", "--cascade-existing-forgets"])+ with patch.object(+ backfill, "backfill_all_graphs", AsyncMock(return_value=totals)+ ):+ assert await backfill.main(args) == code++ out = capsys.readouterr().out+ assert ("run again" in out) is bool(code)+ assert "cascade from" in outdiff --git a/autogpt_platform/backend/backend/copilot/graphiti/migrations/backfill_fake.py b/autogpt_platform/backend/backend/copilot/graphiti/migrations/backfill_fake.pynew file mode 100644index 0000000000..68ae46db44--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/migrations/backfill_fake.py@@ -0,0 +1,91 @@+"""A graph as the derivation backfill's reads see it, recording its writes,+for ``backfill_derivations_test.py``, ``legacy_citations_test.py`` and+``backfill_cascade_test.py``. Not collected by pytest.+"""++from typing import Any+from unittest.mock import AsyncMock++from backend.copilot.graphiti.recall_reconcile import MARKERS_QUERY++from . import backfill_cascade+from . import backfill_derivations as backfill+from . import legacy_citations+++class BackfillGraph:+ """A graph as the backfill's reads see it, recording its writes."""++ graph_name = "user_a"++ def __init__(+ self,+ episodes: list[dict[str, Any]],+ facts: list[dict[str, Any]],+ forgotten: list[str] | None = None,+ ) -> None:+ self.answers = {+ backfill.DREAM_EPISODES_QUERY: episodes,+ backfill.UNSTAMPED_FACTS_QUERY: facts,+ backfill_cascade.FORGOTTEN_FACTS_QUERY: [+ {"uuid": u} for u in forgotten or []+ ],+ backfill_cascade.FACT_NAMES_QUERY: [],+ backfill_cascade.EPISODE_NAMES_QUERY: [],+ }+ self.writes: list[tuple[str, list[dict[str, Any]]]] = []+ # Whether it read the pending dream records (none here).+ self.reconciled = False+ # The sources the graph has, all unscoped, for the citations read+ # back from descriptions.+ self.sources = {"e0", "e9", "f1", "f2"}+ self.close = AsyncMock()++ async def execute_query(self, query: str, **params: Any):+ if query == MARKERS_QUERY:+ self.reconciled = True+ return [], [], None+ if query == backfill_cascade.MARKER_NAMES_QUERY:+ return [], [], None+ if query == legacy_citations.FACT_SCOPES_QUERY:+ found = [{"uuid": u, "scope": None} for u in params["uuids"]]+ return [row for row in found if row["uuid"] in self.sources], [], None+ if query == legacy_citations.CITED_EPISODES_QUERY:+ found = [{"uuid": u} for u in params["uuids"]]+ return [row for row in found if row["uuid"] in self.sources], [], None+ if query in self.answers:+ rows = [r for r in self.answers[query] if r["uuid"] > params["after"]]+ return rows[: params["limit"]], [], None+ self.writes.append((query, params["rows"]))+ return [], [], None+++def dream_row(uuid: str, description: str | None, **record: list[str]) -> dict:+ return {+ "uuid": uuid,+ "description": description,+ "content": None,+ "facts": record.get("facts"),+ "episodes": record.get("episodes"),+ }+++def older_dreams() -> BackfillGraph:+ """Two dream episodes written before records (a consolidation listing+ episodes, a proposal listing facts), one recorded since, one listing+ nothing; and the facts they and a user's chat turn produced."""+ return BackfillGraph(+ episodes=[+ dream_row("d1", "dream-pass consolidation; src_episodes=e0,e9"),+ dream_row("d2", "dream-pass proposal; rationale=r; src_facts=f1"),+ dream_row("d3", "dream-pass consolidation", facts=["f2"], episodes=[]),+ dream_row("d4", "dream-pass consolidation; src_episodes="),+ ],+ facts=[+ {"uuid": "c1", "episodes": ["d1"]},+ {"uuid": "c2", "episodes": ["d2", "d3"]},+ {"uuid": "c4", "episodes": ["d4"]},+ {"uuid": "merged", "episodes": ["d1", "chat"]},+ {"uuid": "user", "episodes": ["chat"]},+ ],+ )diff --git a/autogpt_platform/backend/backend/copilot/graphiti/migrations/backfill_pages.py b/autogpt_platform/backend/backend/copilot/graphiti/migrations/backfill_pages.pynew file mode 100644index 0000000000..8b540a2452--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/migrations/backfill_pages.py@@ -0,0 +1,37 @@+"""Paged reads and batched writes for the derivation backfill+(``backfill_derivations.py``, ``backfill_cascade.py``).++A read pages by uuid (``$after``, ``$limit``), so a dry run never writes, nor+creates, a graph; a write sends ``BATCH_SIZE`` rows at a time.+"""++from typing import Any++from backend.copilot.graphiti.falkordb_driver import AutoGPTFalkorDriver++BATCH_SIZE = 500+++async def pages(driver: AutoGPTFalkorDriver, query: str) -> list[dict[str, Any]]:+ """Every row of ``query``, read ``BATCH_SIZE`` at a time by uuid."""+ found: list[dict[str, Any]] = []+ after = ""+ while True:+ result = await driver.execute_query(query, after=after, limit=BATCH_SIZE)+ page = rows(result)+ found.extend(page)+ if len(page) < BATCH_SIZE:+ return found+ after = page[-1]["uuid"]+++async def write_rows(+ driver: AutoGPTFalkorDriver, query: str, batch: list[dict[str, Any]]+) -> None:+ """``query`` over ``batch`` as ``$rows``, ``BATCH_SIZE`` rows at a time."""+ for start in range(0, len(batch), BATCH_SIZE):+ await driver.execute_query(query, rows=batch[start : start + BATCH_SIZE])+++def rows(result: Any) -> list[dict[str, Any]]:+ return result[0] if result else []diff --git a/autogpt_platform/backend/backend/copilot/graphiti/migrations/dream_markers.py b/autogpt_platform/backend/backend/copilot/graphiti/migrations/dream_markers.pynew file mode 100644index 0000000000..7eeea8bc11--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/migrations/dream_markers.py@@ -0,0 +1,301 @@+"""List and resolve the dream citation markers left in the memory graphs.++A dream write's citation marker (``recall_derivation.py``) goes once its+writer has recorded and settled the write, or once+``recall_reconcile.reconcile`` completes it after the write landed or drops+it after the write was aborted. A marker whose write never landed and whose+writer is gone (it crashed, or its process was stopped) stays ``pending``:+for ``MARKER_EXPIRY_SECONDS`` (a day) a forget of anything it cites reports+``cleanup_error``; then it is marked ``expired`` and kept, never deleted for+its age alone. Only this command resolves an expired marker.++This lists every marker, one JSON line each: its graph, state and age, the+uuid its write's episode goes under, whether graphiti saved that episode,+whether the write landed, and how many facts and episodes it cites.+``--resolve <uuid>`` (repeatable) and ``--resolve-expired`` resolve markers,+a dry run unless ``--apply``: one whose episode graphiti saved is completed+as reconcile completes a landed one (its citations recorded, the write+settled, then the marker deleted); any other is deleted with the episode its+writer placed, by a statement that first confirms no saved episode has its+uuid. A pending marker younger than the bound is refused unless ``--force``,+since its writer may still be writing; the command says so and says what+``--force`` would do. A write that lands after its marker went is still+recorded and settled by its writer, but should that writer die before its+record, nothing names its facts' provenance. Each graph is resolved holding+its write lock; a graph locked past ``MARKERS_LOCK_WAIT_SECONDS``, or not+lockable, is skipped and counted busy, one that raises is counted failed,+and either, or a refusal, makes the script exit 1.++Usage:++ poetry run python -m backend.copilot.graphiti.migrations.dream_markers \\+ [--graph <name>] [--resolve <marker uuid> ...] [--resolve-expired] \\+ [--force] [--apply]+"""++import argparse+import asyncio+import logging+import math+import sys+from datetime import datetime, timezone+from typing import Any++from graphiti_core.driver.driver import GraphDriver+from pydantic import BaseModel, Field++from backend.copilot.graphiti.falkordb_driver import open_graph_driver+from backend.copilot.graphiti.graphs import list_graph_names+from backend.copilot.graphiti.recall_derivation import EXPIRED, PENDING+from backend.copilot.graphiti.recall_reconcile import (+ LANDED,+ MARKER_EXPIRY_SECONDS,+ MARKERS_QUERY,+ age_seconds,+ complete,+ drop_unlanded,+)+from backend.copilot.graphiti.scope_lock import LockState, graph_write_lock++from .backfill_legacy_forgets import MEMORY_GRAPH_PREFIXES++logger = logging.getLogger(__name__)++MARKERS_LOCK_WAIT_SECONDS = 60+# Markers read from one graph: far more than a graph should ever hold.+LIST_LIMIT = 10_000+++class Marker(BaseModel):+ """One marker as listed: whether graphiti ``saved`` its episode and+ whether its write ``landed`` (reconcile completes it), with how many+ facts and episodes it cites."""++ graph: str+ uuid: str+ state: str+ age_hours: float | None+ episode: str+ saved: bool+ landed: bool+ facts: int+ episodes: int+++class Selection(BaseModel):+ """The markers to resolve: these ``uuids``, and every expired one when+ ``expired``; a pending one under the bound only when ``force``."""++ uuids: set[str] = Field(default_factory=set)+ expired: bool = False+ force: bool = False++ def wanted(self, row: dict[str, Any]) -> bool:+ return row["uuid"] in self.uuids or (self.expired and row["state"] == EXPIRED)++ def refuses(self, row: dict[str, Any], now: datetime) -> bool:+ """A pending marker younger than the bound, without ``force``."""+ young = age_seconds(row["created_at"], now) < MARKER_EXPIRY_SECONDS+ return not self.force and row["state"] == PENDING and young++ def any(self) -> bool:+ return bool(self.uuids) or self.expired+++class Resolved(BaseModel):+ """Markers ``completed`` (their write landed) and ``deleted`` (it never+ did), or with a dry run that would be; ``kept`` (an episode under the+ uuid was saved since it was read, or the completion stopped short);+ ``refused`` (pending under the bound), each with a ``note``; and the+ graphs ``busy`` or ``failed``."""++ completed: int = 0+ deleted: int = 0+ kept: int = 0+ refused: int = 0+ busy: int = 0+ failed: int = 0+ notes: list[str] = Field(default_factory=list)++ def plus(self, other: "Resolved") -> "Resolved":+ mine = self.model_dump(exclude={"notes"})+ theirs = other.model_dump(exclude={"notes"})+ counts = {key: value + theirs[key] for key, value in mine.items()}+ return Resolved(**counts, notes=[*self.notes, *other.notes])+++async def list_markers(driver: GraphDriver, graph: str) -> list[Marker]:+ """Every marker in ``graph``, landed ones first."""+ now = datetime.now(timezone.utc)+ return [_listed(graph, row, now) for row in await _markers(driver, graph)]+++async def resolve_graph(+ driver: GraphDriver, graph: str, selection: Selection, *, apply: bool+) -> Resolved:+ """Resolve ``graph``'s selected markers holding its write lock; a dry+ run counts what it would do."""+ now = datetime.now(timezone.utc)+ if not apply:+ done = Resolved()+ for row in await _markers(driver, graph):+ if selection.wanted(row):+ _plan(graph, row, selection.refuses(row, now), now, done)+ return done+ async with graph_write_lock(graph, wait_seconds=MARKERS_LOCK_WAIT_SECONDS) as lock:+ if lock is not LockState.HELD:+ logger.warning(f"Skipped graph {graph[:20]}: write lock {lock.value}")+ return Resolved(busy=1)+ done = Resolved()+ for row in await _markers(driver, graph):+ if not selection.wanted(row):+ continue+ if selection.refuses(row, now):+ _refuse(graph, row, now, done)+ else:+ await _resolve(driver, graph, row, done)+ return done+++def _plan(+ graph: str, row: dict[str, Any], refused: bool, now: datetime, done: Resolved+) -> None:+ """Count what resolving ``row`` would do."""+ if refused:+ _refuse(graph, row, now, done)+ elif _saved(row):+ done.completed += 1+ else:+ done.deleted += 1+++def _refuse(graph: str, row: dict[str, Any], now: datetime, done: Resolved) -> None:+ """Refuse a pending marker under the bound, saying what ``--force`` does."""+ hours = age_seconds(row["created_at"], now) / 3600+ would = (+ "complete it: record and settle its write, then delete it"+ if _saved(row)+ else "delete it, and the episode its writer placed if none under its "+ "uuid was saved by then"+ )+ done.refused += 1+ done.notes.append(+ f"refused marker {row['uuid']} in graph {graph[:20]}: pending for "+ f"{hours:.1f}h, under the {MARKER_EXPIRY_SECONDS // 3600}h bound, so its "+ f"writer may still be writing; --force would {would}"+ )+++async def _resolve(+ driver: GraphDriver, graph: str, row: dict[str, Any], done: Resolved+) -> None:+ """Complete a marker whose episode graphiti saved, else delete it."""+ if _saved(row):+ if await complete(driver, graph, row):+ done.completed += 1+ else:+ done.kept += 1+ elif await drop_unlanded(driver, row["uuid"]):+ done.deleted += 1+ else:+ done.kept += 1+++def _saved(row: dict[str, Any]) -> bool:+ return row["rank"] == LANDED or bool(row["saved"])+++def _listed(graph: str, row: dict[str, Any], now: datetime) -> Marker:+ age = age_seconds(row["created_at"], now)+ return Marker(+ graph=graph,+ uuid=row["uuid"],+ state=row["state"],+ age_hours=None if math.isinf(age) else round(age / 3600, 1),+ episode=row["episode"],+ saved=bool(row["saved"]),+ landed=row["rank"] == LANDED,+ facts=len(row["facts"]),+ episodes=len(row["episodes"]),+ )+++async def _markers(driver: GraphDriver, graph: str) -> list[dict[str, Any]]:+ result = await driver.execute_query(MARKERS_QUERY, group_id=graph, limit=LIST_LIMIT)+ return result[0] if result else []+++async def run(+ selection: Selection, *, apply: bool, graph: str | None = None+) -> Resolved:+ """List every memory graph's markers (or ``graph``'s), printing one JSON+ line each, and resolve the selected ones."""+ names = [graph] if graph else await _memory_graph_names()+ totals = Resolved()+ for name in names:+ driver = open_graph_driver(name)+ try:+ for marker in await list_markers(driver, name):+ print(marker.model_dump_json())+ if selection.any():+ found = await resolve_graph(driver, name, selection, apply=apply)+ totals = totals.plus(found)+ except Exception:+ logger.warning(f"Markers failed for graph {name[:20]}", exc_info=True)+ totals.failed += 1+ finally:+ await driver.close()+ return totals+++async def _memory_graph_names() -> list[str]:+ names = await list_graph_names()+ return sorted(name for name in names if name.startswith(MEMORY_GRAPH_PREFIXES))+++async def main(args: argparse.Namespace) -> int:+ logging.basicConfig(+ level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s"+ )+ selection = Selection(+ uuids=set(args.resolve or []), expired=args.resolve_expired, force=args.force+ )+ totals = await run(selection, apply=args.apply, graph=args.graph)+ for note in totals.notes:+ print(note)+ if selection.any():+ verb = "resolved" if args.apply else "would resolve (dry run)"+ print(+ f"{verb}: {totals.completed} completed, {totals.deleted} deleted; "+ f"{totals.kept} kept, {totals.refused} refused"+ )+ if totals.refused:+ print("a refused marker is resolved only with --force")+ if totals.busy or totals.failed:+ print(+ f"skipped {totals.busy} busy and {totals.failed} failed graphs: run again"+ )+ return 1 if totals.refused or totals.busy or totals.failed else 0+++def parser() -> argparse.ArgumentParser:+ """The command line: lists, and resolves only with ``--apply``."""+ cli = argparse.ArgumentParser(description=__doc__)+ cli.add_argument("--graph", help="Only this graph.")+ cli.add_argument(+ "--resolve", action="append", metavar="UUID", help="Resolve this marker."+ )+ cli.add_argument(+ "--resolve-expired", action="store_true", help="Resolve every expired marker."+ )+ cli.add_argument(+ "--force",+ action="store_true",+ help="Resolve a pending marker under the bound too (its writer may be alive).",+ )+ cli.add_argument("--apply", action="store_true", help="Write, under each lock.")+ return cli+++if __name__ == "__main__":+ sys.exit(asyncio.run(main(parser().parse_args())))diff --git a/autogpt_platform/backend/backend/copilot/graphiti/migrations/dream_markers_test.py b/autogpt_platform/backend/backend/copilot/graphiti/migrations/dream_markers_test.pynew file mode 100644index 0000000000..7f4e7f8a4d--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/migrations/dream_markers_test.py@@ -0,0 +1,255 @@+"""Unit tests for the operator's dream marker command: what it lists, what a+dry run would resolve, and what ``--apply`` resolves under the graph's write+lock (a marker whose episode graphiti saved is completed, any other deleted+only while no saved episode has its uuid); a pending marker younger than the+bound is refused unless ``--force``, saying what ``--force`` would do. The+live runs are in+``recall_marker_integration_test.py`` and+``recall_marker_crash_integration_test.py``.+"""++from datetime import datetime, timedelta, timezone+from typing import Any+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from backend.copilot.graphiti.recall_fake_redis import FakeRedis+from backend.copilot.graphiti.recall_reconcile import (+ DROP_UNLANDED_QUERY,+ KEPT,+ LANDED,+ MARKERS_QUERY,+ UNLANDED,+)+from backend.copilot.graphiti.scope import write_lock_key++from . import dream_markers+++def _row(+ uuid: str, rank: int, *, state: str = "pending", saved: bool = False, hours=1+) -> dict[str, Any]:+ created = datetime.now(timezone.utc) - timedelta(hours=hours)+ return {+ "uuid": uuid,+ "episode": f"episode-{uuid}",+ "facts": ["f1", "f2"],+ "episodes": ["ep1"],+ "created_at": created.isoformat(),+ "state": state,+ "saved": saved,+ "rank": rank,+ }+++def _rows() -> list[dict[str, Any]]:+ """A write that landed, one crashed mid-save (its episode saved), one+ that never landed and one expired; aged from the moment the test asks,+ so a test that runs long after this module was imported sees the same+ ages."""+ return [+ _row("landed", LANDED, saved=True),+ _row("partial", UNLANDED, saved=True),+ _row("young", UNLANDED),+ _row("old", KEPT, state="expired", hours=30),+ ]+++def _driver(*results: list[dict[str, Any]]) -> MagicMock:+ driver = MagicMock()+ driver.execute_query = AsyncMock(side_effect=[(r, [], None) for r in results])+ driver.close = AsyncMock()+ return driver+++class TestList:+ @pytest.mark.asyncio+ async def test_lists_every_marker_with_its_state_age_and_landing(self) -> None:+ unreadable = {**_rows()[3], "created_at": "long ago"}+ driver = _driver([*_rows()[:3], unreadable])++ markers = await dream_markers.list_markers(driver, "user_a")++ read = driver.execute_query.await_args_list[0]+ assert read.args[0] == MARKERS_QUERY+ assert read.kwargs == {"group_id": "user_a", "limit": dream_markers.LIST_LIMIT}+ assert [(m.uuid, m.state, m.saved, m.landed) for m in markers] == [+ ("landed", "pending", True, True),+ ("partial", "pending", True, False),+ ("young", "pending", False, False),+ ("old", "expired", False, False),+ ]+ assert markers[0].age_hours == pytest.approx(1, abs=0.1)+ assert markers[3].age_hours is None+ assert (markers[0].facts, markers[0].episodes) == (2, 1)+ assert markers[0].episode == "episode-landed"+++class TestResolve:+ @pytest.mark.asyncio+ async def test_a_dry_run_counts_what_it_would_do_and_writes_nothing(+ self,+ ) -> None:+ driver = _driver(_rows())+ wanted = dream_markers.Selection(+ uuids={"landed", "young"}, expired=True, force=True+ )++ found = await dream_markers.resolve_graph(driver, "user_a", wanted, apply=False)++ assert found == dream_markers.Resolved(completed=1, deleted=2)+ assert [c.args[0] for c in driver.execute_query.await_args_list] == [+ MARKERS_QUERY+ ]++ @pytest.mark.asyncio+ async def test_apply_completes_the_saved_and_deletes_the_rest_under_the_lock(+ self, lock_redis: FakeRedis+ ) -> None:+ """A write whose episode graphiti saved, even in part, is completed+ as reconcile would; the rest go only while no saved episode has+ their uuid (the second's was saved since the read)."""+ driver = _driver(_rows(), [{"dropped": 1}], [{"dropped": 0}])+ held: list[bool] = []++ async def complete(driver_, graph: str, row: dict[str, Any]) -> bool:+ held.append(write_lock_key(graph) in lock_redis.values)+ return row["uuid"] == "landed"++ wanted = dream_markers.Selection(+ uuids={"landed", "partial", "young"}, expired=True, force=True+ )+ with patch.object(dream_markers, "complete", complete):+ found = await dream_markers.resolve_graph(+ driver, "user_a", wanted, apply=True+ )++ assert found == dream_markers.Resolved(completed=1, deleted=1, kept=2)+ assert held == [True, True]+ drops = [+ c.kwargs+ for c in driver.execute_query.await_args_list+ if c.args[0] == DROP_UNLANDED_QUERY+ ]+ assert drops == [{"uuid": "young"}, {"uuid": "old"}]++ @pytest.mark.asyncio+ @pytest.mark.parametrize("apply", [True, False], ids=["apply", "dry run"])+ async def test_a_young_pending_marker_is_refused_without_force(+ self, apply: bool+ ) -> None:+ """Its writer may still be writing. The refusal says what ``--force``+ would do; a pending marker past the bound is resolved."""+ stale = _row("stale", UNLANDED, hours=30)+ driver = _driver([*_rows()[:3], stale], [{"dropped": 1}])+ wanted = dream_markers.Selection(uuids={"landed", "young", "stale"})++ found = await dream_markers.resolve_graph(driver, "user_a", wanted, apply=apply)++ assert (found.completed, found.deleted, found.refused) == (0, 1, 2)+ landed, young = found.notes+ assert landed.startswith("refused marker landed in graph user_a: pending ")+ assert "under the 24h bound, so its writer may still be writing" in landed+ assert landed.endswith(+ "--force would complete it: record and settle its write, then delete it"+ )+ assert young.endswith(+ "--force would delete it, and the episode its writer placed if none "+ "under its uuid was saved by then"+ )+ drops = [+ c.kwargs+ for c in driver.execute_query.await_args_list+ if c.args[0] == DROP_UNLANDED_QUERY+ ]+ assert drops == ([{"uuid": "stale"}] if apply else [])++ @pytest.mark.asyncio+ async def test_only_the_selected_markers_are_touched(self) -> None:+ driver = _driver(_rows(), [{"dropped": 1}])++ found = await dream_markers.resolve_graph(+ driver,+ "user_a",+ dream_markers.Selection(expired=True),+ apply=True,+ )++ assert found == dream_markers.Resolved(deleted=1)+ assert driver.execute_query.await_args_list[1].kwargs == {"uuid": "old"}++ @pytest.mark.asyncio+ async def test_a_busy_graph_is_skipped_unwritten(+ self, lock_redis: FakeRedis+ ) -> None:+ lock_redis.values[write_lock_key("user_a")] = "an ingestion's token"+ driver = _driver(_rows())++ with patch.object(dream_markers, "MARKERS_LOCK_WAIT_SECONDS", 0):+ found = await dream_markers.resolve_graph(+ driver, "user_a", dream_markers.Selection(expired=True), apply=True+ )++ assert found == dream_markers.Resolved(busy=1)+ driver.execute_query.assert_not_awaited()+++class TestCommandLine:+ @pytest.mark.asyncio+ async def test_lists_then_resolves_and_exits_1_while_a_graph_failed(+ self, capsys: pytest.CaptureFixture[str]+ ) -> None:+ good, bad = _driver([_rows()[2]], [_rows()[2]], [{"dropped": 1}]), _driver()+ bad.execute_query.side_effect = RuntimeError("down")+ opened = MagicMock(side_effect=[good, bad])+ args = dream_markers.parser().parse_args(+ ["--resolve", "young", "--resolve", "other", "--force", "--apply"]+ )++ with (+ patch.object(dream_markers, "open_graph_driver", opened),+ patch.object(+ dream_markers,+ "list_graph_names",+ AsyncMock(return_value=["user_b", "other", "user_a"]),+ ),+ ):+ code = await dream_markers.main(args)++ out = capsys.readouterr().out+ assert code == 1+ assert [c.args[0] for c in opened.call_args_list] == ["user_a", "user_b"]+ assert '"uuid":"young"' in out and '"graph":"user_a"' in out+ assert "resolved: 0 completed, 1 deleted; 0 kept" in out+ assert "skipped 0 busy and 1 failed graphs" in out+ good.close.assert_awaited_once()+ bad.close.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_a_refusal_is_printed_and_exits_1(+ self, capsys: pytest.CaptureFixture[str]+ ) -> None:+ driver = _driver([_rows()[2]], [_rows()[2]])+ args = dream_markers.parser().parse_args(+ ["--graph", "user_a", "--resolve", "young", "--apply"]+ )++ with patch.object(+ dream_markers, "open_graph_driver", MagicMock(return_value=driver)+ ):+ code = await dream_markers.main(args)++ out = capsys.readouterr().out+ assert code == 1+ assert "refused marker young in graph user_a" in out+ assert "resolved: 0 completed, 0 deleted; 0 kept, 1 refused" in out+ assert "a refused marker is resolved only with --force" in out++ def test_a_bare_run_only_lists(self) -> None:+ args = dream_markers.parser().parse_args([])+ wanted = dream_markers.Selection(+ uuids=set(args.resolve or []), expired=args.resolve_expired+ )++ assert not wanted.any() and not args.applydiff --git a/autogpt_platform/backend/backend/copilot/graphiti/migrations/legacy_citations.py b/autogpt_platform/backend/backend/copilot/graphiti/migrations/legacy_citations.pynew file mode 100644index 0000000000..02341f32bb--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/migrations/legacy_citations.py@@ -0,0 +1,133 @@+"""What a dream episode written before derivation records cites, read back+from its ``source_description`` for the derivation backfill+(``backfill_derivations.py``), and checked before it is trusted.++Before records, the dream wrote a write's first five citations of each kind+into its description, after the model's free-text rationale and with the+same ``;`` delimiter, so a rationale could forge one (Codex's probe:+``rationale="ordinary; src_facts=<uuid>"``). A description is read only in+the shapes the dream wrote, its keys ending it: ``dream-pass consolidation;+src_episodes=<ids>``, ``dream-pass proposal[; rationale=<text>][;+src_facts=<ids>]``, and this branch's first form, which could end with both+keys. A description naming a ``src_`` key twice, or anywhere before those+final keys (a rationale quoting one), or listing an id with a space in it,+is ambiguous and attributes nothing. Each uuid read back must then name a+fact or an episode the graph has, a fact in the dream episode's own scope+(episode citations are not scoped: ``dream/citations.py``); the others are+dropped and counted. Unlike a new write, an older one citing a fact of+another scope cannot be dropped: it keeps its other citations, so it could+escape that fact's forget. A shape a forged rationale could still take (it+ends the description with one key, naming a real source in the right+scope) cannot be told from a real one; new writes never put citations or a+raw ``;`` in their description.+"""++import re+from collections.abc import Iterable++from pydantic import BaseModel, Field++from backend.copilot.dream.citations import scope_key+from backend.copilot.graphiti.falkordb_driver import AutoGPTFalkorDriver++from .backfill_pages import BATCH_SIZE, rows++_SUFFIX = re.compile(+ r"(?:;\s*src_episodes=(?P<episodes>[^;]*))?"+ r"(?:;\s*src_facts=(?P<facts>[^;]*))?\s*$"+)+_KEYS = ("src_facts=", "src_episodes=")+++class LegacyCitations(BaseModel):+ """What one description lists: ``facts`` and ``episodes``, or nothing+ and ``ambiguous`` when its shape is not one the dream wrote."""++ facts: list[str] = Field(default_factory=list)+ episodes: list[str] = Field(default_factory=list)+ ambiguous: bool = False+++def described_citations(description: str | None) -> LegacyCitations:+ """The citations a pre-record dream description lists at its end."""+ text = description or ""+ if any(text.count(key) > 1 for key in _KEYS):+ return LegacyCitations(ambiguous=True)+ found = _SUFFIX.search(text) # always, possibly empty, at the end+ if found is None:+ return LegacyCitations()+ if "src_" in text[: found.start()]:+ return LegacyCitations(ambiguous=True)+ facts = _ids(found.group("facts"))+ episodes = _ids(found.group("episodes"))+ if facts is None or episodes is None:+ return LegacyCitations(ambiguous=True)+ return LegacyCitations(facts=facts, episodes=episodes)+++def _ids(listed: str | None) -> list[str] | None:+ """The ids a key lists, or None when one of them has a space in it."""+ ids = [part.strip() for part in (listed or "").split(",") if part.strip()]+ return None if any(re.search(r"\s", uuid) for uuid in ids) else ids+++class Checked(BaseModel):+ """The citations of the episodes read back that name a source the graph+ has (a fact in the episode's own scope), by episode uuid; how many it+ ``rejected``."""++ cited: dict[str, LegacyCitations] = Field(default_factory=dict)+ rejected: int = 0+++async def checked(+ driver: AutoGPTFalkorDriver,+ described: dict[str, LegacyCitations],+ scopes: dict[str, str],+) -> Checked:+ """Keep each citation in ``described`` (by dream episode) whose source+ the graph has: a fact in the scope ``scopes`` gives its dream episode,+ an episode of any scope."""+ cited_facts = [uuid for c in described.values() for uuid in c.facts]+ cited_episodes = [uuid for c in described.values() for uuid in c.episodes]+ facts = await _found(driver, FACT_SCOPES_QUERY, cited_facts)+ episodes = await _found(driver, CITED_EPISODES_QUERY, cited_episodes)+ done = Checked()+ for uuid, citations in described.items():+ wanted = scope_key(scopes.get(uuid))+ kept = LegacyCitations(+ facts=[f for f in citations.facts if facts.get(f) == wanted],+ episodes=[e for e in citations.episodes if e in episodes],+ )+ done.rejected += len(citations.facts) + len(citations.episodes)+ done.rejected -= len(kept.facts) + len(kept.episodes)+ done.cited[uuid] = kept+ return done+++async def _found(+ driver: AutoGPTFalkorDriver, query: str, uuids: Iterable[str]+) -> dict[str, str]:+ """Each of ``uuids`` the graph has, with the scope its row gives (a+ fact's; ``UNSCOPED`` for a row that gives none)."""+ listed = list(dict.fromkeys(uuids))+ found: dict[str, str] = {}+ for start in range(0, len(listed), BATCH_SIZE):+ result = await driver.execute_query(+ query, uuids=listed[start : start + BATCH_SIZE]+ )+ found |= {row["uuid"]: scope_key(row.get("scope")) for row in rows(result)}+ return found+++FACT_SCOPES_QUERY = """+MATCH ()-[e:RELATES_TO]->()+WHERE e.uuid IN $uuids+RETURN e.uuid AS uuid, e.scope AS scope+"""++CITED_EPISODES_QUERY = """+MATCH (ep:Episodic)+WHERE ep.uuid IN $uuids+RETURN ep.uuid AS uuid+"""diff --git a/autogpt_platform/backend/backend/copilot/graphiti/migrations/legacy_citations_test.py b/autogpt_platform/backend/backend/copilot/graphiti/migrations/legacy_citations_test.pynew file mode 100644index 0000000000..0347c98958--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/migrations/legacy_citations_test.py@@ -0,0 +1,106 @@+"""Unit tests for reading back what an older dream episode's description+cites (``legacy_citations.py``): only the shapes the dream wrote, keys+ending it, and only uuids the graph has in the episode's own scope. The live+run is ``graphiti/backfill_derivations_integration_test.py``.+"""++import pytest++from . import backfill_derivations as backfill+from .backfill_fake import BackfillGraph, dream_row+from .legacy_citations import LegacyCitations, described_citations+++class TestDescribedCitations:+ @pytest.mark.parametrize(+ "description, cited",+ [+ ("dream-pass consolidation; src_episodes=e1,e2", ([], ["e1", "e2"])),+ ("dream-pass proposal; rationale=r; src_facts=f1", (["f1"], [])),+ (+ "dream-pass proposal; src_episodes=e1; src_facts=f1,f2",+ (["f1", "f2"], ["e1"]),+ ),+ ("dream-pass consolidation; src_episodes=", ([], [])),+ ("dream-pass proposal", ([], [])),+ (None, ([], [])),+ ("dream-pass proposal; rationale=ordinary; src_facts=f9", (["f9"], [])),+ ],+ ids=[+ "consolidation",+ "proposal",+ "both kinds",+ "empty list",+ "nothing listed",+ "no description",+ "a forged key that ends it reads as written",+ ],+ )+ def test_reads_the_uuids_each_kind_lists(+ self, description: str | None, cited: tuple[list[str], list[str]]+ ) -> None:+ read_back = described_citations(description)+ assert ((read_back.facts, read_back.episodes), read_back.ambiguous) == (+ cited,+ False,+ )++ @pytest.mark.parametrize(+ "description",+ [+ "dream-pass proposal; rationale=see src_facts=x; src_facts=f1",+ "dream-pass proposal; rationale=r; src_facts=f9; src_facts=f1",+ "dream-pass proposal; rationale=a, src_facts=f9",+ "dream-pass proposal; rationale=r; src_facts=f1 and f2",+ "dream-pass consolidation; src_facts=f1; src_episodes=e1",+ ],+ ids=[+ "a key before the end",+ "a key twice",+ "a key not its own field",+ "an id with a space",+ "keys out of order",+ ],+ )+ def test_an_ambiguous_shape_attributes_nothing(self, description: str) -> None:+ assert described_citations(description) == LegacyCitations(ambiguous=True)+++class TestLegacyCitationsAreChecked:+ @pytest.mark.asyncio+ async def test_a_source_the_graph_lacks_or_a_fact_of_another_scope_is_dropped(+ self,+ ) -> None:+ """The graph's sources are unscoped. A project write keeps the chat+ turn it cites (episode citations are not scoped) and loses the+ global fact."""+ project = '{"content": "x", "scope": "project:bread"}'+ driver = BackfillGraph(+ episodes=[+ dream_row("d1", "dream-pass proposal; rationale=r; src_facts=f1,ghost"),+ {+ **dream_row("d2", "dream-pass consolidation; src_episodes=e0"),+ "content": project,+ },+ {+ **dream_row("d3", "dream-pass proposal; rationale=r; src_facts=f2"),+ "content": project,+ },+ dream_row(+ "d4",+ "dream-pass proposal; rationale=a; b; src_facts=f1; src_facts=f2",+ ),+ ],+ facts=[],+ )++ found = await backfill.backfill_graph(driver, apply=True, cascade_forgets=False)++ [(_, records)] = driver.writes+ assert records == [+ {"uuid": "d1", "facts": ["f1"], "episodes": []},+ {"uuid": "d2", "facts": [], "episodes": ["e0"]},+ {"uuid": "d3", "facts": [], "episodes": []},+ {"uuid": "d4", "facts": [], "episodes": []},+ ]+ assert (found.rejected, found.ambiguous) == (2, 1)diff --git a/autogpt_platform/backend/backend/copilot/graphiti/provenance_pending.py b/autogpt_platform/backend/backend/copilot/graphiti/provenance_pending.pynew file mode 100644index 0000000000..9ef474092b--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/provenance_pending.py@@ -0,0 +1,117 @@+"""The graphs holding a dream citation marker whose record failed, and the+sweep the dream reaper runs over them.++When a dream write's derivation record fails after its graph write, or the+graph write itself raises after its marker was written, the ingestion worker+(``marked_write.py``) notes the graph here: a Redis set of graph names,+``PENDING_KEY``. Each run of the dream reaper (``dream/reaper.py``) sweeps+up to ``SWEEP_MAX_GRAPHS`` of them: it takes the graph's write lock,+completes or resolves its markers (``recall_reconcile.reconcile``) and,+once none is left that it could still act on (a write that landed but is+not yet recorded, or one that could still land), removes the graph from the+set while it still holds the lock, so a failure noted meanwhile (also under+that lock) is never lost. A graph it cannot lock, or whose reconcile fails,+stays for the next run.++This is the background path; the guarantee does not rest on it. A marker no+failure noted (a worker that died between its marker and its record) and one+whose note was lost (Redis unreachable) are completed by the next forget in+their graph, which reconciles before it cascades (``recall_forget.py``), by+the derivation backfill, or by an operator (``migrations/dream_markers.py``).+"""++import asyncio+import logging+from typing import Any++from pydantic import BaseModel++from backend.data import redis_client++from .falkordb_driver import open_graph_driver+from .recall_reconcile import reconcile+from .scope_lock import LockState, graph_write_lock++logger = logging.getLogger(__name__)++PENDING_KEY = "graphiti:provenance_pending"+SWEEP_MAX_GRAPHS = 20+SWEEP_LOCK_WAIT_SECONDS = 5+_REDIS_TIMEOUT_SECONDS = 2.0+++class Swept(BaseModel):+ """One sweep: the graphs it reconciled, the records it completed, and+ the graphs left for the next run, ``busy`` or ``failed``."""++ graphs: int = 0+ completed: int = 0+ busy: int = 0+ failed: int = 0+++async def note_pending(group_id: str) -> None:+ """Note graph ``group_id`` as holding a marker to reconcile; never+ raises (a lost note leaves the marker to the next forget)."""+ try:+ redis = await _redis()+ await asyncio.wait_for(+ redis.sadd(PENDING_KEY, group_id), _REDIS_TIMEOUT_SECONDS+ )+ except Exception:+ logger.warning(+ f"Could not note graph {group_id[:20]} as holding a pending dream "+ "record; the next forget there completes it",+ exc_info=True,+ )+++async def sweep_pending(*, max_graphs: int = SWEEP_MAX_GRAPHS) -> Swept:+ """Reconcile up to ``max_graphs`` of the noted graphs; never raises."""+ swept = Swept()+ try:+ redis = await _redis()+ graphs = await asyncio.wait_for(+ redis.srandmember(PENDING_KEY, max_graphs), _REDIS_TIMEOUT_SECONDS+ )+ except Exception:+ logger.warning("Could not read the graphs with pending dream records")+ return swept+ for group_id in graphs or []:+ await _sweep_graph(redis, str(group_id), swept)+ return swept+++async def _sweep_graph(redis: Any, group_id: str, swept: Swept) -> None:+ """Reconcile one graph under its write lock; forget the note once no+ marker is left, still holding the lock."""+ try:+ async with graph_write_lock(+ group_id, wait_seconds=SWEEP_LOCK_WAIT_SECONDS+ ) as lock:+ if lock is not LockState.HELD:+ swept.busy += 1+ return+ driver = open_graph_driver(group_id)+ try:+ done = await reconcile(driver, group_id)+ finally:+ await driver.close()+ swept.graphs += 1+ swept.completed += done.completed+ if not (done.incomplete() or done.waiting):+ await asyncio.wait_for(+ redis.srem(PENDING_KEY, group_id), _REDIS_TIMEOUT_SECONDS+ )+ except Exception:+ logger.warning(+ f"Could not reconcile graph {group_id[:20]}; kept for the next run",+ exc_info=True,+ )+ swept.failed += 1+++async def _redis() -> Any:+ return await asyncio.wait_for(+ redis_client.get_redis_async(), _REDIS_TIMEOUT_SECONDS+ )diff --git a/autogpt_platform/backend/backend/copilot/graphiti/provenance_pending_test.py b/autogpt_platform/backend/backend/copilot/graphiti/provenance_pending_test.pynew file mode 100644index 0000000000..4a4f7b9335--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/provenance_pending_test.py@@ -0,0 +1,114 @@+"""Unit tests for ``provenance_pending``: the graphs noted as holding a dream+citation marker whose record failed, and the reaper's sweep over them, each+reconciled under its write lock and forgotten only once nothing is left to+do there. The live run is ``recall_provenance_integration_test.py``.+"""++from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from . import provenance_pending, recall_reconcile+from .recall_fake_redis import FakeRedis+from .scope import write_lock_key+++def _opened() -> MagicMock:+ """``open_graph_driver``, handing out a driver that closes."""+ return MagicMock(return_value=MagicMock(close=AsyncMock()))+++def pending_redis(mocker) -> FakeRedis:+ redis = FakeRedis()+ mocker.patch(+ "backend.data.redis_client.get_redis_async",+ AsyncMock(return_value=redis),+ )+ return redis+++async def test_notes_a_graph_and_never_raises(pending_redis: FakeRedis, mocker) -> None:+ await provenance_pending.note_pending("user_a")+ assert pending_redis.sets[provenance_pending.PENDING_KEY] == {"user_a"}++ mocker.patch(+ "backend.data.redis_client.get_redis_async",+ AsyncMock(side_effect=ConnectionError("redis down")),+ )+ await provenance_pending.note_pending("user_b")+++async def test_reconciles_each_graph_under_its_lock_and_forgets_the_done(+ pending_redis: FakeRedis, lock_redis: FakeRedis+) -> None:+ """A graph with a landed marker left, one that could not be settled, or+ a write that could still land stays noted."""+ pending_redis.sets[provenance_pending.PENDING_KEY] = {+ "user_done",+ "user_left",+ "user_unsettled",+ "user_waiting",+ }+ held: list[bool] = []+ outcomes = {+ "user_done": recall_reconcile.Reconciled(completed=2, expired=1),+ "user_left": recall_reconcile.Reconciled(completed=1, left=True),+ "user_unsettled": recall_reconcile.Reconciled(unsettled=1),+ "user_waiting": recall_reconcile.Reconciled(waiting=1),+ }++ async def reconciled(driver, group_id: str):+ held.append(write_lock_key(group_id) in lock_redis.values)+ return outcomes[group_id]++ with (+ patch.object(provenance_pending, "reconcile", reconciled),+ patch.object(provenance_pending, "open_graph_driver", _opened()),+ ):+ swept = await provenance_pending.sweep_pending()++ assert swept == provenance_pending.Swept(graphs=4, completed=3)+ assert held == [True, True, True, True]+ assert pending_redis.sets[provenance_pending.PENDING_KEY] == {+ "user_left",+ "user_unsettled",+ "user_waiting",+ }+++async def test_a_busy_or_failing_graph_is_kept_for_the_next_run(+ pending_redis: FakeRedis, lock_redis: FakeRedis+) -> None:+ pending_redis.sets[provenance_pending.PENDING_KEY] = {"user_busy", "user_bad"}+ lock_redis.values[write_lock_key("user_busy")] = "an ingestion's token"++ with (+ patch.object(provenance_pending, "SWEEP_LOCK_WAIT_SECONDS", 0),+ patch.object(+ provenance_pending,+ "reconcile",+ AsyncMock(side_effect=RuntimeError("down")),+ ),+ patch.object(provenance_pending, "open_graph_driver", _opened()),+ ):+ swept = await provenance_pending.sweep_pending()++ assert (swept.busy, swept.failed, swept.graphs) == (1, 1, 0)+ assert pending_redis.sets[provenance_pending.PENDING_KEY] == {+ "user_busy",+ "user_bad",+ }+++async def test_without_redis_it_sweeps_nothing(mocker) -> None:+ mocker.patch(+ "backend.data.redis_client.get_redis_async",+ AsyncMock(side_effect=ConnectionError("redis down")),+ )++ assert await provenance_pending.sweep_pending() == provenance_pending.Swept()diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall.py b/autogpt_platform/backend/backend/copilot/graphiti/recall.pyindex 12adb43a5a..9b3e3c6554 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/recall.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall.py@@ -14,12 +14,15 @@ A fact (a ``RELATES_TO`` edge) is live while ``expired_at`` andan edge with no ``status`` predates the ``MemoryFact`` edge type and countsas ``active``. A forgotten fact (``forgotten_fact_predicate``) is never live:a forget stamps ``forgotten_at``, which only a forget writes-(``recall_forget.py``, or the legacy-forget backfill for older ones), so no-other writer's status or reason can make it look remembered again, nor make-a fact look forgotten. An episode is recallable while no forget has stamped+(``recall_forget.py`` and the cascade it runs on what the dream derived+from the fact, or the legacy-forget backfill for older ones), so no other+writer's status or reason can make it look remembered again, nor make a fact+look forgotten. An episode is recallable while no forget has stamped``redacted_at`` on it and none of the facts extracted from it is forgotten:one forgotten fact hides the whole text, since nothing records which-sentence it came from. The episode's other facts stay live.+sentence it came from. The episode's other facts stay live. Nor is an+episode a dream write placed and is still writing (``write_pending``,+``marked_write.py``): graphiti's save of it clears that."""import asyncio@@ -100,6 +103,9 @@ def forgotten_fact_predicate(alias: str = "e") -> str:``expiration_reason``: graphiti's attribute extraction fills both from anepisode's text, and a dream demotion picks its reason, so neither maymake a fact (and every episode and session it came from) look forgotten.++ It is never null: it only asks whether properties are set, so ``NOT`` it+ is the not-forgotten test on every edge, a live one included."""return f"({alias}.forgotten_at IS NOT NULL OR ({legacy_forget_predicate(alias)}))"@@ -145,16 +151,22 @@ def recallable_episode_predicate(alias: str = "e", forgotten: str = "forgotten")``alias``; the query must open with ``forgotten_facts_clause(forgotten)``."""return (f"{alias}.redacted_at IS NULL"+ f" AND {alias}.write_pending IS NULL"f" AND none(x IN coalesce({alias}.entity_edges, []) WHERE x IN {forgotten})")def is_recallable_episode(- entity_edges: list[str], forgotten: set[str], *, redacted: bool+ entity_edges: list[str],+ forgotten: set[str],+ *,+ redacted: bool,+ write_pending: bool = False,) -> bool:- """The recallable-episode test in Python: not ``redacted`` and citing no- uuid in ``forgotten`` among its ``entity_edges``."""- return not redacted and forgotten.isdisjoint(entity_edges)+ """The recallable-episode test in Python: not ``redacted``, not still+ being written, and citing no uuid in ``forgotten`` among its+ ``entity_edges``."""+ return not (redacted or write_pending) and forgotten.isdisjoint(entity_edges)async def search_facts(diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_ancestry_integration_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_ancestry_integration_test.pynew file mode 100644index 0000000000..1e43f37f55--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_ancestry_integration_test.py@@ -0,0 +1,228 @@+"""A dream write citing a derived fact that went out of use after the pass+read it, whose own source a forget then reached, on a live FalkorDB through+``dream/apply.py`` and the production ingestion worker+(``recall_sources.py``, ``recall_citations.py``, ``recall_landing.py``).++A pass reads the consolidation ``supplies``, derived from the user's flour+fact, while it is live, and proposes a fact citing it. The dream then+supersedes ``supplies``, and the user forgets the flour fact: the forget's+cascade walks through ``supplies``, no longer live and so left as it is.+Before, the proposal was then written live, resting on the forgotten fact+through ``supplies`` and carrying its content. Now:++- where the lock holds, the check before the write walks up from+ ``supplies`` to the forgotten flour fact and drops the write;+- where it does not (Redis down), with the forget landing while the write+ is in flight, the forget reports ``cleanup_error`` (the write cites a+ fact its cascade walked through), and the write's settle walks up the+ same way and retracts what the write made, erasing it for a hard forget,+ under the flour fact's name;+- a superseded fact whose own source is still there leaves the write alone.++Run with FalkorDB reachable (see ``conftest.py``)::++ poetry run pytest -m integration backend/copilot/graphiti/recall_ancestry_integration_test.py+"""++from collections.abc import AsyncIterator+from datetime import datetime, timezone+from unittest.mock import AsyncMock++import pytest+import pytest_asyncio+from pytest_mock import MockerFixture++from backend.copilot.dream import apply+from backend.copilot.dream.fetch import DreamInput+from backend.copilot.dream.schemas import (+ ConsolidatedFact,+ DreamOperations,+ ProposedFinding,+)++from . import scope_lock+from .falkordb_driver import AutoGPTFalkorDriver+from .memory_model import MemoryForgetFailureCode+from .recall import FORGOTTEN_FACT+from .recall_cascade_fixtures import (+ FLOUR,+ SUPPLIES,+ WEEKLY,+ dream,+ dream_once,+ fact_uuid,+ gather,+)+from .recall_cascade_walk import derived_reason+from .recall_derivation import MARKER_LABEL+from .recall_forget import retract+from .recall_integration_fixtures import (+ BuildClient,+ count,+ edge_row,+ ingest_facts,+ live_facts,+ patch_recall_boundaries,+ rows,+ stop_ingestion_workers,+)+from .recall_marker_fixtures import forget_mid_save+from .scope import MemoryScope++_MARKERS = f"MATCH (m:{MARKER_LABEL}) RETURN count(m) AS c"++[email protected](autouse=True)+def boundaries(mocker, scope_graph, stub_graphiti_client):+ patch_recall_boundaries(mocker, scope_graph[0], stub_graphiti_client)+++@pytest_asyncio.fixture(loop_scope="function")+async def dream_apply(mocker: MockerFixture) -> AsyncIterator[None]:+ """apply's Postgres side (the dream session and its summary) stubbed."""+ mocker.patch.object(apply, "_create_dream_session", AsyncMock(return_value="s"))+ mocker.patch.object(apply, "_write_dream_summary_message", AsyncMock())+ mocker.patch(+ "backend.copilot.dream.registry.ensure_dream_system_scheduled",+ AsyncMock(return_value=None),+ )+ yield+ await stop_ingestion_workers()+++async def _read_then_supersede(+ driver: AutoGPTFalkorDriver, scope: MemoryScope, build: BuildClient+) -> tuple[str, str, DreamInput]:+ """The flour fact, the consolidation ``supplies`` derived from it, and+ what a pass read while ``supplies`` was live; then the dream supersedes+ ``supplies``, as a demotion leaves a fact."""+ said, edges = await ingest_facts(driver, scope, build, [FLOUR], session_id="s-1")+ flour = edges[FLOUR[2]]+ consolidation = ConsolidatedFact(+ content=SUPPLIES[2],+ confidence=0.9,+ source_fact_uuids=[flour],+ source_episode_uuids=[said],+ )+ await dream_once(driver, scope, build, consolidation, SUPPLIES, "pass-1")+ supplies = await fact_uuid(driver, SUPPLIES[2])+ read = await gather(scope)+ assert supplies in read.known_fact_uuids, "the pass read it live"+ await driver.execute_query(+ "MATCH ()-[e:RELATES_TO {uuid: $uuid}]->() "+ "SET e.status = 'superseded', e.expiration_reason = 'stale_fact', "+ "e.expired_at = $now",+ uuid=supplies,+ now=datetime.now(timezone.utc).isoformat(),+ )+ return flour, supplies, read+++def _citing(supplies: str) -> DreamOperations:+ return DreamOperations(+ proposals=[+ ProposedFinding(+ content=WEEKLY[2],+ confidence=0.6,+ rationale="the bakery's flour",+ source_fact_uuids=[supplies],+ )+ ]+ )+++async def _weekly(driver: AutoGPTFalkorDriver, supplies: str) -> list[str]:+ """The proposal's fact, forgotten, erased or not; none if never+ written."""+ found = await rows(+ driver,+ "MATCH ()-[e:RELATES_TO]->() "+ "WHERE $supplies IN coalesce(e.derived_from_facts, []) "+ "OR coalesce(e.fact_redacted, e.fact) = $sentence "+ "RETURN e.uuid AS uuid",+ supplies=supplies,+ sentence=WEEKLY[2],+ )+ return [row["uuid"] for row in found]+++async def test_a_write_resting_on_a_forget_through_a_superseded_fact_is_dropped(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ """Fenced: the forget lands before the write, under the lock."""+ driver, scope = scope_graph+ flour, supplies, read = await _read_then_supersede(+ driver, scope, stub_graphiti_client+ )++ forgotten = await retract(scope, [flour])+ stats = await dream(+ driver, scope, stub_graphiti_client, _citing(supplies), read, WEEKLY, "pass-2"+ )++ assert (forgotten.failures, forgotten.passed) == ([], [supplies])+ assert stats["dropped_forgotten"] == 1, "dropped by the check before the write"+ assert await _weekly(driver, supplies) == [], "never written"+ assert await live_facts(driver) == {}+ assert await count(driver, _MARKERS) == 0, "its marker withdrawn"+++async def test_a_superseded_fact_whose_source_stays_leaves_the_write_alone(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ driver, scope = scope_graph+ _, supplies, read = await _read_then_supersede(driver, scope, stub_graphiti_client)++ stats = await dream(+ driver, scope, stub_graphiti_client, _citing(supplies), read, WEEKLY, "pass-2"+ )++ assert (stats["dropped_forgotten"], stats["proposal_count"]) == (0, 1)+ [weekly] = await _weekly(driver, supplies)+ assert weekly in await live_facts(driver)+ assert await count(driver, _MARKERS) == 0+++async def test_a_write_in_flight_resting_through_a_superseded_fact_is_settled(+ scope_graph, stub_graphiti_client, dream_apply, mocker: MockerFixture+) -> None:+ """Unfenced (Redis down): a hard forget of the flour fact runs right+ before graphiti saves the proposal's fact."""+ driver, scope = scope_graph+ flour, supplies, read = await _read_then_supersede(+ driver, scope, stub_graphiti_client+ )+ down = AsyncMock(side_effect=ConnectionError("redis down"))+ mocker.patch.object(scope_lock, "get_redis_async", down)++ with forget_mid_save(scope, flour, True, AsyncMock()) as forgotten:+ await dream(+ driver,+ scope,+ stub_graphiti_client,+ _citing(supplies),+ read,+ WEEKLY,+ "pass-2",+ )++ [first] = forgotten+ assert [f.code for f in first.failures] == [MemoryForgetFailureCode.CLEANUP_ERROR]+ assert first.passed == [supplies], "the write cites what its cascade walked"+ [weekly] = await _weekly(driver, supplies)+ assert await live_facts(driver) == {}, "its writer settled it on landing"+ row = await edge_row(driver, weekly)+ assert (row["reason"], row["fact"], row["fact_redacted"]) == (+ derived_reason(flour),+ FORGOTTEN_FACT,+ "",+ ), "retracted under the flour fact's name, and erased: it was purged"+ assert await count(driver, _MARKERS) == 0+ retried = await retract(scope, [flour], hard=True)+ assert (retried.failures, retried.resumed) == ([], [flour])diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade.pynew file mode 100644index 0000000000..b6bebb0f4c--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade.py@@ -0,0 +1,275 @@+"""A forget reaches what the dream derived from what it forgot.++Each dream write records what it was derived from, on its episode and on the+facts only dream episodes state (``recall_derivation.py``). After a forget+has retracted its facts and hidden their episodes, still holding the graph's+write lock (``recall_forget.py``), ``cascade`` retracts every live fact whose+record names a forgotten fact or a hidden episode, and hides every dream+episode whose record does; then again over what that retracted and hid,+until a round finds nothing new. Any one forgotten source is enough: a fact+derived from a forgotten one carries its content. A derived fact that is no+longer live (superseded, contradicted, or forgotten already) is left as it+is, but the walk goes on through it: what was derived from it rests on the+forgotten fact too. A fact the user stated has no record, and a derived fact+a user's own episode states too is passed over, as is what rests on it: it+has a source the forget did not reach.++A derived fact is retracted as a soft forget retracts one: ``forgotten_at``,+``status='retracted'``, its sentence moved to its audit copy, its entities'+summaries and attributes and their communities' summaries blanked, and every+episode citing it redacted. Its ``expiration_reason`` is+``derived_from_forgotten:<uuid>``, naming the root it descends from: the+fact the user forgot (``recall_cascade_walk.py``). A dream episode reached+without a live fact of its own (its fact was superseded, merged into a fact+a user stated, or never extracted) is redacted and the entities it mentions+blanked, since the dream reads its text. A hard forget cascades the same+way, softly: retraction takes the derived facts out of every read, and+deleting them would purge what the model's citations name, which can be+more than a fact truly rests on, beyond undoing. It also erases the text it+reaches (``recall_erase.py``): every derived fact's, retracted or walked+through, and every hidden dream episode's.++At most ``CASCADE_MAX_ROUNDS`` rounds and ``CASCADE_MAX_ITEMS`` derived facts+and dream episodes per forget. A forget stopped by a bound or a failed step+reports a ``cleanup_error`` on each root, and forgetting it again picks up+the facts already retracted for it (their reason names it) and goes on from+there, erasing when the root is gone: a hard forget purges its facts even+when its cascade stops short, and a repeated forget of one of them starts+from the records that name it (``recall_forget.py``). The episodes a forget+hid remember which forgotten facts they were hidden for (``redacted_for``,+``recall_hide.py``), so a cascade resumed after the purge still starts from+the root's own episodes. The derivation backfill starts one from hidden+episodes too (``seeds``).+"""++import logging+from collections.abc import Mapping+from typing import Any++from graphiti_core.driver.driver import GraphDriver++from .memory_model import ForgetResult, MemoryForgetFailure, MemoryStatus+from .recall_cascade_queries import (+ DERIVED_EPISODES_QUERY,+ DERIVED_FACTS_QUERY,+ EARLIER_QUERY,+ REDACT_DERIVED_QUERY,+ RETRACT_QUERY,+)+from .recall_cascade_walk import (+ DERIVED_FROM_FORGOTTEN,+ Found,+ Frontier,+ Walk,+ derived_reason,+)+from .recall_erase import erase as erase_text+from .recall_hide import REDACT_EPISODES_QUERY, scrub, scrub_entities++logger = logging.getLogger(__name__)++CASCADE_MAX_ITEMS = 500+CASCADE_MAX_ROUNDS = 10+++async def cascade(+ driver: GraphDriver,+ group_id: str,+ roots: list[str],+ now: str,+ result: ForgetResult,+ *,+ erase: bool = False,+ seeds: Mapping[str, str] | None = None,+ named: Mapping[str, str] | None = None,+) -> None:+ """Retract what the dream derived from ``roots``, the facts this forget+ has just retracted and hidden, or that are gone: ``result.derived`` gets+ each fact it retracts, ``result.passed`` each derived fact it only walks+ through and ``result.redacted_episodes`` each episode it hides; with+ ``erase`` (a hard forget, or a root that is gone), their text goes too.+ ``seeds`` are hidden episodes to start from as well, each mapped to the+ root it names, and ``named`` maps a root that is not itself the fact the+ user forgot (a derived fact a cascade retracted) to the one it names. A+ failed step or a bound reached is a failure on every root."""+ walk = Walk.start(+ roots,+ dict(seeds or {}),+ named=dict(named or {}),+ budget=CASCADE_MAX_ITEMS,+ erase=erase,+ )+ if not walk.names:+ return+ try:+ finished = await _run(driver, group_id, walk, now, result)+ except Exception as exc:+ logger.warning(f"Forget cascade failed in graph {group_id[:20]}", exc_info=True)+ result.failures.extend(+ MemoryForgetFailure.cleanup_error(name, exc) for name in walk.names+ )+ return+ if not finished:+ logger.warning(+ f"Forget cascade in graph {group_id[:20]} stopped at its bound "+ f"after {len(result.derived)} derived facts"+ )+ result.failures.extend(+ MemoryForgetFailure.derived_left(name) for name in walk.names+ )+++async def _run(+ driver: GraphDriver,+ group_id: str,+ walk: Walk,+ now: str,+ result: ForgetResult,+) -> bool:+ """Rounds until one finds nothing (True) or a bound stops them (False).+ A round the budget cut short, to nothing at all once it is spent, still+ left something behind."""+ frontier = await _resume(driver, walk, now, result)+ for _ in range(CASCADE_MAX_ROUNDS):+ found = await _derived(driver, walk, frontier)+ if not (found.facts or found.episodes):+ return not found.truncated+ frontier = await _retire(driver, group_id, walk, found, now, result)+ if found.truncated:+ return False+ last = await _derived(driver, walk, frontier)+ return not (last.facts or last.episodes or last.truncated)+++async def _resume(+ driver: GraphDriver, walk: Walk, now: str, result: ForgetResult+) -> Frontier:+ """The first frontier: the roots, the facts an earlier try retracted for+ them (their clean-up finished again), every episode citing one or hidden+ for one, and the seeds."""+ reasons = [derived_reason(name) for name in walk.names]+ earlier = _rows(await driver.execute_query(EARLIER_QUERY, reasons=reasons))+ for row in earlier:+ walk.root_of[row["uuid"]] = row["reason"].removeprefix(+ f"{DERIVED_FROM_FORGOTTEN}:"+ )+ again = [row["uuid"] for row in earlier]+ if again:+ await scrub(driver, again)+ facts = [*walk.roots, *again]+ hidden = [*walk.seeds, *await _redact_citing(driver, walk, facts, now, result)]+ if walk.erase:+ result.dream_writes.extend(await erase_text(driver, again, hidden))+ return Frontier(facts=facts, episodes=hidden)+++async def _derived(driver: GraphDriver, walk: Walk, frontier: Frontier) -> Found:+ """The facts and the dream episodes not reached yet whose record names+ something in ``frontier``, no more than the walk's budget."""+ if not (frontier.facts or frontier.episodes):+ return Found()+ params = {+ "facts": frontier.facts,+ "episodes": frontier.episodes,+ "seen": list(walk.root_of),+ "limit": walk.budget + 1,+ }+ facts = _rows(await driver.execute_query(DERIVED_FACTS_QUERY, **params))+ episodes = _rows(await driver.execute_query(DERIVED_EPISODES_QUERY, **params))+ if len(facts) + len(episodes) <= walk.budget:+ return Found(facts=facts, episodes=episodes)+ kept = facts[: walk.budget]+ rest = episodes[: walk.budget - len(kept)]+ return Found(facts=kept, episodes=rest, truncated=True)+++async def _retire(+ driver: GraphDriver,+ group_id: str,+ walk: Walk,+ found: Found,+ now: str,+ result: ForgetResult,+) -> Frontier:+ """Retract ``found``'s live facts and hide its episodes, each marker+ before any clean-up (erasing their text, and that of the facts passed+ through, on a hard forget); the next frontier: every fact it reached,+ retracted or passed through, and every episode it hid."""+ targets = [+ {"uuid": row["uuid"], "reason": derived_reason(walk.root(row["via"]))}+ for row in found.facts+ if row["live"]+ ]+ landed = await _retract(driver, group_id, targets, now)+ retracted = walk.reach([row for row in found.facts if row["uuid"] in landed])+ passed = walk.reach([row for row in found.facts if row["uuid"] not in landed])+ result.derived.extend(retracted)+ result.passed.extend(passed)+ tainted = walk.reach(found.episodes)+ mentioned: list[str] = []+ if tainted:+ rows = _rows(+ await driver.execute_query(REDACT_DERIVED_QUERY, uuids=tainted, now=now)+ )+ _note_redacted(result, tainted)+ mentioned = rows[0]["mentioned"] if rows else []+ walk.budget -= len(retracted) + len(passed) + len(tainted)+ if mentioned:+ await scrub_entities(driver, [], mentioned)+ reached = [*retracted, *passed]+ scrubbed = reached if walk.erase else retracted+ if scrubbed:+ await scrub(driver, scrubbed)+ hidden = await _redact_citing(driver, walk, reached, now, result)+ if walk.erase:+ erased = await erase_text(driver, reached, [*tainted, *hidden])+ result.dream_writes.extend(erased)+ return Frontier(facts=reached, episodes=[*tainted, *hidden])+++async def _retract(+ driver: GraphDriver, group_id: str, targets: list[dict[str, str]], now: str+) -> set[str]:+ """Retract each target still live, as a soft forget does, with its own+ reason; the uuids retracted."""+ if not targets:+ return set()+ rows = _rows(+ await driver.execute_query(+ RETRACT_QUERY,+ targets=targets,+ group_id=group_id,+ now=now,+ status=MemoryStatus.retracted.value,+ )+ )+ return {row["uuid"] for row in rows}+++async def _redact_citing(+ driver: GraphDriver,+ walk: Walk,+ facts: list[str],+ now: str,+ result: ForgetResult,+) -> list[str]:+ """Redact every episode citing one of ``facts``, or hidden for one, that+ the recall policy hides (all of them, for a fact forgotten now), as a+ forget does; the episodes the walk had not reached."""+ if not facts:+ return []+ rows = _rows(+ await driver.execute_query(REDACT_EPISODES_QUERY, uuids=facts, now=now)+ )+ _note_redacted(result, [row["uuid"] for row in rows])+ return walk.reach(rows)+++def _note_redacted(result: ForgetResult, episodes: list[str]) -> None:+ result.redacted_episodes = list(+ dict.fromkeys([*result.redacted_episodes, *episodes])+ )+++def _rows(result: Any) -> list[dict[str, Any]]:+ return result[0] if result else []diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_erase_integration_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_erase_integration_test.pynew file mode 100644index 0000000000..74b4dac791--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_erase_integration_test.py@@ -0,0 +1,182 @@+"""A hard forget erases the sentences the dream derived from the fact it+forgot, on a live FalkorDB, the dream's facts written through+``dream/apply.py`` and the production ingestion worker (``recall_erase.py``).++The bakery is ``recall_cascade_fixtures.py``'s. A soft forget of its flour+supplier keeps each derived sentence as an audit copy+(``recall_cascade_integration_test.py``); a hard one leaves none anywhere in+the graph, while a user's own words stay.++Run with FalkorDB reachable (see ``conftest.py``)::++ poetry run pytest -m integration backend/copilot/graphiti/recall_cascade_erase_integration_test.py+"""++from collections.abc import AsyncIterator+from datetime import datetime, timezone+from unittest.mock import AsyncMock++import pytest+import pytest_asyncio+from pytest_mock import MockerFixture++from backend.copilot.dream import apply+from backend.copilot.dream.forget_copies import pass_of++from . import recall+from .falkordb_driver import AutoGPTFalkorDriver+from .recall import FORGOTTEN_FACT+from .recall_cascade_fixtures import (+ BOULE,+ BOULE_FLOUR,+ FLOUR,+ SUPPLIES,+ WEEKLY,+ build_bakery,+)+from .recall_forget import retract+from .recall_integration_fixtures import (+ Fact,+ edge_row,+ episode_row,+ ingest_facts,+ live_facts,+ patch_recall_boundaries,+ rows,+ sentence_properties,+ stop_ingestion_workers,+)++[email protected](autouse=True)+def boundaries(mocker, scope_graph, stub_graphiti_client):+ patch_recall_boundaries(mocker, scope_graph[0], stub_graphiti_client)+++@pytest_asyncio.fixture(loop_scope="function")+async def dream_apply(mocker: MockerFixture) -> AsyncIterator[None]:+ """apply's Postgres side (the dream session and its summary) stubbed."""+ mocker.patch.object(apply, "_create_dream_session", AsyncMock(return_value="s"))+ mocker.patch.object(apply, "_write_dream_summary_message", AsyncMock())+ mocker.patch(+ "backend.copilot.dream.registry.ensure_dream_system_scheduled",+ AsyncMock(return_value=None),+ )+ yield+ await stop_ingestion_workers()+++async def _user_turn(driver: AutoGPTFalkorDriver, fact: str) -> str:+ """The user's chat turn that stated ``fact``."""+ [row] = await rows(+ driver,+ "MATCH (ep:Episodic) "+ "WHERE $fact IN ep.entity_edges AND ep.derived_from_facts IS NULL "+ "RETURN ep.uuid AS uuid",+ fact=fact,+ )+ return row["uuid"]+++# Said after the forget, between the entities the erased facts still join, so+# graphiti's dedup compares it with them, embeddings gone.+_PAYS: Fact = (+ "Sunrise Bakery",+ "Hill Country Mills",+ "Sunrise Bakery pays Hill Country Mills every month",+)+++async def _keys(driver: AutoGPTFalkorDriver, uuids: list[str]) -> dict[str, set[str]]:+ """Every property each of the edges ``uuids`` holds."""+ found = await rows(+ driver,+ "MATCH ()-[e:RELATES_TO]->() WHERE e.uuid IN $uuids "+ "RETURN e.uuid AS uuid, keys(e) AS keys",+ uuids=uuids,+ )+ return {row["uuid"]: set(row["keys"]) for row in found}+++async def test_a_hard_forget_erases_the_sentences_the_dream_derived(+ scope_graph, stub_graphiti_client, dream_apply, dream_copies: AsyncMock+) -> None:+ """The forgotten fact is deleted and its chat turn emptied. What the dream+ derived is retracted softly, its edges and reasons kept, and its text+ erased: each derived fact reads the placeholder with blank audit copies,+ the superseded proposal the walk only went through included, and each+ dream episode loses its body and rationale. Each derived fact's+ embedding goes too, and graphiti's searches still run past it. The+ user's turn about the boule, hidden too because graphiti listed the+ consolidation among its edges (as it lists an edge an episode+ invalidated), keeps its text."""+ driver, scope = scope_graph+ bakery = await build_bakery(driver, scope, stub_graphiti_client)+ await driver.execute_query(+ "MATCH ()-[e:RELATES_TO {uuid: $uuid}]->() "+ "SET e.status = 'superseded', e.expiration_reason = 'stale_fact', "+ "e.expired_at = $now",+ uuid=bakery.boule_flour,+ now=datetime.now(timezone.utc).isoformat(),+ )+ boule_turn = await _user_turn(driver, bakery.boule)+ await driver.execute_query(+ "MATCH (ep:Episodic {uuid: $uuid}) "+ "SET ep.entity_edges = ep.entity_edges + [$edge]",+ uuid=boule_turn,+ edge=bakery.supplies,+ )++ before = await _keys(driver, bakery.derived())+ assert all("fact_embedding" in keys for keys in before.values())++ result = await retract(scope, [bakery.flour], hard=True)++ assert (result.deleted, result.failures) == ([bakery.flour], [])+ assert sorted(result.derived) == sorted([bakery.supplies, bakery.weekly])+ assert await edge_row(driver, bakery.flour) == {}+ assert (await episode_row(driver, bakery.said))["hard_deleted_at"] is not None+ retired = {uuid: await edge_row(driver, uuid) for uuid in bakery.derived()}+ assert {uuid: row["reason"] for uuid, row in retired.items()} == {+ bakery.supplies: f"derived_from_forgotten:{bakery.flour}",+ bakery.weekly: f"derived_from_forgotten:{bakery.flour}",+ bakery.boule_flour: "stale_fact",+ }+ for row in retired.values():+ assert (row["fact"], row["fact_redacted"]) == (FORGOTTEN_FACT, "")+ assert (row["name"], row["name_redacted"]) == (FORGOTTEN_FACT, "")+ after = await _keys(driver, bakery.derived())+ assert {uuid: before[uuid] - keys for uuid, keys in after.items()} == dict.fromkeys(+ bakery.derived(), {"fact_embedding"}+ ), "the embedding goes, and nothing else the edge had"+ searched = await recall.search_facts(scope, "Sunrise Bakery flour", limit=10)+ assert {fact.uuid for fact in searched} <= {bakery.boule, bakery.cafe}+ _, pays = await ingest_facts(driver, scope, stub_graphiti_client, [_PAYS])+ assert pays[_PAYS[2]] in await live_facts(driver), "dedup ran past them"+ for sentence in (FLOUR[2], SUPPLIES[2], BOULE_FLOUR[2], WEEKLY[2]):+ assert await sentence_properties(driver, sentence) == set(), sentence+ dream_texts = await rows(+ driver,+ "MATCH (ep:Episodic) WHERE ep.uuid IN $uuids "+ "RETURN ep.content AS content, ep.source_description AS description, "+ "ep.redacted_at IS NOT NULL AS hidden ORDER BY ep.name",+ uuids=[+ bakery.supplies_episode,+ bakery.boule_flour_episode,+ bakery.weekly_episode,+ ],+ )+ assert [(r["content"], r["description"], r["hidden"]) for r in dream_texts] == [+ ("", "dream-pass consolidation", True),+ ("", "dream-pass proposal", True),+ ("", "dream-pass proposal", True),+ ]+ turn = await episode_row(driver, boule_turn)+ assert turn["redacted_at"] is not None and BOULE[2] in turn["content"]+ assert set(await live_facts(driver)) == {bakery.boule, bakery.cafe, pays[_PAYS[2]]}+ passes = sorted(pass_of(name) or name for name in result.dream_writes)+ assert passes == ["pass-1", "pass-2", "pass-3"], "each erased write names its pass"+ assert dream_copies.await_args is not None+ assert dream_copies.await_args.kwargs["dream_writes"] == result.dream_writesdiff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_fake.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_fake.pynew file mode 100644index 0000000000..5931c63df4--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_fake.py@@ -0,0 +1,148 @@+"""A small in-memory graph that answers a forget's cascade as FalkorDB would,+for ``recall_cascade_test.py``: each of the cascade's queries (by identity)+and the forget's own scrub, redaction and erasure, over facts and episodes+that carry a dream's record or not. Not collected by pytest.+"""++from typing import Any++from pydantic import BaseModel, Field++from . import recall_cascade, recall_erase, recall_hide+++class FakeFact(BaseModel):+ """A ``RELATES_TO`` fact: its record (``facts``, ``episodes``), the+ episodes that state it (``sources``), whether it is live, and its+ ``expiration_reason``."""++ facts: list[str] = Field(default_factory=list)+ episodes: list[str] = Field(default_factory=list)+ sources: list[str] = Field(default_factory=list)+ live: bool = True+ reason: str | None = None+++class FakeEpisode(BaseModel):+ """An episode: the facts it cites (``entity_edges``), a dream's record+ (``facts`` is None on a user's episode), and whether it is redacted."""++ cites: list[str] = Field(default_factory=list)+ facts: list[str] | None = None+ episodes: list[str] = Field(default_factory=list)+ redacted: bool = False+++class CascadeGraph:+ """What the cascade's queries see and change, by query."""++ def __init__(+ self, facts: dict[str, FakeFact], episodes: dict[str, FakeEpisode]+ ) -> None:+ self.facts = facts+ self.episodes = episodes+ self.queries: list[str] = []+ self.scrubbed: list[str] = []+ self.erased: list[str] = []+ # A query that raises, and facts the dream demotes between the+ # cascade's read and its retraction.+ self.fail_on: str | None = None+ self.demoted_meanwhile: set[str] = set()++ async def execute_query(+ self, query: str, **params: Any+ ) -> tuple[list[dict[str, Any]], list[str], None]:+ self.queries.append(query)+ if query == self.fail_on:+ raise RuntimeError("down")+ return self._answer(query, params), [], None++ def _answer(self, query: str, p: dict[str, Any]) -> list[dict[str, Any]]:+ if query == recall_cascade.EARLIER_QUERY:+ return [+ {"uuid": uuid, "reason": fact.reason}+ for uuid, fact in self.facts.items()+ if fact.reason in p["reasons"]+ ]+ if query == recall_hide.REDACT_EPISODES_QUERY:+ return self._redact_citing(p["uuids"])+ if query == recall_cascade.DERIVED_FACTS_QUERY:+ return self._derived_facts(p)[: p["limit"]]+ if query == recall_cascade.DERIVED_EPISODES_QUERY:+ return self._derived_episodes(p)[: p["limit"]]+ if query == recall_cascade.RETRACT_QUERY:+ return self._retract(p["targets"])+ if query == recall_cascade.REDACT_DERIVED_QUERY:+ for uuid in p["uuids"]:+ self.episodes[uuid].redacted = True+ return [{"mentioned": []}]+ if query == recall_hide.SCRUB_FACTS_QUERY:+ self.scrubbed.extend(p["uuids"])+ return [{"ends": []}]+ if query == recall_erase.ERASE_FACTS_QUERY:+ self.erased.extend(p["uuids"])+ if query == recall_erase.ERASE_DREAM_EPISODES_QUERY:+ dream = [x for x in p["uuids"] if self.episodes[x].facts is not None]+ self.erased.extend(dream)+ return []++ def _redact_citing(self, uuids: list[str]) -> list[dict[str, Any]]:+ rows = []+ for uuid, episode in sorted(self.episodes.items()):+ via = [x for x in episode.cites if x in uuids]+ if via:+ episode.redacted = True+ rows.append({"uuid": uuid, "via": via})+ return rows++ def _derived_facts(self, p: dict[str, Any]) -> list[dict[str, Any]]:+ rows = []+ for uuid, fact in sorted(self.facts.items()):+ via = [x for x in fact.facts if x in p["facts"]]+ via += [x for x in fact.episodes if x in p["episodes"]]+ stated = [s for s in fact.sources if self.episodes[s].facts is None]+ if via and not stated and uuid not in p["seen"]:+ rows.append({"uuid": uuid, "via": via, "live": fact.live})+ return rows++ def _derived_episodes(self, p: dict[str, Any]) -> list[dict[str, Any]]:+ rows = []+ for uuid, episode in sorted(self.episodes.items()):+ via = [x for x in episode.facts or [] if x in p["facts"]]+ via += [x for x in episode.episodes if x in p["episodes"]]+ if via and uuid not in p["seen"]:+ rows.append({"uuid": uuid, "via": via})+ return rows++ def _retract(self, targets: list[dict[str, str]]) -> list[dict[str, Any]]:+ for uuid in self.demoted_meanwhile:+ self.facts[uuid].live = False+ rows = []+ for target in targets:+ fact = self.facts[target["uuid"]]+ if fact.live:+ fact.live, fact.reason = False, target["reason"]+ rows.append({"uuid": target["uuid"]})+ return rows+++def chain() -> CascadeGraph:+ """User fact ``f`` (said in ``e0``); a consolidation ``c`` citing it (its+ dream episode ``dc``); a proposal ``p`` citing ``c`` (``dp``); a+ proposal ``q`` citing the dream episode ``dp``; and ``u``, a user fact+ that shares ``e0`` but was derived from nothing."""+ return CascadeGraph(+ facts={+ "f": FakeFact(live=False, reason="user_signal", sources=["e0"]),+ "u": FakeFact(sources=["e0"]),+ "c": FakeFact(facts=["f"], episodes=["e0"], sources=["dc"]),+ "p": FakeFact(facts=["c"], sources=["dp"]),+ "q": FakeFact(episodes=["dp"], sources=["dq"]),+ },+ episodes={+ "e0": FakeEpisode(cites=["f", "u"]),+ "dc": FakeEpisode(cites=["c"], facts=["f"], episodes=["e0"]),+ "dp": FakeEpisode(cites=["p"], facts=["c"]),+ "dq": FakeEpisode(cites=["q"], facts=[], episodes=["dp"]),+ },+ )diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_fixtures.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_fixtures.pynew file mode 100644index 0000000000..b5faa6ea78--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_fixtures.py@@ -0,0 +1,262 @@+"""A bakery's memory and what three dream passes derived from it, for the+live cascade tests (``recall_cascade*_integration_test.py``).++The user's facts go in through graphiti's ``add_episode``, the dream's+through ``dream/apply.py`` and the production ingestion worker, each dream+write extracting the fact its test scripts: so each derived fact carries the+record ingestion writes in production (``recall_derivation.py``). Not+collected by pytest.+"""++import ast+import re+from contextlib import AbstractContextManager, nullcontext+from types import SimpleNamespace+from typing import Any+from unittest.mock import AsyncMock, patch++from graphiti_core import Graphiti+from pydantic import BaseModel++from backend.copilot.dream import apply, fetch+from backend.copilot.dream.citations import fact_scopes+from backend.copilot.dream.fetch import DreamInput+from backend.copilot.dream.schemas import (+ ConsolidatedFact,+ DreamOperations,+ ProposedFinding,+)++from . import ingest+from .falkordb_driver import AutoGPTFalkorDriver+from .recall_integration_fixtures import (+ BuildClient,+ Fact,+ ingest_facts,+ model_of,+ rows,+ scripted_responses,+)+from .scope import MemoryScope++FLOUR: Fact = (+ "Sunrise Bakery",+ "Hill Country Mills",+ "Sunrise Bakery uses Hill Country Mills as its flour supplier",+)+# A fact the user stated about the same bakery, and one about something else.+BOULE: Fact = (+ "Sunrise Bakery",+ "Sourdough Boule",+ "Sunrise Bakery bakes a sourdough boule",+)+CAFE: Fact = ("Maria", "Oak Street Cafe", "Maria runs the Oak Street Cafe")+# What the dream derives: a consolidation of FLOUR, a proposal from it and+# BOULE, and a proposal from the consolidation's own dream text.+SUPPLIES: Fact = (+ "Hill Country Mills",+ "Sunrise Bakery",+ "Hill Country Mills supplies the flour Sunrise Bakery bakes with",+)+BOULE_FLOUR: Fact = (+ "Hill Country Mills",+ "Sourdough Boule",+ "Hill Country Mills supplies the flour for the sourdough boule",+)+WEEKLY: Fact = (+ "Sunrise Bakery",+ "Hill Country Mills",+ "Sunrise Bakery orders flour weekly from Hill Country Mills",+)++_SECTION = re.compile(r"<EXISTING FACTS>\s*(.*?)\s*</EXISTING FACTS>", re.DOTALL)+++class Bakery(BaseModel):+ """The user's facts and the chat turn that named the flour supplier;+ each dream fact and the dream episode that wrote it."""++ flour: str+ boule: str+ cafe: str+ said: str+ supplies: str+ supplies_episode: str+ boule_flour: str+ boule_flour_episode: str+ weekly: str+ weekly_episode: str++ def derived(self) -> list[str]:+ return sorted([self.supplies, self.boule_flour, self.weekly])+++async def build_bakery(+ driver: AutoGPTFalkorDriver, scope: MemoryScope, build: BuildClient+) -> Bakery:+ """The user's three facts, then three dream passes, each reading the+ graph first: ``supplies`` consolidates FLOUR (citing its fact and chat+ turn); ``boule_flour`` rests on ``supplies`` and BOULE; ``weekly`` rests+ on the dream episode that wrote ``supplies``."""+ said, flour = await ingest_facts(driver, scope, build, [FLOUR], session_id="s-1")+ _, boule = await ingest_facts(driver, scope, build, [BOULE], session_id="s-2")+ _, cafe = await ingest_facts(driver, scope, build, [CAFE], session_id="s-3")+ consolidation = ConsolidatedFact(+ content=SUPPLIES[2],+ confidence=0.9,+ source_fact_uuids=[flour[FLOUR[2]]],+ source_episode_uuids=[said],+ )+ await dream_once(driver, scope, build, consolidation, SUPPLIES, "pass-1")+ supplies = await fact_uuid(driver, SUPPLIES[2])+ supplies_episode = await dream_episode(driver, "pass-1")+ proposal = _proposal(BOULE_FLOUR[2], facts=[supplies, boule[BOULE[2]]])+ await dream_once(driver, scope, build, proposal, BOULE_FLOUR, "pass-2")+ weekly = _proposal(WEEKLY[2], episodes=[supplies_episode])+ await dream_once(driver, scope, build, weekly, WEEKLY, "pass-3")+ return Bakery(+ flour=flour[FLOUR[2]],+ boule=boule[BOULE[2]],+ cafe=cafe[CAFE[2]],+ said=said,+ supplies=supplies,+ supplies_episode=supplies_episode,+ boule_flour=await fact_uuid(driver, BOULE_FLOUR[2]),+ boule_flour_episode=await dream_episode(driver, "pass-2"),+ weekly=await fact_uuid(driver, WEEKLY[2]),+ weekly_episode=await dream_episode(driver, "pass-3"),+ )+++def _proposal(+ content: str, *, facts: list[str] | None = None, episodes: list[str] | None = None+) -> ProposedFinding:+ return ProposedFinding(+ content=content,+ confidence=0.6,+ rationale="the bakery's flour",+ source_fact_uuids=facts or [],+ source_episode_uuids=episodes or [],+ )+++async def dream_once(+ driver: AutoGPTFalkorDriver,+ scope: MemoryScope,+ build: BuildClient,+ write: ConsolidatedFact | ProposedFinding,+ extracted: Fact,+ pass_id: str,+ *,+ merge_into: str | None = None,+) -> dict[str, Any]:+ """One dream pass that reads the graph, then writes ``write``, graphiti+ extracting ``extracted`` from it (and, with ``merge_into``, its model+ naming the fact of that sentence a duplicate); its stats, after+ checking it wrote."""+ read = await gather(scope)+ ops = (+ DreamOperations(writes=[write])+ if isinstance(write, ConsolidatedFact)+ else DreamOperations(proposals=[write])+ )+ stats = await dream(driver, scope, build, ops, read, extracted, pass_id, merge_into)+ written = stats["consolidated_count"] + stats["proposal_count"]+ assert (written, stats["dropped_forgotten"]) == (1, 0), f"{pass_id}: {stats}"+ return stats+++async def dream(+ driver: AutoGPTFalkorDriver,+ scope: MemoryScope,+ build: BuildClient,+ ops: DreamOperations,+ read: DreamInput,+ extracted: Fact,+ pass_id: str,+ merge_into: str | None = None,+) -> dict[str, Any]:+ """``ops`` through ``apply_operations`` and the worker, as a pass that+ read ``read``; its stats."""+ client = build(driver, scripted_responses([extracted]))+ with (+ patch.object(ingest, "get_graphiti_client", AsyncMock(return_value=client)),+ _merging(client, merge_into),+ ):+ return await apply.apply_operations(+ scope,+ pass_id,+ ops,+ known_fact_uuids=read.known_fact_uuids,+ known_episode_uuids=read.known_episode_uuids,+ fact_scopes=fact_scopes(read),+ ingestion_drain_timeout=25,+ )+++def _merging(client: Graphiti, sentence: str | None) -> AbstractContextManager:+ """While it is open, graphiti's model names the existing fact+ ``sentence`` a duplicate of whatever a write states, as it would a+ paraphrase; nothing changes when ``sentence`` is None."""+ if sentence is None:+ return nullcontext()+ model = model_of(client)+ answer = model._generate_response++ async def resolve(messages, response_model=None, *args: Any, **kwargs: Any):+ if response_model is not None and response_model.__name__ == "EdgeDuplicate":+ found = _SECTION.search(messages[-1].content)+ existing = ast.literal_eval(found.group(1)) if found else []+ same = [c["idx"] for c in existing if c["fact"] == sentence]+ return {"duplicate_facts": same[:1], "contradicted_facts": []}+ return await answer(messages, response_model, *args, **kwargs)++ return patch.object(model, "_generate_response", side_effect=resolve)+++async def gather(scope: MemoryScope) -> DreamInput:+ """What a dream pass reads now (no chat sessions)."""+ sessions = SimpleNamespace(get_user_chat_sessions=AsyncMock(return_value=[]))+ with patch.object(fetch, "chat_db", return_value=sessions):+ return await fetch.gather_dream_input(scope)+++async def fact_uuid(driver: AutoGPTFalkorDriver, sentence: str) -> str:+ """The one fact whose sentence (or audit copy) is ``sentence``."""+ [row] = await rows(+ driver,+ "MATCH ()-[e:RELATES_TO]->() "+ "WHERE coalesce(e.fact_redacted, e.fact) = $sentence "+ "RETURN e.uuid AS uuid",+ sentence=sentence,+ )+ return row["uuid"]+++async def dream_episode(driver: AutoGPTFalkorDriver, pass_id: str) -> str:+ [row] = await rows(+ driver,+ "MATCH (ep:Episodic) WHERE ep.name STARTS WITH $prefix "+ "RETURN ep.uuid AS uuid",+ prefix=f"dream_{pass_id}_",+ )+ return row["uuid"]+++async def derivation(driver: AutoGPTFalkorDriver, uuid: str) -> dict[str, Any]:+ """What fact or episode ``uuid`` records it was derived from, its+ sources and, for a fact, the reason it was retired."""+ found = await rows(+ driver,+ """+ OPTIONAL MATCH ()-[e:RELATES_TO {uuid: $uuid}]->()+ OPTIONAL MATCH (ep:Episodic {uuid: $uuid})+ RETURN coalesce(e.derived_from_facts, ep.derived_from_facts) AS facts,+ coalesce(e.derived_from_episodes, ep.derived_from_episodes)+ AS episodes,+ e.episodes AS sources, e.expiration_reason AS reason,+ e.status AS status, ep.redacted_at AS redacted_at+ """,+ uuid=uuid,+ )+ return found[0]diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_integration_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_integration_test.pynew file mode 100644index 0000000000..26dab97661--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_integration_test.py@@ -0,0 +1,249 @@+"""A forget reaches what the dream derived from the fact it forgot, on a live+FalkorDB, the dream's facts written through ``dream/apply.py`` and the+production ingestion worker (``recall_cascade.py``).++The bakery (``recall_cascade_fixtures.py``): the user said Sunrise Bakery+uses Hill Country Mills as its flour supplier, and three dream passes derived+a consolidation from it, a proposal from that and from a fact the user+stated about the same bakery, and a proposal from the consolidation's dream+text. On the test bed a forget left 16 such facts live, and the assistant+went on answering with the forgotten one. A hard forget's erasure is in+``recall_cascade_erase_integration_test.py``.++Run with FalkorDB reachable (see ``conftest.py``)::++ poetry run pytest -m integration backend/copilot/graphiti/recall_cascade_integration_test.py+"""++import asyncio+from collections.abc import AsyncIterator+from datetime import datetime, timezone+from unittest.mock import AsyncMock++import pytest+import pytest_asyncio+from pytest_mock import MockerFixture++from backend.copilot.dream import apply+from backend.copilot.dream.schemas import (+ ConsolidatedFact,+ DreamOperations,+ ProposedFinding,+)+from backend.copilot.model import ChatSession+from backend.copilot.tools import graphiti_search+from backend.copilot.tools.models import MemorySearchResponse++from . import context+from .falkordb_driver import AutoGPTFalkorDriver+from .recall import FORGOTTEN_FACT+from .recall_cascade_fixtures import (+ BOULE,+ BOULE_FLOUR,+ FLOUR,+ SUPPLIES,+ WEEKLY,+ Bakery,+ build_bakery,+ dream,+ dream_episode,+ dream_once,+ fact_uuid,+ gather,+)+from .recall_forget import retract+from .recall_integration_fixtures import (+ Fact,+ capture_spawned_tasks,+ edge_row,+ episode_row,+ live_facts,+ patch_recall_boundaries,+ stop_ingestion_workers,+)+from .recall_stamp import stamp_recalls+from .scope import MemoryScope++_DERIVED = [SUPPLIES[2], BOULE_FLOUR[2], WEEKLY[2]]+_DEPENDS: Fact = (+ "Sourdough Boule",+ "Hill Country Mills",+ "The sourdough boule depends on Hill Country Mills",+)++[email protected](autouse=True)+def boundaries(mocker, scope_graph, stub_graphiti_client):+ patch_recall_boundaries(mocker, scope_graph[0], stub_graphiti_client)+++@pytest_asyncio.fixture(loop_scope="function")+async def dream_apply(mocker: MockerFixture) -> AsyncIterator[None]:+ """apply's Postgres side (the dream session and its summary) stubbed."""+ mocker.patch.object(apply, "_create_dream_session", AsyncMock(return_value="s"))+ mocker.patch.object(apply, "_write_dream_summary_message", AsyncMock())+ mocker.patch(+ "backend.copilot.dream.registry.ensure_dream_system_scheduled",+ AsyncMock(return_value=None),+ )+ yield+ await stop_ingestion_workers()+++def hit_tasks(mocker) -> list[asyncio.Task]:+ return capture_spawned_tasks(mocker)+++async def _shown(scope: MemoryScope, hit_tasks: list[asyncio.Task]) -> str:+ """Everything the assistant is shown for the bakery's flour: warm+ context, and ``memory_search``'s facts and episodes."""+ query = "Sunrise Bakery flour Hill Country Mills"+ warm = await context._fetch(scope, query)+ await asyncio.gather(*context._pending_hit_tasks)+ session = ChatSession.new(scope.owner_user_id, dry_run=False)+ found = await graphiti_search.MemorySearchTool()._execute(+ scope.owner_user_id, session, query=query+ )+ await asyncio.gather(*hit_tasks)+ assert isinstance(found, MemorySearchResponse)+ return "\n".join([warm or "", *found.facts, *found.recent_episodes])+++async def _set_status(+ driver: AutoGPTFalkorDriver, uuid: str, status: str, reason: str | None = None+) -> None:+ """As ratification or a dream demotion leaves a fact (expired with a+ reason, when ``reason`` is given)."""+ await driver.execute_query(+ "MATCH ()-[e:RELATES_TO {uuid: $uuid}]->() "+ "SET e.status = $status, e.expiration_reason = $reason, "+ "e.expired_at = CASE WHEN $reason IS NULL THEN NULL ELSE $now END",+ uuid=uuid,+ status=status,+ reason=reason,+ now=datetime.now(timezone.utc).isoformat(),+ )+++async def _assert_retracted_for(+ driver: AutoGPTFalkorDriver, bakery: Bakery, sentences: dict[str, str]+) -> None:+ """Each derived fact retracted as a soft forget retracts one, naming the+ forgotten flour supplier, its sentence kept only as the audit copy."""+ for uuid, sentence in sentences.items():+ row = await edge_row(driver, uuid)+ assert row["forgotten_at"] is not None, sentence+ assert row["status"] == "retracted"+ assert row["reason"] == f"derived_from_forgotten:{bakery.flour}"+ assert (row["fact"], row["fact_redacted"]) == (FORGOTTEN_FACT, sentence)+++async def test_a_forget_retracts_everything_the_dream_derived_from_the_fact(+ scope_graph, stub_graphiti_client, dream_apply, hit_tasks+) -> None:+ driver, scope = scope_graph+ bakery = await build_bakery(driver, scope, stub_graphiti_client)+ before = await _shown(scope, hit_tasks)+ assert all(sentence in before for sentence in [FLOUR[2], *_DERIVED])++ result = await retract(scope, [bakery.flour])++ assert (result.deleted, result.failures) == ([bakery.flour], [])+ assert sorted(result.derived) == bakery.derived()+ assert set(await live_facts(driver)) == {bakery.boule, bakery.cafe}+ derived = [bakery.supplies, bakery.boule_flour, bakery.weekly]+ await _assert_retracted_for(driver, bakery, dict(zip(derived, _DERIVED)))+ for episode in (+ bakery.said,+ bakery.supplies_episode,+ bakery.boule_flour_episode,+ bakery.weekly_episode,+ ):+ assert (await episode_row(driver, episode))["redacted_at"] is not None+ after = await _shown(scope, hit_tasks)+ assert not [s for s in [FLOUR[2], *_DERIVED] if s in after], "still shown"+ assert BOULE[2] in after, "the bakery fact the user stated stays"+ assert await stamp_recalls(driver, derived, owner="test") == 0, "never stamped"+++async def test_the_next_dream_pass_cannot_teach_them_again(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ """A pass reading after the forget sees none of it; one that read before+ has its writes resting on any of it dropped under the write lock."""+ driver, scope = scope_graph+ bakery = await build_bakery(driver, scope, stub_graphiti_client)+ stale = await gather(scope)++ await retract(scope, [bakery.flour])++ fresh = await gather(scope)+ assert {fact.uuid for fact in fresh.facts} == {bakery.boule, bakery.cafe}+ hidden = {+ bakery.said,+ bakery.supplies_episode,+ bakery.boule_flour_episode,+ bakery.weekly_episode,+ }+ assert not hidden & {episode.uuid for episode in fresh.episodes}+ ops = DreamOperations(+ writes=[+ ConsolidatedFact(+ content=SUPPLIES[2],+ confidence=0.9,+ source_fact_uuids=[bakery.supplies],+ )+ ],+ proposals=[+ ProposedFinding(+ content=WEEKLY[2],+ confidence=0.6,+ rationale="read before the forget",+ source_episode_uuids=[bakery.weekly_episode],+ )+ ],+ )+ live = await live_facts(driver)+ stats = await dream(+ driver, scope, stub_graphiti_client, ops, stale, SUPPLIES, "pass-late"+ )+ assert (stats["dropped_forgotten"], stats["uncited_writes_dropped"]) == (2, 0)+ assert await live_facts(driver) == live+++async def test_the_walk_goes_on_through_a_derived_fact_no_longer_live(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ """The proposal ``boule_flour`` was ratified, a later pass derived a fact+ from it, and then the dream superseded it: the proposal keeps its status,+ but the walk goes on through it, retracting the fact derived from it and+ hiding both dream texts."""+ driver, scope = scope_graph+ bakery = await build_bakery(driver, scope, stub_graphiti_client)+ await _set_status(driver, bakery.boule_flour, "active")+ later = ProposedFinding(+ content=_DEPENDS[2],+ confidence=0.6,+ rationale="the ratified proposal",+ source_fact_uuids=[bakery.boule_flour],+ )+ await dream_once(driver, scope, stub_graphiti_client, later, _DEPENDS, "pass-4")+ depends = await fact_uuid(driver, _DEPENDS[2])+ await _set_status(driver, bakery.boule_flour, "superseded", "stale_fact")++ result = await retract(scope, [bakery.flour])++ assert sorted(result.derived) == sorted([bakery.supplies, bakery.weekly, depends])+ row = await edge_row(driver, bakery.boule_flour)+ assert (row["status"], row["reason"]) == ("superseded", "stale_fact")+ episodes = (bakery.boule_flour_episode, await dream_episode(driver, "pass-4"))+ for episode in episodes:+ assert (await episode_row(driver, episode))["redacted_at"] is not None+ assert set(await live_facts(driver)) == {bakery.boule, bakery.cafe}diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_lock_integration_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_lock_integration_test.pynew file mode 100644index 0000000000..1dcf3706ea--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_lock_integration_test.py@@ -0,0 +1,204 @@+"""A forget and a dream write at the same time, and a forget repeated after+its cascade failed, on a live FalkorDB whose own Redis holds the graph's+write lock (``scope_lock.py``), through ``dream/apply.py`` and the+production ingestion worker.++The worker records what a dream write was derived from before it lets the+lock go (``recall_derivation.py``), so a forget waiting for the write finds+the new fact and retracts it; a write waiting for a forget is checked after+it and dropped (``recall_citations.py``). A forget whose cascade stopped+partway goes on from the facts it already retracted when repeated.++Run with FalkorDB reachable (see ``conftest.py``)::++ poetry run pytest -m integration backend/copilot/graphiti/recall_cascade_lock_integration_test.py+"""++import asyncio+from collections.abc import AsyncIterator, Awaitable, Callable, Iterator+from contextlib import contextmanager+from typing import Any+from unittest.mock import AsyncMock, patch++import pytest+import pytest_asyncio+from graphiti_core.driver.falkordb_driver import FalkorDriverSession+from pytest_mock import MockerFixture++from backend.copilot.dream import apply+from backend.copilot.dream.schemas import ConsolidatedFact, DreamOperations++from . import recall_forget+from .falkordb_driver import AutoGPTFalkorDriver+from .memory_model import ForgetResult, MemoryForgetFailureCode+from .recall import FORGOTTEN_FACT+from .recall_cascade_fixtures import (+ FLOUR,+ SUPPLIES,+ build_bakery,+ dream,+ fact_uuid,+ gather,+)+from .recall_forget import retract+from .recall_hide import SCRUB_FACTS_QUERY+from .recall_integration_fixtures import (+ count,+ edge_row,+ ingest_facts,+ live_facts,+ patch_recall_boundaries,+ stop_ingestion_workers,+)+from .recall_marker_fixtures import saves_facts++[email protected](autouse=True)+def boundaries(mocker, scope_graph, stub_graphiti_client):+ patch_recall_boundaries(mocker, scope_graph[0], stub_graphiti_client)+++@pytest_asyncio.fixture(loop_scope="function")+async def dream_apply(mocker: MockerFixture) -> AsyncIterator[None]:+ mocker.patch.object(apply, "_create_dream_session", AsyncMock(return_value="s"))+ mocker.patch.object(apply, "_write_dream_summary_message", AsyncMock())+ mocker.patch(+ "backend.copilot.dream.registry.ensure_dream_system_scheduled",+ AsyncMock(return_value=None),+ )+ yield+ await stop_ingestion_workers()+++@contextmanager+def _forget_while_saving(+ forget: Callable[[], Awaitable[ForgetResult]],+) -> Iterator[tuple[list[asyncio.Future], list[bool]]]:+ """Start ``forget`` right before graphiti saves the dream write's edges,+ holding the graph's write lock, and note whether it is still waiting+ for the lock 0.6 s later."""+ original = FalkorDriverSession.run+ started: list[asyncio.Future] = []+ waited: list[bool] = []++ async def run(session: FalkorDriverSession, query: Any, **params: Any) -> Any:+ if not started and saves_facts(query):+ started.append(asyncio.ensure_future(forget()))+ await asyncio.sleep(0.6)+ waited.append(not started[0].done())+ return await original(session, query, **params)++ with patch.object(FalkorDriverSession, "run", run):+ yield started, waited+++def _consolidation(flour: str, said: str) -> DreamOperations:+ return DreamOperations(+ writes=[+ ConsolidatedFact(+ content=SUPPLIES[2],+ confidence=0.9,+ source_fact_uuids=[flour],+ source_episode_uuids=[said],+ )+ ]+ )+++async def test_a_forget_waiting_for_a_dream_write_retracts_what_it_wrote(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ driver, scope = scope_graph+ build = stub_graphiti_client+ said, edges = await ingest_facts(driver, scope, build, [FLOUR], session_id="s-1")+ flour = edges[FLOUR[2]]+ read = await gather(scope)++ with _forget_while_saving(lambda: retract(scope, [flour])) as (started, waited):+ stats = await dream(+ driver, scope, build, _consolidation(flour, said), read, SUPPLIES, "p-1"+ )+ result = await asyncio.wait_for(started[0], 30)++ assert waited == [True], "the forget did not wait for the dream write"+ assert (stats["consolidated_count"], stats["dropped_forgotten"]) == (1, 0)+ supplies = await fact_uuid(driver, SUPPLIES[2])+ assert (result.derived, result.failures) == ([supplies], [])+ assert await live_facts(driver) == {}+++async def test_a_dream_write_waiting_for_a_forget_is_dropped(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ driver, scope = scope_graph+ build = stub_graphiti_client+ said, edges = await ingest_facts(driver, scope, build, [FLOUR], session_id="s-1")+ flour = edges[FLOUR[2]]+ read = await gather(scope)+ entered, release = asyncio.Event(), asyncio.Event()+ query = AutoGPTFalkorDriver.execute_query++ async def hold_the_forget(self, cypher_query_, **params):+ if cypher_query_ == recall_forget._EXISTING_EDGES_QUERY:+ entered.set()+ await release.wait()+ return await query(self, cypher_query_, **params)++ with patch.object(AutoGPTFalkorDriver, "execute_query", hold_the_forget):+ forget = asyncio.ensure_future(retract(scope, [flour]))+ await asyncio.wait_for(entered.wait(), 10)+ write = asyncio.ensure_future(+ dream(+ driver, scope, build, _consolidation(flour, said), read, SUPPLIES, "p"+ )+ )+ await asyncio.sleep(0.6)+ assert not write.done(), "the dream write did not wait for the forget"+ release.set()+ result = await asyncio.wait_for(forget, 30)+ stats = await asyncio.wait_for(write, 30)++ assert (result.deleted, result.derived) == ([flour], [])+ assert stats["dropped_forgotten"] == 1+ dreamed = (+ "MATCH (ep:Episodic) WHERE ep.name STARTS WITH 'dream_' RETURN count(ep) AS c"+ )+ assert await count(driver, dreamed) == 0+++async def test_a_forget_whose_cascade_failed_finishes_it_when_repeated(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ """The consolidation is retracted, then scrubbing its sentence fails:+ the forget reports a clean-up error, and forgetting again finishes the+ scrub and retracts what rests on it."""+ driver, scope = scope_graph+ bakery = await build_bakery(driver, scope, stub_graphiti_client)+ query = AutoGPTFalkorDriver.execute_query++ async def fail_its_scrub(self, cypher_query_, **params):+ if cypher_query_ == SCRUB_FACTS_QUERY and bakery.supplies in params["uuids"]:+ raise RuntimeError("scrub lost")+ return await query(self, cypher_query_, **params)++ with patch.object(AutoGPTFalkorDriver, "execute_query", fail_its_scrub):+ first = await retract(scope, [bakery.flour])++ assert first.derived == [bakery.supplies]+ assert [(f.uuid, f.code) for f in first.failures] == [+ (bakery.flour, MemoryForgetFailureCode.CLEANUP_ERROR)+ ]+ assert {bakery.boule_flour, bakery.weekly} <= set(await live_facts(driver))++ second = await retract(scope, [bakery.flour])++ assert second.failures == []+ assert sorted(second.derived) == sorted([bakery.boule_flour, bakery.weekly])+ assert (await edge_row(driver, bakery.supplies))["fact"] == FORGOTTEN_FACT+ assert set(await live_facts(driver)) == {bakery.boule, bakery.cafe}diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_queries.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_queries.pynew file mode 100644index 0000000000..065ee58d05--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_queries.py@@ -0,0 +1,95 @@+"""The Cypher a forget's cascade runs (``recall_cascade.py``).++A dream's record is ``derived_from_facts`` and ``derived_from_episodes`` on+its episode and on the facts only dream episodes state+(``recall_derivation.py``); ``$facts`` and ``$episodes`` are the frontier,+what the cascade last retracted and hid, and ``$seen`` everything it has+reached.+"""++from .recall import live_fact_predicate++_LIVE = live_fact_predicate("e")++# Of ``$uuids``, gone from the graph, those something still names as what+# it rests on or was hidden for: a record (a fact's or an episode's), a+# pending citation marker, an episode's ``redacted_for``, or the reason of a+# fact an earlier cascade retracted for it (``$prefix`` + the uuid). One of+# these was a fact a hard forget purged (``recall_forget.py``).+NAMED_ROOTS_QUERY = """+UNWIND $uuids AS root+OPTIONAL MATCH ()-[e:RELATES_TO]->()+WHERE root IN coalesce(e.derived_from_facts, [])+ OR e.expiration_reason = $prefix + root+WITH root, count(e) AS facts+OPTIONAL MATCH (n)+WHERE (n:Episodic OR n:DreamCitations)+ AND (root IN coalesce(n.derived_from_facts, [])+ OR root IN coalesce(n.redacted_for, []))+WITH root, facts, count(n) AS nodes+WHERE facts + nodes > 0+RETURN root AS uuid+"""++# The facts an earlier try of this forget retracted.+EARLIER_QUERY = """+MATCH ()-[e:RELATES_TO]->()+WHERE e.expiration_reason IN $reasons+RETURN e.uuid AS uuid, e.expiration_reason AS reason+"""++# A fact derived from the frontier, not reached yet, that no user's episode+# states (every episode it names carries a dream's record), and whether it+# is live.+DERIVED_FACTS_QUERY = f"""+MATCH ()-[e:RELATES_TO]->()+WHERE NOT e.uuid IN $seen+ AND (any(x IN coalesce(e.derived_from_facts, []) WHERE x IN $facts)+ OR any(x IN coalesce(e.derived_from_episodes, []) WHERE x IN $episodes))+OPTIONAL MATCH (stated:Episodic)+WHERE stated.uuid IN coalesce(e.episodes, [])+ AND stated.derived_from_facts IS NULL+WITH e, count(stated) AS independent+WHERE independent = 0+RETURN e.uuid AS uuid,+ [x IN coalesce(e.derived_from_facts, []) WHERE x IN $facts]+ + [x IN coalesce(e.derived_from_episodes, []) WHERE x IN $episodes] AS via,+ ({_LIVE}) AS live+ORDER BY uuid+LIMIT $limit+"""++# A dream episode derived from the frontier, hidden or not, not yet reached.+DERIVED_EPISODES_QUERY = """+MATCH (ep:Episodic)+WHERE NOT ep.uuid IN $seen+ AND (any(x IN coalesce(ep.derived_from_facts, []) WHERE x IN $facts)+ OR any(x IN coalesce(ep.derived_from_episodes, []) WHERE x IN $episodes))+RETURN ep.uuid AS uuid,+ [x IN coalesce(ep.derived_from_facts, []) WHERE x IN $facts]+ + [x IN coalesce(ep.derived_from_episodes, []) WHERE x IN $episodes] AS via+ORDER BY uuid+LIMIT $limit+"""++# A soft forget's retraction (``recall_forget._RETRACT_EDGE_QUERY``) of each+# target still live, with its own reason.+RETRACT_QUERY = f"""+UNWIND $targets AS target+MATCH ()-[e:RELATES_TO {{uuid: target.uuid}}]->()+WHERE (e.group_id = $group_id OR e.group_id IS NULL) AND {_LIVE}+SET e.forgotten_at = coalesce(e.forgotten_at, $now),+ e.expired_at = coalesce(e.expired_at, $now),+ e.status = $status,+ e.expiration_reason = target.reason+RETURN e.uuid AS uuid+"""++REDACT_DERIVED_QUERY = """+MATCH (ep:Episodic)+WHERE ep.uuid IN $uuids+SET ep.redacted_at = coalesce(ep.redacted_at, $now)+WITH ep+OPTIONAL MATCH (ep)-[:MENTIONS]->(entity:Entity)+RETURN collect(DISTINCT entity.uuid) AS mentioned+"""diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_resume_integration_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_resume_integration_test.pynew file mode 100644index 0000000000..b298cddce4--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_resume_integration_test.py@@ -0,0 +1,257 @@+"""A hard forget whose cascade stopped short, resumed, on a live FalkorDB+through ``dream/apply.py`` and the production ingestion worker+(``recall_forget.py``, ``migrations/backfill_cascade.py``).++A hard forget purges the fact the user named even when its cascade stops+(a failed step, or a bound), so the fact is gone and repeating the forget+used to find nothing: Codex's finding. Now a repeated forget of the purged+uuid starts a cascade from whatever still names it (a record, an earlier+cascade's reason, the ``redacted_for`` of the episodes it hid), erasing,+and the backfill's ``--cascade-existing-forgets`` finds the purged roots and+the hidden episodes the dream cited. Each stop (before the first query,+between rounds, at the item bound and at the round bound) is resumed both+ways, until everything the dream derived from the bakery's flour supplier+is retracted and erased under that root's name. ``echo`` cites only the+user's chat turn, which the purge emptied: only ``redacted_for`` links it to+the forgotten fact. When that turn also stated another fact, the purge hides+it without emptying it, so no emptied episode carries the erasure: the+backfill must find the root gone from what names it and erase from there.++Run with FalkorDB reachable (see ``conftest.py``)::++ poetry run pytest -m integration backend/copilot/graphiti/recall_cascade_resume_integration_test.py+"""++from collections.abc import AsyncIterator, Iterator+from contextlib import contextmanager+from unittest.mock import AsyncMock, patch++import pytest+import pytest_asyncio+from pytest_mock import MockerFixture++from backend.copilot.dream import apply+from backend.copilot.dream.schemas import ProposedFinding++from . import recall_cascade+from .falkordb_driver import AutoGPTFalkorDriver+from .migrations.backfill_derivations import backfill_graph+from .recall import FORGOTTEN_FACT+from .recall_cascade_fixtures import (+ BOULE_FLOUR,+ FLOUR,+ SUPPLIES,+ WEEKLY,+ Bakery,+ build_bakery,+ dream_episode,+ dream_once,+ fact_uuid,+)+from .recall_cascade_queries import DERIVED_FACTS_QUERY, EARLIER_QUERY+from .recall_forget import retract+from .recall_integration_fixtures import (+ Fact,+ edge_row,+ episode_row,+ live_facts,+ patch_recall_boundaries,+ rows,+ sentence_properties,+ stop_ingestion_workers,+)+from .scope import MemoryScope++_ECHO: Fact = (+ "Sunrise Bakery",+ "Flour Order",+ "Sunrise Bakery keeps a standing flour order",+)+_TRIES = 12++[email protected](autouse=True)+def boundaries(mocker, scope_graph, stub_graphiti_client):+ patch_recall_boundaries(mocker, scope_graph[0], stub_graphiti_client)+++@pytest_asyncio.fixture(loop_scope="function")+async def dream_apply(mocker: MockerFixture) -> AsyncIterator[None]:+ """apply's Postgres side (the dream session and its summary) stubbed."""+ mocker.patch.object(apply, "_create_dream_session", AsyncMock(return_value="s"))+ mocker.patch.object(apply, "_write_dream_summary_message", AsyncMock())+ mocker.patch(+ "backend.copilot.dream.registry.ensure_dream_system_scheduled",+ AsyncMock(return_value=None),+ )+ yield+ await stop_ingestion_workers()+++@contextmanager+def _failing_once(query: str, nth: int) -> Iterator[None]:+ """The ``nth`` run of ``query`` raises; every other run goes through."""+ original = AutoGPTFalkorDriver.execute_query+ seen = [0]++ async def failing(self, cypher_query_, **params):+ if cypher_query_ == query:+ seen[0] += 1+ if seen[0] == nth:+ raise RuntimeError("injected cascade failure")+ return await original(self, cypher_query_, **params)++ with patch.object(AutoGPTFalkorDriver, "execute_query", failing):+ yield+++@contextmanager+def _stop(point: str) -> Iterator[None]:+ """Where the first hard forget's cascade stops."""+ if point == "first_query":+ with _failing_once(EARLIER_QUERY, 1):+ yield+ elif point == "between_rounds":+ with _failing_once(DERIVED_FACTS_QUERY, 2):+ yield+ else:+ bound = "CASCADE_MAX_ITEMS" if point == "item_bound" else "CASCADE_MAX_ROUNDS"+ with patch.object(recall_cascade, bound, 1):+ yield+++@contextmanager+def _bounded(point: str) -> Iterator[None]:+ """A bound stays in place while the forget is resumed; a failure does+ not happen again."""+ if point.endswith("_bound"):+ with _stop(point):+ yield+ else:+ yield+++async def _with_echo(driver, scope: MemoryScope, build) -> tuple[Bakery, str, str]:+ """The bakery, plus a proposal citing only the user's chat turn."""+ bakery = await build_bakery(driver, scope, build)+ echo = ProposedFinding(+ content=_ECHO[2],+ confidence=0.6,+ rationale="the chat turn",+ source_episode_uuids=[bakery.said],+ )+ await dream_once(driver, scope, build, echo, _ECHO, "pass-echo")+ return (+ bakery,+ await fact_uuid(driver, _ECHO[2]),+ await dream_episode(driver, "pass-echo"),+ )+++async def _resume(scope: MemoryScope, driver, root: str, recovery: str) -> list[dict]:+ """Forget again, or run the backfill, until nothing is left to resume;+ what each try reported."""+ tries: list[dict] = []+ for _ in range(_TRIES):+ if recovery == "forget_again":+ result = await retract(scope, [root], hard=True)+ tries.append({"resumed": result.resumed, "failed": bool(result.failures)})+ else:+ found = await backfill_graph(driver, apply=True, cascade_forgets=True)+ tries.append({"roots": found.roots, "failed": bool(found.failed)})+ if not tries[-1]["failed"]:+ return tries+ raise AssertionError(f"not finished after {_TRIES} tries: {tries}")++[email protected]("recovery", ["forget_again", "backfill"])+ "point", ["first_query", "between_rounds", "item_bound", "round_bound"]+)+async def test_a_hard_forget_that_stopped_short_is_resumed_and_erases_everything(+ scope_graph, stub_graphiti_client, dream_apply, point: str, recovery: str+) -> None:+ driver, scope = scope_graph+ bakery, echo, echo_episode = await _with_echo(driver, scope, stub_graphiti_client)+ derived = [*bakery.derived(), echo]++ with _stop(point):+ first = await retract(scope, [bakery.flour], hard=True)++ assert first.failures, "the cascade stopped short"+ assert await edge_row(driver, bakery.flour) == {}, "the root is purged anyway"+ assert set(await live_facts(driver)) - {bakery.boule, bakery.cafe}++ with _bounded(point):+ tries = await _resume(scope, driver, bakery.flour, recovery)++ if recovery == "forget_again":+ assert all(t["resumed"] == [bakery.flour] for t in tries)+ assert set(await live_facts(driver)) == {bakery.boule, bakery.cafe}+ for uuid in derived:+ row = await edge_row(driver, uuid)+ assert row["reason"] == f"derived_from_forgotten:{bakery.flour}", uuid+ assert (row["fact"], row["fact_redacted"]) == (FORGOTTEN_FACT, "")+ embeddings = await rows(+ driver,+ "MATCH ()-[e:RELATES_TO]->() WHERE e.uuid IN $uuids "+ "AND e.fact_embedding IS NOT NULL RETURN e.uuid AS uuid",+ uuids=derived,+ )+ assert embeddings == []+ for sentence in (FLOUR[2], SUPPLIES[2], BOULE_FLOUR[2], WEEKLY[2], _ECHO[2]):+ assert await sentence_properties(driver, sentence) == set(), sentence+ episodes = [+ bakery.supplies_episode,+ bakery.boule_flour_episode,+ bakery.weekly_episode,+ echo_episode,+ ]+ bodies = await rows(+ driver,+ "MATCH (ep:Episodic) WHERE ep.uuid IN $uuids RETURN ep.content AS content",+ uuids=episodes,+ )+ assert [row["content"] for row in bodies] == [""] * len(episodes)+++async def test_the_backfill_erases_from_a_purged_root_whose_chat_turn_stays(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ """The flour supplier's chat turn also named the cafe, so the hard forget+ hides the turn but keeps its text for the cafe and empties no episode:+ no hard seed carries the erasure. The backfill finds the purged root+ from the records and the ``redacted_for`` that name it, and erases+ every derived fact under its name."""+ driver, scope = scope_graph+ bakery, echo, _ = await _with_echo(driver, scope, stub_graphiti_client)+ await driver.execute_query(+ "MATCH (ep:Episodic {uuid: $said}), ()-[e:RELATES_TO {uuid: $cafe}]->() "+ "SET ep.entity_edges = ep.entity_edges + [$cafe], "+ "e.episodes = e.episodes + [$said]",+ said=bakery.said,+ cafe=bakery.cafe,+ )+ derived = [*bakery.derived(), echo]++ with _stop("first_query"):+ first = await retract(scope, [bakery.flour], hard=True)++ assert first.failures and await edge_row(driver, bakery.flour) == {}+ turn = await episode_row(driver, bakery.said)+ assert turn["redacted_at"] is not None and turn["hard_deleted_at"] is None++ tries = await _resume(scope, driver, bakery.flour, "backfill")++ assert tries[0]["roots"] >= 1+ assert set(await live_facts(driver)) == {bakery.boule, bakery.cafe}+ for uuid in derived:+ row = await edge_row(driver, uuid)+ assert row["reason"] == f"derived_from_forgotten:{bakery.flour}", uuid+ assert (row["fact"], row["fact_redacted"]) == (FORGOTTEN_FACT, ""), uuid+ for sentence in (SUPPLIES[2], BOULE_FLOUR[2], WEEKLY[2], _ECHO[2]):+ assert await sentence_properties(driver, sentence) == set(), sentencediff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_sources_integration_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_sources_integration_test.pynew file mode 100644index 0000000000..dfaaa04a4d--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_sources_integration_test.py@@ -0,0 +1,201 @@+"""Which facts carry a dream's record, when graphiti merges a statement into+an existing fact, and so which a forget's cascade may retract, on a live+FalkorDB through ``dream/apply.py`` and the production ingestion worker+(``recall_derivation.py``, ``recall_cascade.py``).++A fact a dream write merges into that a user's own episode states is never+stamped, and a derived fact the user later states word for word keeps its+record but stays: both have a source the forget did not reach. A dream write+graphiti's model merges into a derived fact, which rewrites the fact's+attributes, leaves it the union of both writes' records.++Run with FalkorDB reachable (see ``conftest.py``)::++ poetry run pytest -m integration backend/copilot/graphiti/recall_cascade_sources_integration_test.py+"""++from collections.abc import AsyncIterator+from unittest.mock import AsyncMock++import pytest+import pytest_asyncio+from pytest_mock import MockerFixture++from backend.copilot.dream import apply+from backend.copilot.dream.schemas import ConsolidatedFact, ProposedFinding++from .falkordb_driver import AutoGPTFalkorDriver+from .recall_cascade_fixtures import (+ BOULE,+ BOULE_FLOUR,+ FLOUR,+ SUPPLIES,+ derivation,+ dream_episode,+ dream_once,+ fact_uuid,+)+from .recall_forget import retract+from .recall_integration_fixtures import (+ BuildClient,+ Fact,+ episode_row,+ ingest_facts,+ live_facts,+ patch_recall_boundaries,+ stop_ingestion_workers,+)+from .scope import MemoryScope++_PARAPHRASE: Fact = (+ "Hill Country Mills",+ "Sunrise Bakery",+ "Sunrise Bakery's flour comes from Hill Country Mills",+)++[email protected](autouse=True)+def boundaries(mocker, scope_graph, stub_graphiti_client):+ patch_recall_boundaries(mocker, scope_graph[0], stub_graphiti_client)+++@pytest_asyncio.fixture(loop_scope="function")+async def dream_apply(mocker: MockerFixture) -> AsyncIterator[None]:+ mocker.patch.object(apply, "_create_dream_session", AsyncMock(return_value="s"))+ mocker.patch.object(apply, "_write_dream_summary_message", AsyncMock())+ mocker.patch(+ "backend.copilot.dream.registry.ensure_dream_system_scheduled",+ AsyncMock(return_value=None),+ )+ yield+ await stop_ingestion_workers()+++async def _said(+ driver: AutoGPTFalkorDriver, scope: MemoryScope, build: BuildClient, fact: Fact+) -> tuple[str, str]:+ """``fact`` in a chat turn of its own: the turn's episode and the fact."""+ episode, edges = await ingest_facts(+ driver, scope, build, [fact], session_id=f"s-{fact[2][:12]}"+ )+ return episode, edges[fact[2]]+++async def _supplies(+ driver: AutoGPTFalkorDriver, scope: MemoryScope, build: BuildClient+) -> tuple[str, str, str]:+ """The flour supplier said, then consolidated by a dream pass citing+ it: the fact, the consolidation and its dream episode."""+ said, flour = await _said(driver, scope, build, FLOUR)+ consolidation = ConsolidatedFact(+ content=SUPPLIES[2],+ confidence=0.9,+ source_fact_uuids=[flour],+ source_episode_uuids=[said],+ )+ await dream_once(driver, scope, build, consolidation, SUPPLIES, "pass-1")+ return (+ flour,+ await fact_uuid(driver, SUPPLIES[2]),+ await dream_episode(driver, "pass-1"),+ )+++async def test_a_dream_write_merged_into_a_users_fact_leaves_it_alone(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ """The dream restates the user's own fact word for word, citing the+ flour supplier: graphiti adds its episode to the user's fact, which is+ not stamped and outlives the forget; the dream's text is hidden."""+ driver, scope = scope_graph+ build = stub_graphiti_client+ _, flour = await _said(driver, scope, build, FLOUR)+ stated, boule = await _said(driver, scope, build, BOULE)+ restating = ConsolidatedFact(+ content=BOULE[2], confidence=0.9, source_fact_uuids=[flour, boule]+ )+ await dream_once(driver, scope, build, restating, BOULE, "pass-1")+ restated = await dream_episode(driver, "pass-1")+ before = await derivation(driver, boule)+ assert before["sources"] == [stated, restated], "merged into the user's fact"+ assert before["facts"] is None, "never stamped"++ result = await retract(scope, [flour])++ assert (result.derived, result.failures) == ([], [])+ assert boule in await live_facts(driver)+ assert (await derivation(driver, restated))["redacted_at"] is not None+++async def test_a_derived_fact_the_user_states_again_stays_and_so_does_its_kin(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ """The user later says the consolidation word for word: graphiti adds+ their turn to it and keeps its record, but the forget passes it over,+ and the proposal resting on it, since the user stated it themselves."""+ driver, scope = scope_graph+ build = stub_graphiti_client+ flour, supplies, supplies_episode = await _supplies(driver, scope, build)+ _, boule = await _said(driver, scope, build, BOULE)+ proposal = ProposedFinding(+ content=BOULE_FLOUR[2],+ confidence=0.6,+ rationale="the bakery's flour",+ source_fact_uuids=[supplies, boule],+ )+ await dream_once(driver, scope, build, proposal, BOULE_FLOUR, "pass-2")+ again, _ = await _said(driver, scope, build, SUPPLIES)+ row = await derivation(driver, supplies)+ assert row["sources"] == [supplies_episode, again]+ assert row["facts"] == [flour], "graphiti's exact-text merge keeps the record"++ result = await retract(scope, [flour])++ assert (result.derived, result.failures) == ([], [])+ live = await live_facts(driver)+ assert {supplies, boule, await fact_uuid(driver, BOULE_FLOUR[2])} <= set(live)+ assert (await episode_row(driver, supplies_episode))["redacted_at"] is not None+ assert (await episode_row(driver, again))["redacted_at"] is None+++async def test_a_dream_write_merged_into_a_derived_fact_keeps_both_records(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ """A later pass paraphrases the consolidation citing only the user's+ bakery fact, and graphiti's model merges it in, rewriting the fact's+ attributes: the fact's record is rebuilt as the union of both dream+ episodes', so forgetting the flour supplier still retracts it."""+ driver, scope = scope_graph+ build = stub_graphiti_client+ flour, supplies, supplies_episode = await _supplies(driver, scope, build)+ _, boule = await _said(driver, scope, build, BOULE)+ said = (await derivation(driver, supplies_episode))["episodes"]+ paraphrase = ConsolidatedFact(+ content=_PARAPHRASE[2], confidence=0.9, source_fact_uuids=[boule]+ )+ await dream_once(+ driver,+ scope,+ build,+ paraphrase,+ _PARAPHRASE,+ "pass-2",+ merge_into=SUPPLIES[2],+ )+ merged = await derivation(driver, supplies)+ assert merged["sources"] == [+ supplies_episode,+ await dream_episode(driver, "pass-2"),+ ]+ assert (merged["facts"], merged["episodes"]) == ([flour, boule], said)++ result = await retract(scope, [flour])++ assert result.derived == [supplies]+ assert boule in await live_facts(driver)diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_test.pynew file mode 100644index 0000000000..e2881544ab--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_test.py@@ -0,0 +1,247 @@+"""Unit tests for ``recall_cascade.cascade`` over a small in-memory graph+that answers the cascade's queries as FalkorDB would+(``recall_cascade_fake.py``).++Pin the transitive walk, what a derived fact is retracted with, the root its+reason names, resuming an earlier try, the bounds and failure reporting+(what a hard forget erases: ``recall_erase_test.py``). The Cypher itself runs+on FalkorDB in ``recall_cascade_integration_test.py``.+"""++from unittest.mock import AsyncMock, patch++import pytest++from . import recall_cascade, recall_hide+from .memory_model import ForgetResult, MemoryForgetFailureCode+from .recall import live_fact_predicate+from .recall_cascade_fake import CascadeGraph, FakeEpisode, FakeFact, chain++_NOW = "2026-09-28T00:00:00+00:00"+_CLEANUP = MemoryForgetFailureCode.CLEANUP_ERROR+++async def _cascade(graph: CascadeGraph, roots: list[str]) -> ForgetResult:+ result = ForgetResult(redacted_episodes=["e0"])+ await recall_cascade.cascade(graph, "user_a", roots, _NOW, result)+ return result+++def _reasons(graph: CascadeGraph, *uuids: str) -> dict[str, str | None]:+ return {uuid: graph.facts[uuid].reason for uuid in uuids}+++class TestTheWalk:+ @pytest.mark.asyncio+ async def test_retracts_what_was_derived_level_by_level_and_nothing_else(+ self,+ ) -> None:+ graph = chain()++ result = await _cascade(graph, ["f"])++ assert (result.derived, result.failures) == (["c", "p", "q"], [])+ assert _reasons(graph, "c", "p", "q") == dict.fromkeys(+ ["c", "p", "q"], "derived_from_forgotten:f"+ )+ assert graph.facts["u"].live, "a user fact sharing the episode stays"+ assert result.redacted_episodes == ["e0", "dc", "dp", "dq"]+ assert graph.scrubbed == ["c", "p", "q"]++ @pytest.mark.asyncio+ async def test_one_forgotten_source_is_enough(self) -> None:+ graph = chain()+ graph.facts["g"] = FakeFact(sources=["e1"])+ graph.episodes["e1"] = FakeEpisode(cites=["g"])+ graph.facts["c"].facts.append("g")++ result = await _cascade(graph, ["f"])++ assert "c" in result.derived+ assert graph.facts["g"].live++ @pytest.mark.asyncio+ async def test_each_round_writes_its_markers_before_any_clean_up(self) -> None:+ graph = chain()++ await _cascade(graph, ["f"])++ first_round = graph.queries[4:10]+ assert first_round == [+ recall_cascade.RETRACT_QUERY,+ recall_cascade.REDACT_DERIVED_QUERY,+ recall_hide.SCRUB_FACTS_QUERY,+ recall_hide._ENTITY_KEYS_QUERY,+ recall_hide.REDACT_EPISODES_QUERY,+ recall_cascade.DERIVED_FACTS_QUERY,+ ]++ @pytest.mark.asyncio+ async def test_a_derived_fact_a_user_episode_also_states_is_passed_over(+ self,+ ) -> None:+ """``c`` was also said by the user (``e5`` merged into it): it stays,+ and so does what rests on it; its own dream text is still hidden."""+ graph = chain()+ graph.facts["c"].sources.append("e5")+ graph.episodes["e5"] = FakeEpisode(cites=["c"])++ result = await _cascade(graph, ["f"])++ assert result.derived == []+ assert all(graph.facts[uuid].live for uuid in ("c", "p", "q"))+ assert graph.episodes["dc"].redacted+ assert not graph.episodes["e5"].redacted++ @pytest.mark.asyncio+ async def test_the_walk_goes_on_through_a_derived_fact_no_longer_live(+ self,+ ) -> None:+ """``c`` was superseded: it is left as it is, but ``p`` and ``q`` rest+ on it and are retracted, and its dream text is hidden."""+ graph = chain()+ graph.facts["c"].live = False++ result = await _cascade(graph, ["f"])++ assert result.derived == ["p", "q"]+ assert graph.facts["c"].reason is None+ assert graph.episodes["dc"].redacted++ @pytest.mark.asyncio+ async def test_each_retraction_names_the_forgotten_fact_it_descends_from(+ self,+ ) -> None:+ graph = chain()+ graph.facts["g"] = FakeFact(live=False, reason="user_signal", sources=["e1"])+ graph.episodes["e1"] = FakeEpisode(cites=["g"])+ graph.facts["d"] = FakeFact(episodes=["e1"], sources=["dd"])+ graph.episodes["dd"] = FakeEpisode(cites=["d"], facts=[], episodes=["e1"])++ await _cascade(graph, ["f", "g"])++ assert _reasons(graph, "c", "d", "p") == {+ "c": "derived_from_forgotten:f",+ "d": "derived_from_forgotten:g",+ "p": "derived_from_forgotten:f",+ }+++class TestResuming:+ @pytest.mark.asyncio+ async def test_a_second_forget_goes_on_from_what_the_first_retracted(+ self,+ ) -> None:+ graph = chain()+ graph.facts["c"].live = False+ graph.facts["c"].reason = "derived_from_forgotten:f"++ result = await _cascade(graph, ["f"])++ assert result.derived == ["p", "q"], "c is not counted twice"+ assert graph.scrubbed[0] == "c", "its clean-up is finished again"+ assert graph.queries[0] == recall_cascade.EARLIER_QUERY+++class TestBounds:+ @pytest.mark.asyncio+ async def test_a_chain_deeper_than_the_rounds_is_reported_and_resumed(+ self,+ ) -> None:+ graph = chain()++ with patch.object(recall_cascade, "CASCADE_MAX_ROUNDS", 2):+ first = await _cascade(graph, ["f"])+ second = await _cascade(graph, ["f"])++ assert first.derived == ["c", "p"]+ assert [(f.uuid, f.code) for f in first.failures] == [("f", _CLEANUP)]+ assert "Forget it again" in first.failures[0].reason+ assert (second.derived, second.failures) == (["q"], [])++ @pytest.mark.asyncio+ async def test_more_derived_items_than_the_budget_are_reported(self) -> None:+ graph = chain()++ with patch.object(recall_cascade, "CASCADE_MAX_ITEMS", 1):+ result = await _cascade(graph, ["f"])++ assert result.derived == ["c"], "the first item, then it stops"+ assert graph.facts["p"].live and graph.facts["q"].live+ assert [f.code for f in result.failures] == [_CLEANUP]++ @pytest.mark.asyncio+ async def test_a_budget_used_up_exactly_still_reports_what_is_left(+ self,+ ) -> None:+ """Each round finds a fact and its dream episode, exactly the budget:+ the next round, cut to nothing, is still reported, and each forget+ again goes one level further."""+ graph = chain()++ with patch.object(recall_cascade, "CASCADE_MAX_ITEMS", 2):+ tries = [await _cascade(graph, ["f"]) for _ in range(3)]++ assert [t.derived for t in tries] == [["c"], ["p"], ["q"]]+ assert [[f.code for f in t.failures] for t in tries] == [+ [_CLEANUP],+ [_CLEANUP],+ [],+ ]+++class TestFailures:+ @pytest.mark.asyncio+ async def test_a_failed_step_is_a_cleanup_error_on_every_root(self) -> None:+ graph = chain()+ graph.fail_on = recall_hide.SCRUB_FACTS_QUERY+ result = ForgetResult()++ await recall_cascade.cascade(graph, "user_a", ["f", "g"], _NOW, result)++ assert [(f.uuid, f.code) for f in result.failures] == [+ ("f", _CLEANUP),+ ("g", _CLEANUP),+ ]+ assert "RuntimeError: down" in result.failures[0].reason+ assert graph.facts["c"].reason == "derived_from_forgotten:f", "marked first"++ @pytest.mark.asyncio+ async def test_a_fact_that_stopped_being_live_is_passed_through_uncounted(+ self,+ ) -> None:+ """The dream demoted ``c`` between the cascade's read and its write:+ the retraction matches nothing, and the walk goes on through it."""+ graph = chain()+ graph.demoted_meanwhile = {"c"}++ result = await _cascade(graph, ["f"])++ assert result.derived == ["p", "q"]+ assert graph.facts["c"].reason is None++ @pytest.mark.asyncio+ async def test_no_roots_reads_nothing(self) -> None:+ driver = AsyncMock()++ await recall_cascade.cascade(driver, "user_a", [], _NOW, ForgetResult())++ driver.execute_query.assert_not_awaited()+++class TestQueries:+ def test_a_fact_a_user_episode_states_is_never_matched(self) -> None:+ query = recall_cascade.DERIVED_FACTS_QUERY+ assert "stated.derived_from_facts IS NULL" in query+ assert "WHERE independent = 0" in query++ def test_the_retraction_is_a_soft_forgets_on_a_fact_still_live(self) -> None:+ query = recall_cascade.RETRACT_QUERY+ assert live_fact_predicate("e") in query+ assert "e.forgotten_at = coalesce(e.forgotten_at, $now)" in query+ assert "e.expired_at = coalesce(e.expired_at, $now)" in query+ assert "e.expiration_reason = target.reason" in query+ assert "invalid_at" not in query, "a forget is not a world change"++ def test_the_reason_names_the_root(self) -> None:+ assert recall_cascade.derived_reason("f1") == "derived_from_forgotten:f1"diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_walk.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_walk.pynew file mode 100644index 0000000000..0e0c552c98--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_cascade_walk.py@@ -0,0 +1,86 @@+"""What a forget's cascade carries from round to round (``recall_cascade.py``).++A ``Walk`` maps every fact and episode the cascade has reached to the root+it descends from, the uuid its retractions name in their reason+(``derived_from_forgotten:<root>``): the fact the user forgot, or, for a+cascade the derivation backfill starts from a hidden episode whose forgotten+fact is gone, the root that episode was hidden for (``redacted_for``) or the+episode itself.+"""++from typing import Any++from pydantic import BaseModel, Field++# ``expiration_reason`` of a fact the cascade retracted, before the colon.+DERIVED_FROM_FORGOTTEN = "derived_from_forgotten"+++def derived_reason(root: str) -> str:+ """The reason recorded on a fact derived from forgotten fact ``root``."""+ return f"{DERIVED_FROM_FORGOTTEN}:{root}"+++class Walk(BaseModel):+ """The fact ``roots`` a cascade starts from and the hidden ``seeds``+ (episode to the root it names), the ``names`` of all their roots, every+ fact and episode reached so far mapped to its root, how many more+ derived items it may retire, and whether it erases their text."""++ roots: list[str]+ seeds: dict[str, str] = Field(default_factory=dict)+ names: list[str]+ root_of: dict[str, str]+ budget: int+ erase: bool = False++ @classmethod+ def start(+ cls,+ roots: list[str],+ seeds: dict[str, str],+ *,+ named: dict[str, str] | None = None,+ budget: int,+ erase: bool,+ ) -> "Walk":+ """A walk from ``roots``, each naming itself unless ``named`` maps it+ to another root, and from ``seeds``."""+ root_of = {root: (named or {}).get(root, root) for root in roots} | seeds+ names = list(dict.fromkeys([*(root_of[r] for r in roots), *seeds.values()]))+ return cls(+ roots=roots,+ seeds=seeds,+ names=names,+ root_of=root_of,+ budget=budget,+ erase=erase,+ )++ def root(self, via: list[str]) -> str:+ """The root of the first item in ``via`` the walk has reached."""+ return next((self.root_of[x] for x in via if x in self.root_of), self.names[0])++ def reach(self, rows: list[dict[str, Any]]) -> list[str]:+ """Record each row's ``uuid`` as reached ``via`` its items; the uuids+ not reached before."""+ new = [row for row in rows if row["uuid"] not in self.root_of]+ for row in new:+ self.root_of[row["uuid"]] = self.root(row["via"])+ return [row["uuid"] for row in new]+++class Frontier(BaseModel):+ """What the last round retracted and hid: the next round's search."""++ facts: list[str] = Field(default_factory=list)+ episodes: list[str] = Field(default_factory=list)+++class Found(BaseModel):+ """One round's finds, rows of ``uuid`` and ``via`` (and, for a fact,+ ``live``); ``truncated`` when the walk's budget cut them short."""++ facts: list[dict[str, Any]] = Field(default_factory=list)+ episodes: list[dict[str, Any]] = Field(default_factory=list)+ truncated: bool = Falsediff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_citation_scope_integration_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_citation_scope_integration_test.pynew file mode 100644index 0000000000..60a06b99d2--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_citation_scope_integration_test.py@@ -0,0 +1,191 @@+"""The scope rule on a live FalkorDB, through ``dream/apply.py`` and the+production ingestion worker (``dream/citations.py``).++Codex's probe: a scripted model writes a ``project:unrelated`` conclusion+about the bakery citing an unrelated ``real:global`` fact the pass read (the+cafe). Before the check it was written: forgetting the bakery's real source+missed it, and forgetting the cafe retracted it. Now it is dropped unwritten,+and so is a write citing a fact of its own scope beside one of another:+trimmed to its own scope it could restate the other fact and escape that+fact's forget. Episode citations are not scoped: a project write citing only+the user's chat turn is written, and forgetting the fact that turn stated+hides the turn and retracts the write through ``derived_from_episodes``.++Run with FalkorDB reachable (see ``conftest.py``)::++ poetry run pytest -m integration backend/copilot/graphiti/recall_citation_scope_integration_test.py+"""++from collections.abc import AsyncIterator+from unittest.mock import AsyncMock++import pytest+import pytest_asyncio+from pytest_mock import MockerFixture++from backend.copilot.dream import apply+from backend.copilot.dream.schemas import ConsolidatedFact, DreamOperations++from .recall_cascade_fixtures import CAFE, FLOUR, derivation, dream, fact_uuid, gather+from .recall_forget import retract+from .recall_integration_fixtures import (+ Fact,+ ingest_facts,+ live_facts,+ patch_recall_boundaries,+ rows,+ stop_ingestion_workers,+)++_QUARTERLY: Fact = (+ "Sunrise Bakery",+ "Quarterly Plan",+ "Sunrise Bakery changes suppliers every quarter",+)+_BREAD: Fact = (+ "Bread Program",+ "Hill Country Mills",+ "The bread program runs on Hill Country Mills flour",+)+_COUNTS = (+ "consolidated_count",+ "uncited_writes_dropped",+ "cross_scope_citations_dropped",+)++[email protected](autouse=True)+def boundaries(mocker, scope_graph, stub_graphiti_client):+ patch_recall_boundaries(mocker, scope_graph[0], stub_graphiti_client)+++@pytest_asyncio.fixture(loop_scope="function")+async def dream_apply(mocker: MockerFixture) -> AsyncIterator[None]:+ """apply's Postgres side (the dream session and its summary) stubbed."""+ mocker.patch.object(apply, "_create_dream_session", AsyncMock(return_value="s"))+ mocker.patch.object(apply, "_write_dream_summary_message", AsyncMock())+ mocker.patch(+ "backend.copilot.dream.registry.ensure_dream_system_scheduled",+ AsyncMock(return_value=None),+ )+ yield+ await stop_ingestion_workers()+++async def test_a_write_citing_only_another_scope_is_never_written(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ driver, scope = scope_graph+ _, facts = await ingest_facts(+ driver, scope, stub_graphiti_client, [FLOUR, CAFE], session_id="s-scope"+ )+ cafe = facts[CAFE[2]]+ wrong = ConsolidatedFact(+ content=_QUARTERLY[2],+ scope="project:unrelated",+ confidence=0.9,+ source_fact_uuids=[cafe],+ )++ stats = await dream(+ driver,+ scope,+ stub_graphiti_client,+ DreamOperations(writes=[wrong]),+ await gather(scope),+ _QUARTERLY,+ "pass-wrong-scope",+ )++ assert [stats[key] for key in _COUNTS] == [0, 1, 1]+ assert await _written(driver, _QUARTERLY[2]) == []+ assert (await retract(scope, [cafe])).derived == [], "nothing rests on it"+++async def test_a_write_citing_a_fact_of_another_scope_beside_its_own_is_dropped(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ """The flour fact is the bread project's; the cafe fact is global. A+ bread-project write citing both is dropped whole, and neither forget has+ anything to retract."""+ driver, scope = scope_graph+ _, facts = await ingest_facts(+ driver, scope, stub_graphiti_client, [FLOUR, CAFE], session_id="s-bread"+ )+ flour, cafe = facts[FLOUR[2]], facts[CAFE[2]]+ await driver.execute_query(+ "MATCH ()-[e:RELATES_TO {uuid: $uuid}]->() SET e.scope = 'project:bread'",+ uuid=flour,+ )+ write = ConsolidatedFact(+ content=_BREAD[2],+ scope="project:bread",+ confidence=0.9,+ source_fact_uuids=[flour, cafe],+ )++ stats = await dream(+ driver,+ scope,+ stub_graphiti_client,+ DreamOperations(writes=[write]),+ await gather(scope),+ _BREAD,+ "pass-bread",+ )++ assert [stats[key] for key in _COUNTS] == [0, 1, 1]+ assert await _written(driver, _BREAD[2]) == []+ assert (await retract(scope, [cafe])).derived == []+ assert (await retract(scope, [flour])).derived == []+++async def test_a_project_write_citing_only_a_chat_turn_is_written_and_reached(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ """The user's chat turn named the flour supplier; a bread-project write+ cites only that turn. It is written with the turn as its source, and+ forgetting the flour fact hides the turn and retracts the write."""+ driver, scope = scope_graph+ said, facts = await ingest_facts(+ driver, scope, stub_graphiti_client, [FLOUR], session_id="s-turn"+ )+ write = ConsolidatedFact(+ content=_BREAD[2],+ scope="project:bread",+ confidence=0.9,+ source_episode_uuids=[said],+ )++ stats = await dream(+ driver,+ scope,+ stub_graphiti_client,+ DreamOperations(writes=[write]),+ await gather(scope),+ _BREAD,+ "pass-turn",+ )++ assert [stats[key] for key in _COUNTS] == [1, 0, 0]+ bread = await fact_uuid(driver, _BREAD[2])+ record = await derivation(driver, bread)+ assert (record["facts"], record["episodes"]) == ([], [said])+ assert (await retract(scope, [facts[FLOUR[2]]])).derived == [bread]+ assert bread not in await live_facts(driver)+++async def _written(driver, sentence: str) -> list[str]:+ """The facts whose sentence is ``sentence``."""+ found = await rows(+ driver,+ "MATCH ()-[e:RELATES_TO {fact: $sentence}]->() RETURN e.uuid AS uuid",+ sentence=sentence,+ )+ return [row["uuid"] for row in found]diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_citations.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_citations.pyindex 86bbb41225..0986b8adff 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/recall_citations.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_citations.py@@ -14,13 +14,27 @@ the check drops the write; one that comes later waits for the lock and thenreaches the facts it names, as after any write.A write rests on a forget when a fact it cites is forgotten or gone (a hard-forget deletes it), or an episode it cites is no longer recallable or gone.-A write that cites nothing is taken to cite everything its pass read-(``apply._citations``), and its statement is also compared with the-sentence every forgotten fact keeps for audit, lower-cased and with its-whitespace collapsed, as graphiti's own exact-match dedup compares. It has-no endpoints until graphiti extracts it, so that comparison spans the-graph: it drops more than a same-endpoints one would, never less.+forget deletes it, a forget's cascade retracts it), or an episode it cites is+no longer recallable or gone; and when a derived fact it cites that is no+longer live (superseded, contradicted) rests on one of those, as far up the+records as derived facts no longer live go (``recall_sources.ancestry``,+bounded like the cascade; a walk stopped by its bound drops the write too).+A forget's cascade walks through such a fact to what rests on it, and the+write would rest on it the same way, carrying that content.++The ``statement`` comparison is kept as defence in depth. apply no longer+sends a write that cites nothing: it drops one before it is queued+(``dream/citations.py``), so nothing in the codebase sets ``statement``+today. Should a write carrying only a ``statement`` reach the worker, it is+compared with the sentence every forgotten fact keeps for audit, lower-cased+and with its whitespace collapsed, as graphiti's own exact-match dedup+compares. It has no endpoints until graphiti extracts it, so that comparison+spans the graph: it drops more than a same-endpoints one would, never less.+A sentence a hard forget erased (``recall_erase.py``) is no longer there to+compare with.++Once written, what a dream write cites is recorded where a later forget+finds it (``recall_derivation.py``)."""from graphiti_core.driver.driver import GraphDriver@@ -31,6 +45,7 @@ from .recall import (forgotten_facts_clause,recallable_episode_predicate,)+from .recall_sources import ancestry, fact_statesclass Citations(BaseModel):@@ -39,7 +54,8 @@ class Citations(BaseModel):fact_uuids: list[str] = Field(default_factory=list)episode_uuids: list[str] = Field(default_factory=list)# Set on a write that cites nothing itself: compared with the sentences- # forgotten facts keep for audit.+ # forgotten facts keep for audit. apply sends none; kept as defence in+ # depth (see the module docstring).statement: str | None = None@@ -51,6 +67,8 @@ async def rests_on_a_forget(if citations is None:return Nonereason = await _cites_a_forget(driver, citations)+ if reason is None and citations.fact_uuids:+ reason = await _rests_through_derived(driver, citations.fact_uuids)if reason is None and citations.statement is not None:reason = await _restates_a_forget(driver, citations.statement)return reason@@ -74,6 +92,17 @@ async def _cites_a_forget(driver: GraphDriver, citations: Citations) -> str | Noreturn None+async def _rests_through_derived(driver: GraphDriver, facts: list[str]) -> str | None:+ """Why a write citing ``facts`` rests on a forget through the derived+ facts no longer live among them, else None."""+ walk = await ancestry(driver, list((await fact_states(driver, facts)).values()))+ if walk.reached:+ return "cites a derived fact no longer live that rests on a forget"+ if walk.unfinished:+ return "cites derived facts no longer live deeper than the check follows"+ return None++async def _restates_a_forget(driver: GraphDriver, statement: str) -> str | None:result = await driver.execute_query(_RESTATED_QUERY, statement=statement, space=r"\s+"diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_citations_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_citations_test.pyindex fea581d720..1d53eb166c 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/recall_citations_test.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_citations_test.py@@ -1,5 +1,6 @@"""Unit tests for ``recall_citations``: a dream write is checked against-forgets, by what it cites or, citing nothing, by its statement, and the+forgets, by what it cites (up the derived facts no longer live among them,+``recall_sources_test.py``) or, citing nothing, by its statement, and theingestion worker drops one that rests on a forget.The live runs, through the real dream write helpers and the production@@ -13,9 +14,10 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom graphiti_core.nodes import EpisodeType-from . import ingest, recall+from . import ingest, recall, recall_sourcesfrom .recall_citations import Citations, rests_on_a_forgetfrom .recall_fake_redis import FakeRedis+from .recall_sources_fake import factfrom .scope import write_lock_key_NOW = datetime(2026, 9, 26, 12, 0, tzinfo=timezone.utc)@@ -43,16 +45,49 @@ class TestRestsOnAForget:@pytest.mark.asyncioasync def test_everything_it_cites_still_there_lets_it_through(self) -> None:- driver = _driver(_still(["f1", "f2"], ["ep1"]))+ """Then it reads the cited facts' states: none is a derived fact no+ longer live, so there is nothing to walk up."""+ driver = _driver(_still(["f1", "f2"], ["ep1"]), [])cited = Citations(fact_uuids=["f1", "f2"], episode_uuids=["ep1"])assert await rests_on_a_forget(driver, cited) is None- [call] = driver.execute_query.await_args_list+ call, states = driver.execute_query.await_args_listquery = call.args[0]assert query.startswith(recall.forgotten_facts_clause())assert "NOT (fact.uuid IN forgotten)" in queryassert recall.recallable_episode_predicate("episode") in queryassert call.kwargs == {"fact_uuids": ["f1", "f2"], "episode_uuids": ["ep1"]}+ assert (states.args[0], states.kwargs) == (+ recall_sources.FACT_STATES_QUERY,+ {"uuids": ["f1", "f2"]},+ )++ @pytest.mark.asyncio+ async def test_a_derived_fact_no_longer_live_resting_on_a_forget_drops_it(+ self,+ ) -> None:+ """``d`` was superseded after the pass read it and ``f``, which it+ was derived from, forgotten: the write would carry ``f``'s content."""+ driver = _driver(+ _still(["d"], []),+ [{"uuid": "d", **fact(facts=("f",))}],+ [{"uuid": "f", **fact(forgotten=True)}],+ )++ found = await rests_on_a_forget(driver, Citations(fact_uuids=["d"]))++ assert found == "cites a derived fact no longer live that rests on a forget"++ @pytest.mark.asyncio+ async def test_a_walk_stopped_at_its_bound_drops_it(self) -> None:+ driver = _driver(_still(["d"], []), [{"uuid": "d", **fact(facts=("p1",))}])++ with patch.object(recall_sources, "CASCADE_MAX_ITEMS", 0):+ found = await rests_on_a_forget(driver, Citations(fact_uuids=["d"]))++ assert found == (+ "cites derived facts no longer live deeper than the check follows"+ )@pytest.mark.parametrize("still, reason",@@ -98,9 +133,9 @@ class TestRestsOnAForget:assert await rests_on_a_forget(forgotten_read, uncited) is not Noneassert forgotten_read.execute_query.await_count == 1- all_live = _driver(_still(["f1"], []), [])+ all_live = _driver(_still(["f1"], []), [], [])assert await rests_on_a_forget(all_live, uncited) is None- assert all_live.execute_query.await_count == 2+ assert all_live.execute_query.await_count == 3def _payload(citations: Citations, completion: ingest.IngestionCompletion) -> dict:@@ -117,9 +152,19 @@ def _payload(citations: Citations, completion: ingest.IngestionCompletion) -> didef _client(still: list[dict]) -> MagicMock:+ """A graphiti client whose driver finds ``still`` of what a write cites,+ each fact of it live (so no derived fact no longer live among it)."""client = MagicMock()client.add_episode = AsyncMock()- client.driver.execute_query = AsyncMock(return_value=(still, [], None))+ live = set(still[0]["facts"]) if still else set()++ async def answer(query: str, **params) -> tuple[list[dict], list, None]:+ if query != recall_sources.FACT_STATES_QUERY:+ return still, [], None+ rows = [{"uuid": u, **fact(live=True)} for u in params["uuids"] if u in live]+ return rows, [], None++ client.driver.execute_query = AsyncMock(side_effect=answer)return clientdiff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_derivation.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_derivation.pynew file mode 100644index 0000000000..e112569c2a--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_derivation.py@@ -0,0 +1,221 @@+"""What a dream write was derived from, recorded where a forget finds it.++A dream write cites the facts and episodes it rests on+(``dream/citations.py``). The ingestion worker holds the graph's write lock+for the whole write (``ingest._write_locked``), and inside it+(``marked_write.py``):++1. first, ``mark`` writes the write's complete citations to a+ ``DreamCitations`` marker in the same graph, ``pending``: the uuid the+ write's episode is given (drawn before the marker and handed to graphiti,+ so the marker names its episode exactly), the episode's name for people+ reading the graph (never used to find it), ``derived_from_facts``,+ ``derived_from_episodes``, the graph's ``group_id`` and when it was+ written; uuids and names only, never text. If the marker cannot be+ written the write is not made: it fails closed. So the provenance of+ every dream fact is in the graph before the fact is. Then the write is+ checked against the forgets made since the pass read the graph+ (``recall_citations.rests_on_a_forget``); one it drops has its marker+ deleted (``withdraw``). Marking first means a forget landing after that+ check, where the lock does not hold, finds the marker;+2. after ``add_episode``, ``record`` stores those citations as+ ``derived_from_facts`` and ``derived_from_episodes``:++ - on the dream's episode, found by its uuid in its graph. graphiti never+ saves an episode node again once ``add_episode`` has written it, so+ this is the lasting record of the write, and it marks the episode as+ the dream's;+ - on every fact the write produced or merged into whose source episodes+ (``episodes``) are all the dream's: the union of their records. A fact+ the write alone produced gets the write's citations. A fact it merged+ into that a user's own episode also states gets none: it has a source+ that no forget of what the dream cited reaches, and so it is never+ retracted for one;++ then settles the write (``recall_landing.settle``): a source it cites+ that a forget reached while it was being written (only where the lock+ did not hold) is cascaded from, which retracts the write's own facts,+ erasing them for a source a hard forget reached; and only then deletes+ the marker. A failure at any step leaves the marker, and the worker+ reports the write as ``provenance_pending``. A write whose+ ``add_episode`` raised marks its marker ``aborted`` (``abort``).++``recall_reconcile.reconcile`` completes or resolves whatever markers a+graph still holds, before every forget's cascade (``recall_forget.py``),+from the dream reaper (``provenance_pending.py``) and in the derivation+backfill.++A statement merged into a fact leaves its attributes as they were+(``fact_attributes.py``), the record among them. A later dream write merging+into a derived fact whose sources are all the dream's rebuilds the union+from the episodes' records. A user's statement merging into one leaves the+record in place, but the fact then names an episode with no record, so the+cascade passes it over (``recall_cascade_queries.py``) as one the user has+said themselves. No extraction writes a record+(``types.POLICY_EDGE_FIELDS``).++A forget retracts the live facts whose record names a fact it forgot or an+episode it hid, and hides the dream episodes whose record does+(``recall_cascade.py``).+"""++import logging+import uuid as uuidlib+from datetime import datetime, timezone++from graphiti_core.driver.driver import GraphDriver++from .recall_citations import Citations+from .recall_landing import settle++logger = logging.getLogger(__name__)++# The label of a pending record's marker, and the states it is in.+MARKER_LABEL = "DreamCitations"+PENDING = "pending"+ABORTED = "aborted"+EXPIRED = "expired"+++async def mark(+ driver: GraphDriver,+ group_id: str,+ episode_uuid: str,+ episode_name: str,+ citations: Citations,+) -> str:+ """Write the ``pending`` marker of the dream write about to be made under+ ``episode_uuid``; the marker's uuid. Raises when it cannot be written:+ the caller must not make the write."""+ marker = str(uuidlib.uuid4())+ await driver.execute_query(+ MARK_QUERY,+ uuid=marker,+ group_id=group_id,+ episode=episode_uuid,+ name=episode_name,+ facts=list(dict.fromkeys(citations.fact_uuids)),+ episodes=list(dict.fromkeys(citations.episode_uuids)),+ state=PENDING,+ now=datetime.now(timezone.utc).isoformat(),+ )+ return marker+++async def abort(driver: GraphDriver, marker: str) -> None:+ """Mark ``marker``'s write as one whose ``add_episode`` raised, for+ reconcile to resolve once it has found no episode under its uuid. Never+ raises: a marker left ``pending`` waits and then expires instead."""+ try:+ await driver.execute_query(+ ABORT_QUERY,+ uuid=marker,+ state=ABORTED,+ now=datetime.now(timezone.utc).isoformat(),+ )+ except Exception:+ logger.warning(f"Could not mark dream marker {marker} aborted", exc_info=True)+++async def withdraw(driver: GraphDriver, marker: str) -> None:+ """Delete the marker of a write dropped before ``add_episode``; one that+ cannot be deleted is marked aborted, for reconcile to drop. Never+ raises."""+ try:+ await driver.execute_query(DROP_MARKER_QUERY, uuid=marker)+ except Exception:+ logger.warning(f"Could not withdraw dream marker {marker}", exc_info=True)+ await abort(driver, marker)+++async def record(+ driver: GraphDriver,+ group_id: str,+ marker: str,+ episode_uuid: str,+ edge_uuids: list[str],+ citations: Citations,+) -> bool:+ """Record ``citations`` on the dream episode ``episode_uuid``, then on+ each of ``edge_uuids`` (the facts it produced or merged into) whose+ sources are all dream episodes, settle the write, then delete its+ ``marker`` (gone already is fine). False, the marker left for+ ``recall_reconcile.reconcile``, when a step failed."""+ try:+ await driver.execute_query(+ RECORD_EPISODE_QUERY,+ episode=episode_uuid,+ group_id=group_id,+ facts=list(dict.fromkeys(citations.fact_uuids)),+ episodes=list(dict.fromkeys(citations.episode_uuids)),+ )+ if edge_uuids:+ await driver.execute_query(+ STAMP_FACTS_QUERY, uuids=edge_uuids, group_id=group_id+ )+ if not await settle(driver, group_id, citations):+ return False+ await driver.execute_query(DROP_MARKER_QUERY, uuid=marker)+ except Exception:+ logger.warning(+ f"Failed to record what dream episode {episode_uuid} was derived "+ f"from in graph {group_id[:20]}; its marker stays for reconcile",+ exc_info=True,+ )+ return False+ return True+++MARK_QUERY = f"""+CREATE (:{MARKER_LABEL} {{+ uuid: $uuid,+ group_id: $group_id,+ episode_uuid: $episode,+ episode_name: $name,+ derived_from_facts: $facts,+ derived_from_episodes: $episodes,+ state: $state,+ created_at: $now+}})+"""++ABORT_QUERY = f"""+MATCH (m:{MARKER_LABEL} {{uuid: $uuid}})+SET m.state = $state, m.aborted_at = $now+"""++DROP_MARKER_QUERY = f"""+MATCH (m:{MARKER_LABEL} {{uuid: $uuid}})+DELETE m+"""++RECORD_EPISODE_QUERY = """+MATCH (ep:Episodic {uuid: $episode})+WHERE ep.group_id = $group_id+SET ep.derived_from_facts = $facts,+ ep.derived_from_episodes = $episodes+"""++# A fact is stamped only when every episode it names is there and carries a+# record: a missing or unrecorded source (a user's episode) could have stated+# it, so it is left unstamped. ``reduce`` builds each union in source order,+# once each.+STAMP_FACTS_QUERY = """+MATCH ()-[e:RELATES_TO]->()+WHERE e.uuid IN $uuids+ AND (e.group_id = $group_id OR e.group_id IS NULL)+ AND e.forgotten_at IS NULL+MATCH (source:Episodic)+WHERE source.uuid IN coalesce(e.episodes, [])+WITH e, collect(DISTINCT source) AS sources+WITH e, sources, [s IN sources | s.uuid] AS found+WHERE all(x IN coalesce(e.episodes, []) WHERE x IN found)+ AND all(s IN sources WHERE s.derived_from_facts IS NOT NULL)+SET e.derived_from_facts = reduce(+ acc = [], s IN sources |+ acc + [x IN s.derived_from_facts WHERE NOT x IN acc]),+ e.derived_from_episodes = reduce(+ acc = [], s IN sources |+ acc + [x IN coalesce(s.derived_from_episodes, []) WHERE NOT x IN acc])+RETURN e.uuid AS uuid+"""diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_derivation_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_derivation_test.pynew file mode 100644index 0000000000..6f4396b8a4--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_derivation_test.py@@ -0,0 +1,229 @@+"""Unit tests for ``recall_derivation``: a dream write's citations are marked+before the ingestion worker writes it, under the uuid its episode is given,+failing closed; then recorded on that episode (found by uuid in its graph)+and on the facts only dream episodes state, the write settled, and only then+the marker dropped; a record or settle that fails leaves the marker. An+``add_episode`` that raised marks the marker aborted; a write dropped before+it has its marker withdrawn.++The worker's side is in ``recall_derivation_worker_test.py``, the settle in+``recall_landing_test.py``. On FalkorDB, through ``dream/apply.py`` and the+production worker: ``recall_cascade_integration_test.py`` (the facts a+dream write produced or merged into), ``recall_provenance_integration_test.py``+(a record that failed, reconciled before the next forget) and+``recall_marker_integration_test.py`` (a colliding episode name, writes that+land late).+"""++from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from . import recall_derivation+from .recall_citations import Citations++_CITED = Citations(fact_uuids=["f1", "f1", "f2"], episode_uuids=["ep1"])+++def _driver() -> MagicMock:+ driver = MagicMock()+ driver.execute_query = AsyncMock(return_value=([], [], None))+ return driver+++class TestMark:+ @pytest.mark.asyncio+ async def test_marks_the_complete_citations_under_the_episodes_uuid(+ self,+ ) -> None:+ driver = _driver()++ marker = await recall_derivation.mark(+ driver, "user_a", "episode-uuid", "dream_p_1", _CITED+ )++ [call] = driver.execute_query.await_args_list+ assert call.args[0] == recall_derivation.MARK_QUERY+ params = dict(call.kwargs)+ assert params.pop("now")+ assert params == {+ "uuid": marker,+ "group_id": "user_a",+ "episode": "episode-uuid",+ "name": "dream_p_1",+ "facts": ["f1", "f2"],+ "episodes": ["ep1"],+ "state": "pending",+ }++ @pytest.mark.asyncio+ async def test_a_marker_that_cannot_be_written_raises(self) -> None:+ driver = _driver()+ driver.execute_query.side_effect = RuntimeError("down")++ with pytest.raises(RuntimeError):+ await recall_derivation.mark(+ driver, "user_a", "episode-uuid", "dream_p_1", _CITED+ )++ def test_a_marker_holds_uuids_and_names_only(self) -> None:+ query = recall_derivation.MARK_QUERY+ assert "CREATE (:DreamCitations {" in query+ assert "episode_uuid: $episode" in query+ for text in ("content", "fact:", "rationale", "source_description"):+ assert text not in query++ def test_the_episode_is_found_by_its_uuid_in_its_graph_never_its_name(+ self,+ ) -> None:+ query = recall_derivation.RECORD_EPISODE_QUERY+ assert "MATCH (ep:Episodic {uuid: $episode})" in query+ assert "ep.group_id = $group_id" in query+ assert "name" not in query+++class TestAbort:+ @pytest.mark.asyncio+ async def test_marks_the_marker_aborted(self) -> None:+ driver = _driver()++ await recall_derivation.abort(driver, "m1")++ [call] = driver.execute_query.await_args_list+ assert call.args[0] == recall_derivation.ABORT_QUERY+ assert (call.kwargs["uuid"], call.kwargs["state"]) == ("m1", "aborted")++ @pytest.mark.asyncio+ async def test_never_raises(self, caplog: pytest.LogCaptureFixture) -> None:+ driver = _driver()+ driver.execute_query.side_effect = RuntimeError("down")++ await recall_derivation.abort(driver, "m1")++ assert "Could not mark dream marker m1 aborted" in caplog.text+++class TestWithdraw:+ """The marker of a write dropped before ``add_episode`` goes."""++ @pytest.mark.asyncio+ async def test_deletes_the_marker(self) -> None:+ driver = _driver()++ await recall_derivation.withdraw(driver, "m1")++ [call] = driver.execute_query.await_args_list+ assert (call.args[0], call.kwargs) == (+ recall_derivation.DROP_MARKER_QUERY,+ {"uuid": "m1"},+ )++ @pytest.mark.asyncio+ async def test_one_that_cannot_be_deleted_is_marked_aborted(self) -> None:+ """For reconcile to drop: it has no episode."""+ driver = _driver()+ driver.execute_query.side_effect = [RuntimeError("down"), ([], [], None)]++ await recall_derivation.withdraw(driver, "m1")++ _, abort = driver.execute_query.await_args_list+ assert (abort.args[0], abort.kwargs["state"]) == (+ recall_derivation.ABORT_QUERY,+ "aborted",+ )+++class TestRecord:+ @pytest.mark.asyncio+ async def test_records_the_episode_the_facts_settles_then_drops_the_marker(+ self,+ ) -> None:+ driver = _driver()+ settled: list[int] = []++ async def settle(driver_, group_id, citations) -> bool:+ settled.append(driver.execute_query.await_count)+ assert (group_id, citations) == ("user_a", _CITED)+ return True++ with patch.object(recall_derivation, "settle", settle):+ recorded = await recall_derivation.record(+ driver, "user_a", "m1", "dream-ep", ["e1"], _CITED+ )++ episode, facts, drop = driver.execute_query.await_args_list+ assert episode.args[0] == recall_derivation.RECORD_EPISODE_QUERY+ assert episode.kwargs == {+ "episode": "dream-ep",+ "group_id": "user_a",+ "facts": ["f1", "f2"],+ "episodes": ["ep1"],+ }+ assert facts.args[0] == recall_derivation.STAMP_FACTS_QUERY+ assert facts.kwargs == {"uuids": ["e1"], "group_id": "user_a"}+ assert settled == [2], "settled after the record, before the drop"+ assert (drop.args[0], drop.kwargs) == (+ recall_derivation.DROP_MARKER_QUERY,+ {"uuid": "m1"},+ )+ assert recorded is True++ @pytest.mark.asyncio+ async def test_a_write_that_touched_no_fact_records_its_episode_only(+ self,+ ) -> None:+ driver = _driver()++ with patch.object(recall_derivation, "settle", AsyncMock(return_value=True)):+ await recall_derivation.record(+ driver, "user_a", "m1", "dream-ep", [], _CITED+ )++ queries = [call.args[0] for call in driver.execute_query.await_args_list]+ assert queries == [+ recall_derivation.RECORD_EPISODE_QUERY,+ recall_derivation.DROP_MARKER_QUERY,+ ]++ @pytest.mark.asyncio+ async def test_a_settle_left_unfinished_keeps_the_marker(self) -> None:+ driver = _driver()++ with patch.object(recall_derivation, "settle", AsyncMock(return_value=False)):+ recorded = await recall_derivation.record(+ driver, "user_a", "m1", "dream-ep", ["e1"], _CITED+ )++ assert recorded is False+ queries = [call.args[0] for call in driver.execute_query.await_args_list]+ assert recall_derivation.DROP_MARKER_QUERY not in queries++ @pytest.mark.asyncio+ async def test_a_failure_leaves_the_marker_and_says_so(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ driver = _driver()+ driver.execute_query.side_effect = [([], [], None), RuntimeError("down")]++ recorded = await recall_derivation.record(+ driver, "user_a", "m1", "dream-ep", ["e1"], _CITED+ )++ assert recorded is False+ assert driver.execute_query.await_count == 2, "the marker is not dropped"+ assert "its marker stays for reconcile" in caplog.text+++class TestTheStamp:+ def test_a_fact_is_stamped_only_when_every_source_is_a_recorded_episode(+ self,+ ) -> None:+ query = recall_derivation.STAMP_FACTS_QUERY+ assert "all(x IN coalesce(e.episodes, []) WHERE x IN found)" in query+ assert "all(s IN sources WHERE s.derived_from_facts IS NOT NULL)" in query+ assert "e.forgotten_at IS NULL" in query++ def test_the_stamp_is_the_union_of_its_sources_records(self) -> None:+ query = recall_derivation.STAMP_FACTS_QUERY+ assert "acc + [x IN s.derived_from_facts WHERE NOT x IN acc]" in query+ assert "SET e.derived_from_facts = reduce(" in querydiff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_derivation_worker_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_derivation_worker_test.pynew file mode 100644index 0000000000..23c83497d1--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_derivation_worker_test.py@@ -0,0 +1,315 @@+"""Unit tests for the ingestion worker's side of ``recall_derivation``+(``marked_write.py``, called from ``ingest._write_locked``): a dream write's+citations are marked first, under the uuid drawn for its episode, failing+closed, and only then checked against forgets (one resting on a forget is+dropped and its marker withdrawn); the episode is placed under that uuid,+``write_pending``, and graphiti's ``add_episode`` handed it; the write is+recorded after it, all under the graph's write lock. A record that fails is+noted for the reaper and reported as ``provenance_pending``; a marked write+that raised marks its marker aborted, is noted and counts as failed. The+record itself is pinned in ``recall_derivation_test.py``.+"""++import asyncio+from collections.abc import Callable+from datetime import datetime, timezone+from types import SimpleNamespace+from unittest.mock import AsyncMock, MagicMock, patch++import pytest+from graphiti_core.nodes import EpisodeType++from . import ingest, marked_write+from .recall_citations import Citations+from .recall_fake_redis import FakeRedis+from .scope import write_lock_key+++def _payload(citations: Citations | None, completion=None) -> dict:+ return {+ "name": "dream_p1_consolidate_000",+ "episode_body": '{"content": "Alice works on Atlas"}',+ "source": EpisodeType.json,+ "source_description": "dream-pass consolidation",+ "reference_time": datetime(2026, 9, 28, tzinfo=timezone.utc),+ "group_id": "user_test",+ "_citations": citations,+ "_completion": completion,+ }+++def _written(episode: str, edges: dict[str, list[str]]) -> SimpleNamespace:+ """What ``add_episode`` returned: its episode and each edge's episodes."""+ return SimpleNamespace(+ episode=SimpleNamespace(uuid=episode),+ edges=[+ SimpleNamespace(uuid=uuid, episodes=episodes, expired_at=None)+ for uuid, episodes in edges.items()+ ],+ )+++def _landed(edges: Callable[[str], dict[str, list[str]]]) -> Callable:+ """graphiti's ``add_episode``: the episode it was handed, with ``edges``."""++ async def add_episode(**kwargs) -> SimpleNamespace:+ uuid = kwargs["uuid"] or "graphiti-drawn"+ return _written(uuid, edges(uuid))++ return add_episode+++class _Worker(SimpleNamespace):+ """What the worker was handed and what it called, in ``order``."""++ add_episode: AsyncMock+ mark: AsyncMock+ record: AsyncMock+ abort: AsyncMock+ withdraw: AsyncMock+ noted: AsyncMock+ queries: AsyncMock+ order: list[str]+++def _tracked(name: str, order: list[str], mock: AsyncMock) -> AsyncMock:+ """``mock``, noting ``name`` in ``order`` each time it is awaited."""++ async def run(*args, **kwargs) -> object:+ order.append(name)+ return await mock(*args, **kwargs)++ return AsyncMock(side_effect=run)+++async def _work(+ payload: dict,+ written: Callable | Exception,+ monkeypatch: pytest.MonkeyPatch,+ *,+ mark: AsyncMock | None = None,+ record: AsyncMock | None = None,+ forgotten: str | None = None,+ check_error: Exception | None = None,+) -> _Worker:+ """One payload through the worker, marking through ``mark``, recording+ through ``record``, and finding the write resting on a forget when+ ``forgotten`` says why, or the check raising ``check_error``."""+ order: list[str] = []+ client = MagicMock()++ async def check(driver, citations: Citations | None) -> str | None:+ if citations is None:+ return None+ order.append("check")+ if check_error is not None:+ raise check_error+ return forgotten++ async def add_episode(**kwargs):+ order.append("add_episode")+ if isinstance(written, Exception):+ raise written+ return await written(**kwargs)++ async def query(cypher: str, **params):+ if cypher == marked_write.PLACE_EPISODE_QUERY:+ order.append("place")+ return [{"facts": ["f1"], "episodes": []}], [], None++ client.add_episode = AsyncMock(side_effect=add_episode)+ client.driver.execute_query = AsyncMock(side_effect=query)+ worker = _Worker(+ add_episode=client.add_episode,+ mark=_tracked("mark", order, mark or AsyncMock(return_value="m1")),+ record=_tracked("record", order, record or AsyncMock(return_value=True)),+ abort=_tracked("abort", order, AsyncMock()),+ withdraw=_tracked("withdraw", order, AsyncMock()),+ noted=AsyncMock(),+ queries=client.driver.execute_query,+ order=order,+ )+ queue: asyncio.Queue = asyncio.Queue(maxsize=10)+ queue.put_nowait(payload)+ monkeypatch.setattr(ingest, "_WORKER_IDLE_TIMEOUT", 0.05)+ with (+ patch.object(ingest, "get_graphiti_client", AsyncMock(return_value=client)),+ patch.object(ingest, "ensure_indices_once", AsyncMock()),+ patch.object(ingest, "previous_episode_uuids", AsyncMock(return_value=[])),+ patch.object(ingest, "rests_on_a_forget", AsyncMock(side_effect=check)),+ patch.object(marked_write, "mark", worker.mark),+ patch.object(marked_write, "record", worker.record),+ patch.object(marked_write, "abort", worker.abort),+ patch.object(marked_write, "withdraw", worker.withdraw),+ patch.object(marked_write, "note_pending", worker.noted),+ ):+ await ingest._ingestion_worker("test-user", "user_test", queue)+ return worker+++class TestTheWorker:+ @pytest.mark.asyncio+ async def test_marks_checks_places_writes_and_records_under_one_uuid(+ self, lock_redis: FakeRedis, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ """An edge the episode invalidated does not name it and is left out."""+ held: list[str] = []+ cited = Citations(fact_uuids=["f1"])++ def step(name: str, answer: object):+ async def run(*args, **kwargs) -> object:+ locked = write_lock_key("user_test") in lock_redis.values+ held.append(f"{name}:{'locked' if locked else 'unlocked'}")+ return answer++ return run++ worker = await _work(+ _payload(cited),+ _landed(+ lambda uuid: {"new": [uuid], "merged": ["user-ep", uuid], "old": ["x"]}+ ),+ monkeypatch,+ mark=AsyncMock(side_effect=step("mark", "m1")),+ record=AsyncMock(side_effect=step("record", True)),+ )++ assert held == ["mark:locked", "record:locked"]+ _, group, episode, name, citations = worker.mark.await_args.args+ assert (group, name, citations) == (+ "user_test",+ "dream_p1_consolidate_000",+ cited,+ )+ assert worker.order == ["mark", "check", "place", "add_episode", "record"]+ [place] = [+ call+ for call in worker.queries.await_args_list+ if call.args[0] == marked_write.PLACE_EPISODE_QUERY+ ]+ assert (place.kwargs["uuid"], place.kwargs["group_id"]) == (+ episode,+ "user_test",+ )+ assert place.kwargs["content"] == '{"content": "Alice works on Atlas"}'+ assert worker.add_episode.await_args.kwargs["uuid"] == episode+ _, group, marker, recorded, touched, citations = worker.record.await_args.args+ assert (group, marker, recorded) == ("user_test", "m1", episode)+ assert (touched, citations) == (["new", "merged"], cited)+ worker.noted.assert_not_awaited()+ worker.abort.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_a_marker_that_cannot_be_written_drops_the_write(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ completion = ingest.IngestionCompletion()+ completion.register()++ worker = await _work(+ _payload(Citations(fact_uuids=["f1"]), completion),+ _landed(lambda uuid: {"new": [uuid]}),+ monkeypatch,+ mark=AsyncMock(side_effect=RuntimeError("down")),+ )++ worker.add_episode.assert_not_awaited()+ worker.record.assert_not_awaited()+ assert worker.order == ["mark"], "nothing checked or placed"+ assert (completion.failed, completion.provenance_pending) == (1, 0)+ assert await completion.wait(0), "the drain is not held up"++ @pytest.mark.asyncio+ async def test_a_write_resting_on_a_forget_is_dropped_its_marker_withdrawn(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ """Marked before the check, so a forget landing after the check+ (where the lock does not hold) finds the marker."""+ completion = ingest.IngestionCompletion()+ completion.register()++ worker = await _work(+ _payload(Citations(fact_uuids=["f1"]), completion),+ _landed(lambda uuid: {"new": [uuid]}),+ monkeypatch,+ forgotten="cites a fact that is forgotten or gone",+ )++ assert worker.order == ["mark", "check", "withdraw"]+ assert worker.withdraw.await_args.args[1] == "m1"+ worker.add_episode.assert_not_awaited()+ assert (completion.dropped_forgotten, completion.failed) == (1, 0)++ @pytest.mark.asyncio+ async def test_a_check_that_raised_withdraws_the_marker_and_fails(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ """Nothing is placed yet: a marker left pending would hold up the+ forgets of what it cites until it expired."""+ completion = ingest.IngestionCompletion()+ completion.register()++ worker = await _work(+ _payload(Citations(fact_uuids=["f1"]), completion),+ _landed(lambda uuid: {"new": [uuid]}),+ monkeypatch,+ check_error=RuntimeError("graph down"),+ )++ assert worker.order == ["mark", "check", "withdraw"]+ assert worker.withdraw.await_args.args[1] == "m1"+ worker.add_episode.assert_not_awaited()+ assert (completion.dropped_forgotten, completion.failed) == (0, 1)++ @pytest.mark.asyncio+ async def test_a_failed_record_is_noted_and_reported_pending(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ completion = ingest.IngestionCompletion()+ completion.register()++ worker = await _work(+ _payload(Citations(fact_uuids=["f1"]), completion),+ _landed(lambda uuid: {"new": [uuid]}),+ monkeypatch,+ record=AsyncMock(return_value=False),+ )++ worker.noted.assert_awaited_once_with("user_test")+ assert (completion.failed, completion.provenance_pending) == (0, 1)++ @pytest.mark.asyncio+ async def test_a_marked_write_that_raised_is_aborted_noted_and_failed(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ """It may have landed in part: reconcile resolves its marker once it+ has found no episode under its uuid."""+ completion = ingest.IngestionCompletion()+ completion.register()++ worker = await _work(+ _payload(Citations(fact_uuids=["f1"]), completion),+ RuntimeError("graphiti down"),+ monkeypatch,+ )++ assert worker.order == ["mark", "check", "place", "add_episode", "abort"]+ assert worker.abort.await_args.args[1] == "m1"+ worker.noted.assert_awaited_once_with("user_test")+ worker.record.assert_not_awaited()+ assert (completion.failed, completion.provenance_pending) == (1, 0)++ @pytest.mark.asyncio+ async def test_a_chat_write_marks_places_and_records_nothing(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ worker = await _work(+ _payload(None),+ _landed(lambda uuid: {"new": [uuid]}),+ monkeypatch,+ )++ worker.mark.assert_not_awaited()+ worker.record.assert_not_awaited()+ assert worker.order == ["add_episode"]+ assert worker.add_episode.await_args.kwargs["uuid"] is Nonediff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_dream_citations_integration_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_dream_citations_integration_test.pyindex 9606d73f66..83618e7eec 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/recall_dream_citations_integration_test.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_dream_citations_integration_test.py@@ -6,9 +6,12 @@ The dream reads Alice's fact and its episode, the user forgets the fact(soft or hard), then the dream writes a consolidation or a proposal restingon what it read: the worker drops it under the graph's write lock(``recall_citations.py``), no edge is added, the fact stays out of recall-and the pass reports the drop. A write resting only on live memory still-lands. Reproduced first by an independent validation-(``r5-quality-delayed-operations.py``, the four stale-dream cases).+and the pass reports the drop. A write citing nothing the pass read never+gets that far: apply drops it first (``dream/citations.py``). A write resting+only on live memory still lands. Reproduced first by an independent+validation (``r5-quality-delayed-operations.py``, the four stale-dream+cases). A forget made after the dream's writes landed is+``recall_cascade_integration_test.py``.Run with FalkorDB reachable (see ``conftest.py``)::@@ -104,7 +107,10 @@ def _resting_on(operation: str, read: DreamInput, content: str) -> DreamOperatiofacts, episodes = sorted(read.known_fact_uuids), sorted(read.known_episode_uuids)if operation == "consolidate":fact = ConsolidatedFact(- content=content, confidence=0.9, source_episode_uuids=episodes+ content=content,+ confidence=0.9,+ source_episode_uuids=episodes,+ source_fact_uuids=facts,)return DreamOperations(writes=[fact])finding = ProposedFinding(@@ -177,13 +183,12 @@ async def test_a_dream_write_resting_on_a_fact_forgotten_since_it_read_is_droppe@pytest.mark.integration@pytest.mark.asyncio@pytest.mark.parametrize("hard", [False, True], ids=["soft", "hard"])-async def test_an_uncited_dream_write_after_a_forget_is_dropped(+async def test_an_uncited_dream_write_is_dropped_before_it_is_queued(scope_graph, stub_graphiti_client, dream_apply, hard: bool) -> None:- """Citing nothing, it rests on everything the pass read, and its- statement is compared with the sentences forgotten facts keep. The soft- case applies with nothing read, so only the statement can drop it; a- hard forget keeps no sentence, so there the pass's read drops it."""+ """A restatement of the forgotten fact citing nothing, and one citing+ only a uuid the pass never read, are dropped by apply and counted: the+ worker never sees them."""driver, scope = scope_graph_, edges = await ingest_facts(driver, scope, stub_graphiti_client, [ALICE], session_id="s-1"@@ -192,16 +197,17 @@ async def test_an_uncited_dream_write_after_a_forget_is_dropped(await retract(scope, list(edges.values()), hard=hard)before = await _edges(driver)restated = ConsolidatedFact(content=" alice WORKS on\tAtlas ", confidence=0.9)+ made_up = restated.model_copy(update={"source_fact_uuids": ["made-up"]})stats = await _apply(driver,scope,stub_graphiti_client,- DreamOperations(writes=[restated]),- read if hard else None,+ DreamOperations(writes=[restated, made_up]),+ read,)- assert stats["dropped_forgotten"] == 1+ assert (stats["uncited_writes_dropped"], stats["dropped_forgotten"]) == (2, 0)assert await _edges(driver) == before, "the dream added an edge"assert await _dream_episodes(driver) == 0assert ALICE[2] not in await _recalled(scope)@@ -214,8 +220,8 @@ async def test_a_dream_write_resting_only_on_live_memory_still_lands(scope_graph, stub_graphiti_client, dream_apply, operation: str) -> None:"""Alice is forgotten after the pass read; a write citing only Bob's- fact and episode is written, like one citing nothing from a pass that- read after the forget."""+ fact and episode is written, like one from a pass that read after the+ forget."""driver, scope = scope_graph_, alice = await ingest_facts(driver, scope, stub_graphiti_client, [ALICE], session_id="s-1"@@ -228,8 +234,8 @@ async def test_a_dream_write_resting_only_on_live_memory_still_lands(cited = await _apply(driver, scope, stub_graphiti_client, ops, read, extracted=_BOREALIS)- uncited = ConsolidatedFact(content="Bob plans Borealis", confidence=0.9)- ops, read = DreamOperations(writes=[uncited]), await _gather(scope)+ read = await _gather(scope)+ ops = _resting_on(operation, read, "Bob plans Borealis")fresh = await _apply(driver,scope,diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_erase.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_erase.pynew file mode 100644index 0000000000..96fba68d16--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_erase.py@@ -0,0 +1,74 @@+"""A hard forget erases the sentences the dream derived from what it forgot.++A soft forget keeps every sentence it hides for audit: a fact's in its+``fact_redacted`` / ``name_redacted`` copies (``recall_hide.py``), a dream+episode's in its body. A hard forget asks for the text itself to go. It+deletes the facts the user named and empties the episodes only they kept+(``recall_orphans.py``); what the dream derived from them its cascade+retracts softly, as always (``recall_cascade.py``), keeping the edges, their+markers, reasons and records, so the walk and a retry still find them, and+erases here:++- every derived fact the walk reached, retracted or walked through: its+ ``fact`` and ``name`` read ``recall.FORGOTTEN_FACT``, as the scrub leaves+ them, both audit copies are blank, and its ``fact_embedding``, the vector+ graphiti computed from the sentence, is removed. FalkorDB's vector+ similarity reads a missing embedding as no score, so graphiti's searches+ and its dedup simply pass the edge by. Nothing else on the edge holds the+ sentence: its other properties are uuids, times and ``MemoryFact`` fields;+- every dream episode it hid: its body, and the rationale and citations its+ ``source_description`` listed after the kind (``dream-pass proposal``).+ Only an episode carrying a dream's record (``recall_derivation.py``) is the+ dream's, so a user's episode the cascade hides keeps its text.++Each write is idempotent, so a hard forget repeated after a failure erases+what the first one left.+"""++from typing import Any++from graphiti_core.driver.driver import GraphDriver++from .recall import FORGOTTEN_FACT+++async def erase(+ driver: GraphDriver, facts: list[str], episodes: list[str]+) -> list[str]:+ """Blank ``facts``' text and audit copies (their entities already+ scrubbed, ``recall_hide.scrub``), then the dream episodes among+ ``episodes``; the names of those, each naming the dream pass that wrote+ it (Postgres keeps that pass's copies of the text,+ ``dream/forget_copies.py``)."""+ if facts:+ await driver.execute_query(+ ERASE_FACTS_QUERY, uuids=facts, placeholder=FORGOTTEN_FACT+ )+ if not episodes:+ return []+ erased = await driver.execute_query(ERASE_DREAM_EPISODES_QUERY, uuids=episodes)+ return [row["name"] for row in _rows(erased)]+++def _rows(result: Any) -> list[dict[str, Any]]:+ return result[0] if result else []+++ERASE_FACTS_QUERY = """+MATCH ()-[e:RELATES_TO]->()+WHERE e.uuid IN $uuids+SET e.fact = $placeholder,+ e.name = $placeholder,+ e.fact_redacted = '',+ e.name_redacted = '',+ e.fact_embedding = NULL+"""++# The description keeps what precedes its first ``;``: the kind of write.+ERASE_DREAM_EPISODES_QUERY = """+MATCH (ep:Episodic)+WHERE ep.uuid IN $uuids AND ep.derived_from_facts IS NOT NULL+SET ep.content = '',+ ep.source_description = split(ep.source_description, ';')[0]+RETURN ep.name AS name+"""diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_erase_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_erase_test.pynew file mode 100644index 0000000000..fb846ffc06--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_erase_test.py@@ -0,0 +1,88 @@+"""Unit tests for a hard forget's erasure (``recall_erase.py``), run by the+cascade over the in-memory graph of ``recall_cascade_fake.py``: what it+erases, the user's episode it never empties, and that a soft forget erases+nothing. The Cypher runs on FalkorDB in+``recall_cascade_erase_integration_test.py``.+"""++import pytest++from . import recall_cascade, recall_erase+from .memory_model import ForgetResult+from .recall_cascade_fake import CascadeGraph, chain++_NOW = "2026-09-28T00:00:00+00:00"+++async def _cascade(graph: CascadeGraph, *, erase: bool) -> ForgetResult:+ result = ForgetResult(redacted_episodes=["e0"])+ await recall_cascade.cascade(graph, "user_a", ["f"], _NOW, result, erase=erase)+ return result+++class TestHardForget:+ @pytest.mark.asyncio+ async def test_it_erases_each_derived_fact_and_dream_text_it_reaches(+ self,+ ) -> None:+ """Softly retracted, as ever, and each fact's text and each dream+ episode's body erased as the walk reaches them; the user's episode+ ``e0``, hidden for the forgotten fact, keeps its text."""+ graph = chain()++ result = await _cascade(graph, erase=True)++ assert (result.derived, result.failures) == (["c", "p", "q"], [])+ assert graph.erased == ["c", "dc", "p", "dp", "q", "dq"]+ assert "e0" in result.redacted_episodes and "e0" not in graph.erased++ @pytest.mark.asyncio+ async def test_it_erases_a_derived_fact_it_only_walks_through(self) -> None:+ graph = chain()+ graph.facts["c"].live = False++ result = await _cascade(graph, erase=True)++ assert result.derived == ["p", "q"]+ assert graph.facts["c"].reason is None, "not retracted"+ assert "c" in graph.scrubbed and "c" in graph.erased++ @pytest.mark.asyncio+ async def test_forgetting_again_erases_what_an_earlier_try_retracted(+ self,+ ) -> None:+ graph = chain()+ graph.facts["c"].live = False+ graph.facts["c"].reason = "derived_from_forgotten:f"++ await _cascade(graph, erase=True)++ assert graph.erased[0] == "c"++ @pytest.mark.asyncio+ async def test_a_soft_forget_erases_nothing(self) -> None:+ graph = chain()+ graph.facts["c"].live = False++ await _cascade(graph, erase=False)++ assert graph.erased == []+ assert graph.scrubbed == ["p", "q"], "a fact walked through keeps its text"+ erasing = {+ recall_erase.ERASE_FACTS_QUERY,+ recall_erase.ERASE_DREAM_EPISODES_QUERY,+ }+ assert not erasing & set(graph.queries)+++class TestQueries:+ def test_a_fact_keeps_the_placeholder_and_blank_audit_copies(self) -> None:+ query = recall_erase.ERASE_FACTS_QUERY+ assert "e.fact = $placeholder" in query and "e.name = $placeholder" in query+ assert "e.fact_redacted = ''" in query and "e.name_redacted = ''" in query+ assert "e.fact_embedding = NULL" in query, "the vector encodes the sentence"++ def test_only_the_dreams_episodes_are_emptied(self) -> None:+ query = recall_erase.ERASE_DREAM_EPISODES_QUERY+ assert "ep.derived_from_facts IS NOT NULL" in query+ assert "ep.content = ''" in querydiff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_fake_redis.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_fake_redis.pyindex bee90e9776..bfae7d6231 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/recall_fake_redis.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_fake_redis.py@@ -13,6 +13,24 @@ class FakeRedis:def __init__(self) -> None:self.values: dict[str, str] = {}self.ttls: dict[str, int] = {}+ # The graphs noted as holding a pending dream record+ # (``provenance_pending.py``), by set.+ self.sets: dict[str, set[str]] = {}++ async def sadd(self, name: str, *values: object) -> int:+ bucket = self.sets.setdefault(name, set())+ added = {str(value) for value in values} - bucket+ bucket |= added+ return len(added)++ async def srem(self, name: str, *values: object) -> int:+ bucket = self.sets.get(name, set())+ removed = {str(value) for value in values} & bucket+ bucket -= removed+ return len(removed)++ async def srandmember(self, name: str, number: int) -> list[str]:+ return sorted(self.sets.get(name, set()))[:number]async def get(self, key: str) -> str | None:return self.values.get(key)diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_forget.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_forget.pyindex 362061abee..0abf1837da 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/recall_forget.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_forget.py@@ -1,12 +1,15 @@"""Forgetting under the recall policy.After ``retract`` the recall paths in ``recall.py`` return neither the-forgotten facts nor the episode text they came from, the sentence is gone-from the fact and from what graphiti read out of it onto entities, and the-audit record stays; ``graphiti/AGENTS.md`` lists the limits. The chat forget-tool and the settings page both forget through here, so a forget means the-same thing wherever it starts. ``forgotten_at``, the policy's marker for a-forgotten fact, is written here and, for forgets made before it existed, by+forgotten facts nor the episode text they came from, nor what the dream+derived from them (``recall_cascade.py``), the sentence is gone from the+fact and from what graphiti read out of it onto entities, and the audit+record stays; ``graphiti/AGENTS.md`` lists the limits. The chat forget tool+and the settings page both forget through here, so a forget means the same+thing wherever it starts. ``forgotten_at``, the policy's marker for a+forgotten fact, is written here (for a derived fact, by the cascade this+runs, which ``migrations/backfill_derivations.py`` also runs for forgets+made before it) and, for forgets made before the marker existed, by``migrations/backfill_legacy_forgets.py``; nothing else writes it."""@@ -16,9 +19,20 @@ from typing import Anyfrom graphiti_core.driver.driver import GraphDriver+from backend.copilot.dream.forget_copies import blank_dream_copies+from .falkordb_driver import open_driver-from .memory_model import ForgetResult, MemoryForgetFailure, MemoryStatus+from .memory_model import (+ ForgetResult,+ MemoryForgetFailure,+ MemoryForgetFailureCode,+ MemoryStatus,+)from .recall import USER_FORGET_REASON+from .recall_cascade import cascade+from .recall_cascade_queries import NAMED_ROOTS_QUERY+from .recall_cascade_walk import derived_reason+from .recall_forget_writes import provenance_incomplete, still_landing, unreconciledfrom .recall_hide import hidefrom .recall_orphans import purgefrom .scope import MemoryScope@@ -26,6 +40,15 @@ from .scope_lock import FORGET_LOCK_WAIT_SECONDS, LockState, graph_write_locklogger = logging.getLogger(__name__)+# Failures that mean the forget did nothing to a uuid.+_NOTHING_DONE = frozenset(+ {+ MemoryForgetFailureCode.NO_MATCH,+ MemoryForgetFailureCode.QUERY_ERROR,+ MemoryForgetFailureCode.BUSY,+ }+)+async def retract(scope: MemoryScope,@@ -39,19 +62,49 @@ async def retract(The forget holds the graph's write lock (``scope_lock.py``), so it neverlands while an ingestion is between reading the graph and saving overit; when the lock stays busy for ``FORGET_LOCK_WAIT_SECONDS`` every uuid- fails as ``busy`` and nothing is written.+ fails as ``busy`` and nothing is written. It first completes the+ derivation records of every dream write still marked in the graph whose+ write landed (``recall_reconcile.reconcile``), so its cascade sees their+ provenance; if that fails or leaves one, each fact forgotten is a+ ``cleanup_error`` and forgetting it again finishes the job. So is each+ when, after its cascade, a dream write that could still land cites+ something the forget reached (``recall_forget_writes.py``): the write's+ own settle retracts it when it lands (``recall_landing.py``), and+ forgetting again once it has reports the forget done.Soft (the default) stamps ``forgotten_at``, sets ``status='retracted'``and ``expiration_reason``, keeps an earlier ``expired_at`` and leaves``invalid_at`` alone: a forget retracts our record of a fact, it doesnot say the world changed (Snodgrass). ``recall_hide.hide`` then movesthe sentence out of what graphiti reads and redacts every episode citing- the fact; edges and episodes stay for audit. Hard does the same first,- then empties and deletes what only those edges kept, the edges last+ the fact; edges and episodes stay for audit. Then, under the same lock,+ ``recall_cascade.cascade`` retracts the facts the dream derived from the+ forgotten ones, transitively, and hides the dream episodes that did+ (``ForgetResult.derived``). It runs even when hiding failed: it redacts+ the forgotten facts' episodes itself before anything else, so the dream's+ copies leave recall now rather than when the forget is repeated. Hard does all that first, stamping+ ``hard_forgotten_at`` on each fact it retracts (a dream write landing+ before the purge, where the lock does not hold, is then erased as one+ landing after it would be), the cascade soft too but erasing the+ derived text it reaches (``recall_erase.py``), then empties and deletes+ what only the forgotten edges kept, the edges last(``recall_orphans.purge``), so forgetting again after any failure- finishes the job. A failed step after the edge write is a- ``cleanup_error`` on each edge it concerned; recall hides the fact and- its text regardless.+ finishes the job, a purged fact's cascade included (below). Last, out of+ the lock, it blanks the dream's copies of that text in Postgres+ (``dream/forget_copies.py``), marking each dream pass of the scope it may+ have reached so that one still running is blanked as it ends: a failure+ there is a ``cleanup_error`` too. A failed+ step after the edge write is a ``cleanup_error`` on each edge it+ concerned, one failure per uuid however many steps failed; recall hides+ the fact and its text regardless.++ A uuid that is no longer in the graph but that a derivation record, a+ citation marker, an episode's ``redacted_for`` or an earlier cascade's+ reason still names was a fact a hard forget purged, perhaps before its+ cascade finished: the forget goes on with that cascade, erasing, and+ lists it in ``ForgetResult.resumed`` instead of failing it as no match;+ when that cannot be read, such a uuid fails as ``query_error`` and the+ rest of the forget goes on."""requested = list(dict.fromkeys(uuids))if not requested:@@ -63,9 +116,45 @@ async def retract(return ForgetResult(failures=busy)driver = open_driver(scope)try:- return await _forget(driver, scope.group_id, requested, hard, reason)+ result = await _forget(driver, scope.group_id, requested, hard, reason)finally:await driver.close()+ if hard:+ await _blank_copies(scope, requested, result)+ return result+++async def _blank_copies(+ scope: MemoryScope, requested: list[str], result: ForgetResult+) -> None:+ """Blank the dream's copies in Postgres of what a hard forget reached+ (``dream/forget_copies.py``); a failure is a ``cleanup_error`` on each+ uuid forgotten."""+ unreached = {f.uuid for f in result.failures if f.code in _NOTHING_DONE}+ forgotten = [uuid for uuid in requested if uuid not in unreached]+ if not forgotten:+ return+ reached = {+ *forgotten,+ *result.derived,+ *result.passed,+ *result.redacted_episodes,+ *result.tombstoned_episodes,+ }+ try:+ await blank_dream_copies(+ scope, reached=reached, dream_writes=result.dream_writes+ )+ except Exception as exc:+ logger.warning(+ f"Hard forget could not blank the dream's copies in graph "+ f"{scope.group_id[:20]}",+ exc_info=True,+ )+ result.failures.extend(+ MemoryForgetFailure.cleanup_error(uuid, exc) for uuid in forgotten+ )+ result.failures = _first_per_uuid(result.failures)async def _forget(@@ -73,16 +162,71 @@ async def _forget() -> ForgetResult:result = ForgetResult()now = datetime.now(timezone.utc).isoformat()+ missing = await unreconciled(driver, group_id)found = await _existing_edges(driver, group_id, uuids, result)- retracted = await _retract_edges(driver, group_id, found, reason, now, result)+ result.resumed = await _purged_roots(driver, group_id, result)+ retracted = await _retract_edges(+ driver, group_id, found, reason, now, result, hard=hard+ )hidden = await hide(driver, group_id, retracted, now, result)+ if retracted:+ await cascade(driver, group_id, retracted, now, result, erase=hard)+ if result.resumed:+ await cascade(driver, group_id, result.resumed, now, result, erase=True)+ roots = [*retracted, *result.resumed]+ pending = missing or await still_landing(driver, group_id, result, roots)+ if pending is not None:+ provenance_incomplete(result, roots, pending)if not hard:result.deleted = retractedelif hidden:await purge(driver, group_id, retracted, now, result)+ result.failures = _first_per_uuid(result.failures)return result+async def _purged_roots(+ driver: GraphDriver, group_id: str, result: ForgetResult+) -> list[str]:+ """The uuids that matched no edge but that something still names as a+ root: taken off ``result.failures``, to resume their cascade. When the+ names cannot be read, those uuids fail as ``query_error`` instead, since+ a retry may find them."""+ missing = [+ failure.uuid+ for failure in result.failures+ if failure.code is MemoryForgetFailureCode.NO_MATCH+ ]+ if not missing:+ return []+ try:+ rows = _rows(+ await driver.execute_query(+ NAMED_ROOTS_QUERY, uuids=missing, prefix=derived_reason("")+ )+ )+ except Exception as exc:+ logger.warning(+ f"Forget could not look for purged facts in graph {group_id[:20]}",+ exc_info=True,+ )+ result.failures = [f for f in result.failures if f.uuid not in missing]+ result.failures.extend(MemoryForgetFailure.query_error(u, exc) for u in missing)+ return []+ named = {row["uuid"] for row in rows}+ result.failures = [f for f in result.failures if f.uuid not in named]+ return [uuid for uuid in missing if uuid in named]+++def _first_per_uuid(failures: list[MemoryForgetFailure]) -> list[MemoryForgetFailure]:+ """One failure per uuid, the first: a later step's failure on a uuid+ an earlier step already failed says nothing more to act on."""+ first: dict[str, MemoryForgetFailure] = {}+ for failure in failures:+ first.setdefault(failure.uuid, failure)+ return list(first.values())++async def _existing_edges(driver: GraphDriver,group_id: str,@@ -122,8 +266,11 @@ async def _retract_edges(reason: str,now: str,result: ForgetResult,+ *,+ hard: bool = False,) -> list[str]:- """Mark each edge forgotten and retracted; the uuids that were.+ """Mark each edge forgotten and retracted (``hard``: by a hard forget);+ the uuids that were.One query per uuid, so one bad edge cannot hide what happened to theothers (SECRT-2371); failures are recorded on ``result``.@@ -139,6 +286,7 @@ async def _retract_edges(now=now,status=MemoryStatus.retracted.value,reason=reason,+ hard=hard,))except Exception as exc:@@ -170,12 +318,16 @@ RETURN DISTINCT e.uuid AS uuid# ``coalesce`` keeps the first forget's ``forgotten_at`` and the first# retirement time on an edge that already had one (a graphiti expiry, a# dream demotion or an earlier forget), so a repeat is harmless.+# ``hard_forgotten_at`` says a hard forget is purging the fact._RETRACT_EDGE_QUERY = """MATCH ()-[e:MENTIONS|RELATES_TO|HAS_MEMBER {uuid: $uuid}]->()WHERE e.group_id = $group_id OR e.group_id IS NULLSET e.forgotten_at = coalesce(e.forgotten_at, $now),e.expired_at = coalesce(e.expired_at, $now),e.status = $status,- e.expiration_reason = $reason+ e.expiration_reason = $reason,+ e.hard_forgotten_at = CASE WHEN $hard+ THEN coalesce(e.hard_forgotten_at, $now)+ ELSE e.hard_forgotten_at ENDRETURN e.uuid AS uuid"""diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_cascade_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_cascade_test.pynew file mode 100644index 0000000000..515b84d000--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_cascade_test.py@@ -0,0 +1,137 @@+"""Unit tests for the cascade a forget runs (``recall_forget.retract``):+on what it retracted, erasing for a hard forget, after a failed hide too,+and resumed from a purged root something still names, or failed as a query+error, the rest of the forget going on, when what names it cannot be read.+The cascade itself is pinned in ``recall_cascade_test.py``; the live runs+are ``recall_cascade_integration_test.py`` and+``recall_cascade_resume_integration_test.py``.+"""++from unittest.mock import AsyncMock, patch++import pytest++from . import recall_forget+from .memory_model import MemoryForgetFailure, MemoryForgetFailureCode+from .recall_forget_fake import CLEANUP, SOFT, forget, own_call, scripted+++class TestCascade:+ """The forget hands the facts it retracted and hid to+ ``recall_cascade.cascade`` (pinned in ``recall_cascade_test.py``)."""++ @pytest.mark.asyncio+ @pytest.mark.parametrize("hard", [False, True], ids=["soft", "hard"])+ async def test_it_runs_on_what_was_retracted_and_its_count_is_returned(+ self, hard: bool+ ) -> None:+ """A hard forget's cascade erases the text it reaches as well."""+ seen: list[tuple[list[str], bool]] = []++ async def cascade(driver, group_id, roots, now, result, *, erase) -> None:+ seen.append((roots, erase))+ result.derived.extend(["d1", "d2"])++ purged = ([], [], [{"uuid": "u1", "deleted_entities": []}]) if hard else ()+ driver = scripted(*SOFT, *purged)++ with patch.object(recall_forget, "cascade", cascade):+ result = await forget(driver, ["u1"], hard=hard)++ assert seen == [(["u1"], hard)]+ assert (result.deleted, result.derived) == (["u1"], ["d1", "d2"])++ @pytest.mark.asyncio+ @pytest.mark.parametrize("cascade_fails", [False, True])+ async def test_a_failed_hide_still_retracts_what_the_dream_derived(+ self, cascade_fails: bool+ ) -> None:+ """Sentry's finding: the cascade waited for the hide, so a hide that+ failed left the dream's copies of the fact in recall until the forget+ was repeated. It runs regardless (it redacts the fact's episodes+ itself first), and the fact fails once, however many steps did."""+ driver = scripted([{"uuid": "u1"}], [{"uuid": "u1"}], RuntimeError("down"))++ async def cascade(driver, group_id, roots, now, result, *, erase) -> None:+ result.derived.append("d1")+ if cascade_fails:+ result.failures.append(+ MemoryForgetFailure.cleanup_error("u1", RuntimeError("down"))+ )++ with patch.object(recall_forget, "cascade", cascade):+ result = await forget(driver, ["u1"])++ assert (result.deleted, result.derived) == (["u1"], ["d1"])+ assert [(f.uuid, f.code) for f in result.failures] == [("u1", CLEANUP)]++ @pytest.mark.asyncio+ async def test_a_hard_forget_whose_hide_failed_cascades_but_purges_nothing(+ self,+ ) -> None:+ """Its cascade erases what the dream derived; the fact's own purge+ waits for a hide that landed, so the next forget finishes it."""+ driver = scripted([{"uuid": "u1"}], [{"uuid": "u1"}], RuntimeError("down"))+ seen: list[bool] = []++ async def cascade(driver, group_id, roots, now, result, *, erase) -> None:+ seen.append(erase)++ purge = AsyncMock()+ with (+ patch.object(recall_forget, "cascade", cascade),+ patch.object(recall_forget, "purge", purge),+ ):+ result = await forget(driver, ["u1"], hard=True)++ assert seen == [True]+ purge.assert_not_awaited()+ assert [(f.uuid, f.code) for f in result.failures] == [("u1", CLEANUP)]++ @pytest.mark.asyncio+ async def test_a_failed_look_for_purged_facts_fails_only_those(self) -> None:+ """Sentry's finding: the read for purged roots raised out of the+ forget, a raw 500 on the settings page and the uuids it found left+ unforgotten. Now a uuid that matched no edge fails as a query error,+ which a retry may clear, and the rest is forgotten."""+ driver = scripted(SOFT[0], RuntimeError("down"), *SOFT[1:])++ with patch.object(recall_forget, "cascade", AsyncMock()):+ result = await forget(driver, ["u1", "gone"])++ assert (result.deleted, result.resumed) == (["u1"], [])+ assert [(f.uuid, f.code) for f in result.failures] == [+ ("gone", MemoryForgetFailureCode.QUERY_ERROR)+ ]++ @pytest.mark.asyncio+ async def test_a_purged_root_something_names_resumes_its_cascade_erasing(+ self,+ ) -> None:+ """``gone`` was purged by a hard forget whose cascade stopped short; a+ record still names it. ``never`` names nothing and stays no match."""+ seen: list[tuple[list[str], bool]] = []++ async def cascade(driver, group_id, roots, now, result, *, erase) -> None:+ seen.append((roots, erase))+ result.derived.append("d1")++ driver = scripted([], [{"uuid": "gone"}]) # the lookup, the names++ with patch.object(recall_forget, "cascade", cascade):+ result = await forget(driver, ["gone", "never"])++ assert seen == [(["gone"], True)], "the root is gone: it was hard"+ assert (result.resumed, result.derived, result.deleted) == (+ ["gone"],+ ["d1"],+ [],+ )+ assert [(f.uuid, f.code) for f in result.failures] == [+ ("never", MemoryForgetFailureCode.NO_MATCH)+ ]+ _, names = own_call(driver, 1)+ assert names == {+ "uuids": ["gone", "never"],+ "prefix": "derived_from_forgotten:",+ }diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_copies_integration_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_copies_integration_test.pynew file mode 100644index 0000000000..e6145ec0ad--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_copies_integration_test.py@@ -0,0 +1,719 @@+"""A hard forget leaves none of the dream's copies of what it forgot in+Postgres (``dream/forget_copies.py``), end to end: production ingestion,+apply and forget against a live FalkorDB and a real Postgres, with canned+model answers.++These are the reproductions of independent reviews of PR 13. Round 1+found the forgotten sentence kept by a pass still open when the forget+landed, by a narrative a cancelled apply wrote after it, by a failed pass's+error, and by a pass behind a thousand newer ones; round 2 by a pass whose+insert failed, by a summary restating a fact no write cited, and by a+narrative a failed blank left for a repeat that found no pass; round 3 by+a pass from before read sets were recorded, its narrative written before+the deploy or after it, its row kept or aged out. Each now ends with the+sentence gone, as does a narrative whose pass's row aged out and the pass+of a dream write forgotten itself, while a forget of what a pass with a+known read set never read leaves it alone. A soft forget blanks+nothing in Postgres, but a write it dropped still keeps the narrative out+(``dream/apply.py``).++Run with FalkorDB reachable (see ``conftest.py``) and ``DATABASE_URL`` on a+migrated database::++ poetry run pytest -m integration backend/copilot/graphiti/recall_forget_copies_integration_test.py+"""++import uuid+from collections.abc import AsyncIterator+from datetime import datetime, timezone+from unittest.mock import AsyncMock++import pytest+import pytest_asyncio+from prisma import Prisma+from prisma.enums import (+ DreamPassPhase,+ DreamPassRoute,+ DreamPassStatus,+ DreamPassTrigger,+)+from prisma.models import ChatMessage, ChatSession, DreamPass, Expert, User+from pytest_mock import MockerFixture++from backend.copilot import db as chat+from backend.copilot.dream import apply, forget_copies, store+from backend.copilot.dream.fetch import DreamInput+from backend.copilot.dream.pass_errors import UNRECORDED, pass_error+from backend.copilot.dream.pass_record import cancelled+from backend.copilot.dream.pass_run import DreamPassRun+from backend.copilot.dream.schemas import (+ ConsolidatedFact,+ ConsolidationOutput,+ DreamOperations,+ DreamPassResult,+ ProposedFinding,+)+from backend.data import db, dream_pass+from backend.data.dream_pass_models import (+ UNKNOWN_READS,+ DreamPassDraft,+ DreamPassUpdate,+)+from backend.util.json import SafeJson++from . import recall_forget+from .recall import FORGOTTEN_FACT+from .recall_cascade_fixtures import CAFE, FLOUR, SUPPLIES, dream, fact_uuid, gather+from .recall_integration_fixtures import (+ edge_row,+ ingest_facts,+ live_facts,+ patch_recall_boundaries,+ stop_ingestion_workers,+)+from .scope import MemoryScope++pytestmark = [pytest.mark.integration, pytest.mark.asyncio(loop_scope="function")]++_DERIVED = SUPPLIES[2]+++@pytest_asyncio.fixture(loop_scope="function", autouse=True)+async def postgres(+ mocker: MockerFixture, scope_graph, stub_graphiti_client+) -> AsyncIterator[None]:+ """The scope's owner in a real Postgres, and the dream's store, its+ narrative and the forget's blanking on it; the graph as ``conftest.py``+ gives it.++ Each test runs on a loop of its own, for its FalkorDB driver, so it gets+ a Prisma client of its own, connected on that loop and registered for+ the test: the session's shared client may already be connected on the+ session's loop by an earlier test, and its HTTP pool works only on the+ loop that opened it."""+ _, scope = scope_graph+ client = Prisma(+ http={"timeout": db.HTTP_TIMEOUT}, datasource={"url": db.DATABASE_URL}+ )+ await client.connect()+ try:+ mocker.patch.object(db, "prisma", client)+ mocker.patch("prisma._registry._registered_client", client)+ await User.prisma().create(+ data={+ "id": scope.owner_user_id,+ "email": f"{uuid.uuid4()}@example.invalid",+ "topUpConfig": SafeJson({}),+ "timezone": "UTC",+ }+ )+ patch_recall_boundaries(mocker, scope_graph[0], stub_graphiti_client)+ mocker.patch.object(+ recall_forget, "blank_dream_copies", forget_copies.blank_dream_copies+ )+ mocker.patch.object(forget_copies, "chat_db", lambda: chat)+ mocker.patch.object(store, "dream_db", lambda: dream_pass)+ mocker.patch("backend.data.db_accessors.chat_db", lambda: chat)+ mocker.patch(+ "backend.copilot.dream.registry.ensure_dream_system_scheduled",+ AsyncMock(return_value=None),+ )+ mocker.patch.object(apply, "_create_dream_session", _dream_session)+ yield+ await stop_ingestion_workers()+ await User.prisma().delete(where={"id": scope.owner_user_id})+ finally:+ await client.disconnect(db.DISCONNECT_TIMEOUT)+++async def _dream_session(scope: MemoryScope, pass_id: str) -> str:+ session = await ChatSession.prisma().create(+ data={+ "userId": scope.owner_user_id,+ "metadata": SafeJson({"kind": "dream", "dream_pass_id": pass_id}),+ }+ )+ return session.id+++async def _open_pass(scope: MemoryScope) -> str:+ pass_id = str(uuid.uuid4())+ await dream_pass.create_dream_pass(+ DreamPassDraft(+ id=pass_id,+ user_id=scope.owner_user_id,+ scope_key=scope.scope_key,+ route=DreamPassRoute.ANTHROPIC_BATCH,+ trigger=DreamPassTrigger.CRON,+ status=DreamPassStatus.QUEUED,+ )+ )+ return pass_id+++async def _source(scope_graph, build) -> str:+ driver, scope = scope_graph+ _, facts = await ingest_facts(+ driver, scope, build, [FLOUR], session_id="source-session"+ )+ return facts[FLOUR[2]]+++async def _kept(pass_id: str, session_id: str | None) -> list[str]:+ """Where the pass's row or its narrative still holds the derived+ sentence."""+ row = await dream_pass.get_dream_pass(pass_id)+ assert row is not None+ kept = [+ name+ for name, text in (+ ("phase outputs", row.phase_outputs.model_dump_json()),+ ("operations", row.operations.model_dump_json()),+ ("error", row.error or ""),+ )+ if _DERIVED in text+ ]+ if session_id is not None:+ message = await ChatMessage.prisma().find_first(+ where={"sessionId": session_id, "sequence": 0}+ )+ if message is not None and _DERIVED in (message.content or ""):+ kept.append("narrative")+ return kept+++def _completed(scope: MemoryScope, pass_id: str, stats: dict, ops) -> DreamPassResult:+ now = datetime.now(timezone.utc)+ return DreamPassResult(+ user_id=scope.owner_user_id,+ pass_id=pass_id,+ started_at=now,+ completed_at=now,+ execution_path="anthropic_batch",+ dropped_forgotten=stats["dropped_forgotten"],+ summary_for_user=ops.summary_for_user,+ dream_session_id=stats["session_id"],+ )+++def _derived_ops(root: str, operation: str) -> tuple[DreamOperations, ConsolidatedFact]:+ write = ConsolidatedFact(content=_DERIVED, confidence=0.9, source_fact_uuids=[root])+ if operation == "propose":+ proposal = ProposedFinding(+ content=_DERIVED,+ rationale=FLOUR[2],+ confidence=0.9,+ source_fact_uuids=[root],+ )+ return DreamOperations(proposals=[proposal], summary_for_user=_DERIVED), write+ return DreamOperations(writes=[write], summary_for_user=_DERIVED), write++[email protected]("operation", ["consolidate", "propose"])+async def test_an_open_pass_that_read_a_hard_forgotten_fact_keeps_none_of_it(+ scope_graph, stub_graphiti_client, operation+):+ """The review's first reproduction: a pass gathers the source, a hard+ forget deletes it while the pass is open, and the pass goes on to record+ its phases, apply (its write is dropped) and complete. The forget+ reports nothing left to do, and the closed pass keeps none of the+ sentence."""+ driver, scope = scope_graph+ root = await _source(scope_graph, stub_graphiti_client)+ pass_id = await _open_pass(scope)+ read = await gather(scope)+ ops, write = _derived_ops(root, operation)++ forgot = await recall_forget.retract(scope, [root], hard=True)++ assert (forgot.deleted, forgot.failures) == ([root], [])+ assert await edge_row(driver, root) == {}+ await store.record_phase_output(+ pass_id, "consolidate", ConsolidationOutput(facts=[write])+ )+ await store.record_applying(pass_id, ops)+ stats = await dream(+ driver, scope, stub_graphiti_client, ops, read, SUPPLIES, pass_id+ )+ assert stats["dropped_forgotten"] == 1+ assert await live_facts(driver) == {}+ await store.record_batch_complete(_completed(scope, pass_id, stats, ops), None)+ await store.record_cleanup_finished(pass_id)++ row = await dream_pass.get_dream_pass(pass_id)+ assert row is not None and row.status == DreamPassStatus.COMPLETE+ assert await _kept(pass_id, stats["session_id"]) == []+++async def test_a_soft_forget_keeps_a_dropped_write_out_of_the_narrative(+ scope_graph, stub_graphiti_client+):+ """A soft forget marks no pass, but the write it made the worker drop+ may be restated anywhere in the summary: the narrative is the+ placeholder."""+ driver, scope = scope_graph+ root = await _source(scope_graph, stub_graphiti_client)+ pass_id = await _open_pass(scope)+ read = await gather(scope)+ ops, _ = _derived_ops(root, "consolidate")+ await recall_forget.retract(scope, [root])++ stats = await dream(+ driver, scope, stub_graphiti_client, ops, read, SUPPLIES, pass_id+ )++ assert stats["dropped_forgotten"] == 1+ message = await ChatMessage.prisma().find_first(+ where={"sessionId": stats["session_id"], "sequence": 0}+ )+ assert message is not None and message.content == FORGOTTEN_FACT+ row = await dream_pass.get_dream_pass(pass_id)+ assert row is not None and row.erased_at is None, "a soft forget marks nothing"+++async def test_a_cancelled_apply_writes_no_narrative_after_the_forget(+ scope_graph, stub_graphiti_client, mocker: MockerFixture+):+ """The review's race: apply's drain finishes (the write is made), the+ pass is cancelled, a hard forget erases the write and finds no+ narrative yet, then apply resumes and writes it. The narrative goes in+ blank: the forget marked the pass."""+ driver, scope = scope_graph+ root = await _source(scope_graph, stub_graphiti_client)+ pass_id = await _open_pass(scope)+ read = await gather(scope)+ ops, _ = _derived_ops(root, "consolidate")+ await store.record_applying(pass_id, ops)+ drain = apply._drain_ingestion+ forgot = []++ async def close_then_forget(*args, **kwargs):+ status = await drain(*args, **kwargs)+ assert await store.write_stop(+ pass_id,+ cancelled(+ pass_error("cancelled", "by_admin"),+ owner_user_id=scope.owner_user_id,+ ),+ )+ forgot.append(await recall_forget.retract(scope, [root], hard=True))+ assert (+ await ChatMessage.prisma().count(+ where={+ "Session": {"is": {"userId": scope.owner_user_id}},+ "sequence": 0,+ }+ )+ == 0+ ), "no narrative yet"+ return status++ mocker.patch.object(apply, "_drain_ingestion", close_then_forget)+ stats = await dream(+ driver, scope, stub_graphiti_client, ops, read, SUPPLIES, pass_id+ )++ assert forgot[0].failures == []+ assert stats["dropped_forgotten"] == 0, "the write was made, then erased"+ row = await dream_pass.get_dream_pass(pass_id)+ assert row is not None and row.status == DreamPassStatus.CANCELLED+ assert await live_facts(driver) == {}+ assert await _kept(pass_id, stats["session_id"]) == []+++async def test_a_failed_passes_error_never_quotes_the_model(+ scope_graph, stub_graphiti_client+):+ """The review's error column: a pass fails with an error quoting the+ model's answer. Only a fixed code is recorded, and the hard forget finds+ the closed pass by what it read and blanks the rest."""+ _, scope = scope_graph+ root = await _source(scope_graph, stub_graphiti_client)+ pass_id, _ = await _gathered_pass(scope)+ write = ConsolidatedFact(content=_DERIVED, confidence=0.9, source_fact_uuids=[root])+ await store.record_phase_output(+ pass_id, "consolidate", ConsolidationOutput(facts=[write])+ )+ quoted = f"recombine: LLM returned non-JSON content — first 200 chars: {_DERIVED}"+ await store.record_sync_outcome(+ DreamPassResult(user_id=scope.owner_user_id, pass_id=pass_id, error=quoted)+ )+ row = await dream_pass.get_dream_pass(pass_id)+ assert row is not None and row.error == UNRECORDED++ forgot = await recall_forget.retract(scope, [root], hard=True)++ assert (forgot.deleted, forgot.failures) == ([root], [])+ assert await _kept(pass_id, None) == []+++async def test_an_affected_pass_behind_a_thousand_newer_ones_is_blanked(+ scope_graph, stub_graphiti_client+):+ """The review's 1,001 passes: the affected one is the oldest of the+ scope, its narrative named by no erased write."""+ _, scope = scope_graph+ root = await _source(scope_graph, stub_graphiti_client)+ old, _ = await _gathered_pass(scope)+ write = ConsolidatedFact(content=_DERIVED, confidence=0.9, source_fact_uuids=[root])+ await store.record_phase_output(+ old, "consolidate", ConsolidationOutput(facts=[write])+ )+ await store.record_sync_outcome(+ DreamPassResult(user_id=scope.owner_user_id, pass_id=old)+ )+ await DreamPass.prisma().create_many(+ data=[+ {+ "id": str(uuid.uuid4()),+ "userId": scope.owner_user_id,+ "scopeKey": scope.scope_key,+ "route": DreamPassRoute.SYNC,+ "trigger": DreamPassTrigger.ADMIN,+ "status": DreamPassStatus.SKIPPED,+ "phase": DreamPassPhase.DONE,+ "readUuids": [],+ }+ for _ in range(1000)+ ]+ )+ session_id = await _dream_session(scope, old)+ await chat.add_chat_message(session_id, "assistant", 0, _DERIVED)++ forgot = await recall_forget.retract(scope, [root], hard=True)++ assert forgot.failures == []+ assert await _kept(old, session_id) == []+++async def _gathered_pass(scope: MemoryScope) -> tuple[str, DreamInput]:+ """An open pass that gathered the scope and recorded what it read, as+ every pass does before its first model call."""+ pass_id = await _open_pass(scope)+ read = await gather(scope)+ assert await store.record_gathered(pass_id, read)+ return pass_id, read+++async def _cafe_pass(+ scope_graph, build+) -> tuple[str, str, DreamOperations, DreamInput]:+ """The source, an unrelated cafe fact, and an open pass that read both+ and is about to write a consolidation of the cafe (citing it) with a+ summary restating the source, citing nothing: the source's uuid, the+ pass id, its operations and what it read."""+ driver, scope = scope_graph+ root = await _source(scope_graph, build)+ _, other = await ingest_facts(+ driver, scope, build, [CAFE], session_id="other-source"+ )+ pass_id, read = await _gathered_pass(scope)+ assert {root, other[CAFE[2]]} <= read.known_fact_uuids+ write = ConsolidatedFact(+ content=CAFE[2], confidence=0.9, source_fact_uuids=[other[CAFE[2]]]+ )+ await store.record_phase_output(+ pass_id, "consolidate", ConsolidationOutput(facts=[write])+ )+ return (+ root,+ pass_id,+ DreamOperations(writes=[write], summary_for_user=_DERIVED),+ read,+ )+++async def _complete(scope: MemoryScope, pass_id: str, stats: dict, ops) -> None:+ await store.record_batch_complete(_completed(scope, pass_id, stats, ops), None)+ assert await store.record_cleanup_finished(pass_id)+++async def test_a_pass_whose_record_did_not_go_in_writes_no_narrative(+ scope_graph, stub_graphiti_client, mocker: MockerFixture+):+ """The round-2 review's first reproduction: the pass's insert fails. The+ pass stops before it gathers (``orchestrator``); should its apply run+ anyway, as the reproduction makes it, after a hard forget erased its+ write, the narrative has no row to go by and is not written."""+ driver, scope = scope_graph+ root = await _source(scope_graph, stub_graphiti_client)+ run = DreamPassRun.begin(scope.owner_user_id, "sync_baseline")+ mocker.patch.object(+ dream_pass, "create_dream_pass", AsyncMock(side_effect=OSError("insert"))+ )+ assert not await store.start_pass(run, scope, trigger="cron")+ assert await dream_pass.get_dream_pass(run.pass_id) is None+ read = await gather(scope)+ ops, _ = _derived_ops(root, "consolidate")+ drain = apply._drain_ingestion+ forgot = []++ async def forget_after_the_write(*args, **kwargs):+ status = await drain(*args, **kwargs)+ forgot.append(await recall_forget.retract(scope, [root], hard=True))+ return status++ mocker.patch.object(apply, "_drain_ingestion", forget_after_the_write)+ stats = await dream(+ driver, scope, stub_graphiti_client, ops, read, SUPPLIES, run.pass_id+ )++ assert forgot[0].failures == [] and forgot[0].dream_writes+ assert await live_facts(driver) == {}+ assert (+ await ChatMessage.prisma().count(where={"sessionId": stats["session_id"]}) == 0+ )+++async def test_a_summary_restating_a_fact_it_did_not_cite_is_blanked(+ scope_graph, stub_graphiti_client+):+ """The round-2 review's second reproduction: the pass's summary restates+ the source, which no write cites. Completed and cleaned up, the pass is+ found by what it read, and the forget blanks its operations and its+ narrative."""+ driver, scope = scope_graph+ root, pass_id, ops, read = await _cafe_pass(scope_graph, stub_graphiti_client)+ await store.record_applying(pass_id, ops)+ stats = await dream(driver, scope, stub_graphiti_client, ops, read, CAFE, pass_id)+ await _complete(scope, pass_id, stats, ops)+ assert await _kept(pass_id, stats["session_id"]) == ["operations", "narrative"]++ forgot = await recall_forget.retract(scope, [root], hard=True)++ assert forgot.failures == []+ row = await dream_pass.get_dream_pass(pass_id)+ assert row is not None and row.erased_at is not None+ assert await _kept(pass_id, stats["session_id"]) == []+++async def test_a_forget_repeated_after_its_narrative_blank_failed_finishes_it(+ scope_graph, stub_graphiti_client, mocker: MockerFixture+):+ """The round-2 review's third reproduction: the forget marks the open+ pass but cannot blank its narrative (a cleanup error, to repeat); the+ pass then completes, its row blanked as it closes. Repeated, the forget+ finds the pass by its mark and blanks the narrative."""+ driver, scope = scope_graph+ root, pass_id, ops, read = await _cafe_pass(scope_graph, stub_graphiti_client)+ await store.record_applying(pass_id, ops)+ stats = await dream(driver, scope, stub_graphiti_client, ops, read, CAFE, pass_id)+ blank = chat.blank_dream_narratives+ mocker.patch.object(+ chat, "blank_dream_narratives", AsyncMock(side_effect=OSError("down"))+ )+ first = await recall_forget.retract(scope, [root], hard=True)+ assert [failure.code for failure in first.failures] == ["cleanup_error"]+ mocker.patch.object(chat, "blank_dream_narratives", blank)+ await _complete(scope, pass_id, stats, ops)+ assert await _kept(pass_id, stats["session_id"]) == ["narrative"]++ again = await recall_forget.retract(scope, [root], hard=True)++ assert again.failures == []+ assert await _kept(pass_id, stats["session_id"]) == []+++async def test_a_narrative_whose_pass_row_aged_out_is_found_by_what_it_read(+ scope_graph, stub_graphiti_client+):+ """The pass's row is deleted by retention after it completed; its+ narrative's session kept what the pass read, so a hard forget of the+ source, which names no write of the pass, still blanks it."""+ driver, scope = scope_graph+ root, pass_id, ops, read = await _cafe_pass(scope_graph, stub_graphiti_client)+ stats = await dream(driver, scope, stub_graphiti_client, ops, read, CAFE, pass_id)+ await _complete(scope, pass_id, stats, ops)+ await DreamPass.prisma().delete(where={"id": pass_id})++ forgot = await recall_forget.retract(scope, [root], hard=True)++ assert forgot.failures == []+ message = await ChatMessage.prisma().find_first(+ where={"sessionId": stats["session_id"], "sequence": 0}+ )+ assert message is not None and message.content == FORGOTTEN_FACT+++async def _pass_before_read_sets(+ scope_graph, build+) -> tuple[str, str, DreamOperations, DreamInput]:+ """``_cafe_pass``'s pass, written as a release before read sets wrote+ one (the round-3 review's legacy pass): its create states no read set,+ so it reads the column's default, as every row that existed when the+ column was added does, and its gather records the window only."""+ driver, scope = scope_graph+ root = await _source(scope_graph, build)+ _, other = await ingest_facts(+ driver, scope, build, [CAFE], session_id="other-source"+ )+ pass_id = str(uuid.uuid4())+ await DreamPass.prisma().create(+ data={+ "id": pass_id,+ "userId": scope.owner_user_id,+ "scopeKey": scope.scope_key,+ "route": DreamPassRoute.ANTHROPIC_BATCH,+ "trigger": DreamPassTrigger.CRON,+ "status": DreamPassStatus.QUEUED,+ }+ )+ read = await gather(scope)+ assert {root, other[CAFE[2]]} <= read.known_fact_uuids+ assert await dream_pass.update_dream_pass(+ pass_id,+ DreamPassUpdate(window_start=read.window_start, window_end=read.window_end),+ )+ write = ConsolidatedFact(+ content=CAFE[2], confidence=0.9, source_fact_uuids=[other[CAFE[2]]]+ )+ await store.record_phase_output(+ pass_id, "consolidate", ConsolidationOutput(facts=[write])+ )+ row = await dream_pass.get_dream_pass(pass_id)+ assert row is not None and row.read_uuids == [UNKNOWN_READS]+ return (+ root,+ pass_id,+ DreamOperations(writes=[write], summary_for_user=_DERIVED),+ read,+ )+++async def _narrative_before_read_sets(+ session_id: str, pass_id: str, content: str, erased: str+) -> bool:+ """The narrative as the release before read sets wrote it: the message+ alone, nothing copied into its session."""+ await chat.add_chat_message(session_id, "assistant", 0, content)+ return True+++async def _narrative(session_id: str) -> str | None:+ message = await ChatMessage.prisma().find_first(+ where={"sessionId": session_id, "sequence": 0}+ )+ return message.content if message is not None else None++[email protected]("row", ["kept", "aged_out"])[email protected]("narrative", ["before_the_deploy", "after_the_deploy"])+async def test_a_pass_from_before_read_sets_keeps_none_of_a_hard_forgotten_fact(+ scope_graph, stub_graphiti_client, mocker: MockerFixture, narrative, row+):+ """The round-3 review's two legacy reproductions, turned around. A pass+ from before read sets were recorded gathered the source and an+ unrelated cafe fact, wrote the cafe citing it and a summary restating+ the source, citing nothing. Its narrative was written by the release+ before (the message alone) or by this one, the pass in flight across+ the deploy (its session takes the pass's unknown read set); its row is+ kept or has aged out. What the pass read is unknown, so a hard forget of+ the source, which no write of the pass cites, blanks the narrative and+ the row."""+ driver, scope = scope_graph+ root, pass_id, ops, read = await _pass_before_read_sets(+ scope_graph, stub_graphiti_client+ )+ if narrative == "before_the_deploy":+ mocker.patch.object(chat, "add_dream_narrative", _narrative_before_read_sets)+ await store.record_applying(pass_id, ops)+ stats = await dream(driver, scope, stub_graphiti_client, ops, read, CAFE, pass_id)+ await _complete(scope, pass_id, stats, ops)+ assert await _kept(pass_id, stats["session_id"]) == ["operations", "narrative"]+ if row == "aged_out":+ await DreamPass.prisma().delete(where={"id": pass_id})++ forgot = await recall_forget.retract(scope, [root], hard=True)++ assert (forgot.deleted, forgot.failures) == ([root], [])+ assert await _narrative(stats["session_id"]) == FORGOTTEN_FACT+ if row == "kept":+ kept = await dream_pass.get_dream_pass(pass_id)+ assert kept is not None and kept.erased_at is not None+ assert await _kept(pass_id, stats["session_id"]) == []+++async def _old_narrative(user_id: str, expert_id: str | None) -> str:+ """A dream session of the user's scope of *expert_id* (the account's when+ ``None``) with a narrative written before sessions kept a read set."""+ session = await ChatSession.prisma().create(+ data={+ "userId": user_id,+ "metadata": SafeJson({"kind": "dream", "dream_pass_id": "p-before"}),+ **({"expertId": expert_id} if expert_id else {}),+ }+ )+ await chat.add_chat_message(session.id, "assistant", 0, _DERIVED)+ return session.id+++async def test_an_unrelated_forget_leaves_a_new_narrative_and_takes_old_ones(+ scope_graph, stub_graphiti_client+):+ """A pass with a known read set is reached only by what it read: a hard+ forget of a fact ingested after it gathered leaves its row and its+ narrative. The same forget blanks a narrative of its scope from before+ sessions kept a read set, and not one of the user's expert's scope."""+ driver, scope = scope_graph+ _, cafe = await ingest_facts(+ driver, scope, stub_graphiti_client, [CAFE], session_id="cafe-source"+ )+ pass_id, read = await _gathered_pass(scope)+ write = ConsolidatedFact(+ content=CAFE[2], confidence=0.9, source_fact_uuids=[cafe[CAFE[2]]]+ )+ ops = DreamOperations(writes=[write], summary_for_user=_DERIVED)+ await store.record_phase_output(+ pass_id, "consolidate", ConsolidationOutput(facts=[write])+ )+ await store.record_applying(pass_id, ops)+ stats = await dream(driver, scope, stub_graphiti_client, ops, read, CAFE, pass_id)+ await _complete(scope, pass_id, stats, ops)+ root = await _source(scope_graph, stub_graphiti_client)+ expert = await Expert.prisma().create(+ data={+ "ownerUserId": scope.owner_user_id,+ "name": "Scout",+ "role": "Research",+ "identity": "Finds things out.",+ }+ )+ old_here = await _old_narrative(scope.owner_user_id, None)+ old_there = await _old_narrative(scope.owner_user_id, expert.id)++ forgot = await recall_forget.retract(scope, [root], hard=True)++ assert (forgot.deleted, forgot.failures) == ([root], [])+ row = await dream_pass.get_dream_pass(pass_id)+ assert row is not None and row.erased_at is None+ assert root not in row.read_uuids and UNKNOWN_READS not in row.read_uuids+ assert await _kept(pass_id, stats["session_id"]) == ["operations", "narrative"]+ assert await _narrative(old_here) == FORGOTTEN_FACT+ assert await _narrative(old_there) == _DERIVED+++async def test_forgetting_a_dream_write_blanks_the_pass_that_wrote_it(+ scope_graph, stub_graphiti_client+):+ """A pass does not read what it writes: forgetting the dream's own write+ reaches the pass through the write's episode, which names it."""+ driver, scope = scope_graph+ root = await _source(scope_graph, stub_graphiti_client)+ pass_id, read = await _gathered_pass(scope)+ ops, write = _derived_ops(root, "consolidate")+ await store.record_phase_output(+ pass_id, "consolidate", ConsolidationOutput(facts=[write])+ )+ await store.record_applying(pass_id, ops)+ stats = await dream(+ driver, scope, stub_graphiti_client, ops, read, SUPPLIES, pass_id+ )+ await _complete(scope, pass_id, stats, ops)+ derived = await fact_uuid(driver, _DERIVED)+ row = await dream_pass.get_dream_pass(pass_id)+ assert row is not None and derived not in row.read_uuids++ forgot = await recall_forget.retract(scope, [derived], hard=True)++ assert forgot.failures == []+ assert await _kept(pass_id, stats["session_id"]) == []diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_copies_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_copies_test.pynew file mode 100644index 0000000000..f0468a8c95--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_copies_test.py@@ -0,0 +1,95 @@+"""Unit tests for the last step of a hard forget in ``recall_forget.retract``:+blanking the dream's copies of what it reached in Postgres+(``dream/forget_copies.py``), against a mock driver and the conftest's+``dream_copies`` stand-in for the blanking itself.++What the blanking does to a pass's row and narrative is+``dream/forget_copies_test.py``.+"""++from unittest.mock import AsyncMock++import pytest++from .memory_model import MemoryForgetFailureCode+from .recall_forget_fake import CLEANUP, NOTHING_DERIVED, SCOPE, SOFT, forget, scripted++_DREAM_WRITE = "dream_p1_consolidate_001"+++def _hard_forget_driver() -> AsyncMock:+ """A hard forget of ``u1``: two episodes cite it, one of them a dream+ write the cascade erases, and nothing is derived from it."""+ redacted = [{"uuid": "ep1", "via": ["u1"]}, {"uuid": "ep2", "via": ["u1"]}]+ return scripted(+ *SOFT[:4],+ redacted, # redact+ [], # the cascade: no earlier try,+ redacted, # the episodes citing the fact,+ [{"name": _DREAM_WRITE}], # the dream's among them erased,+ [], # nothing derived+ [],+ [{"uuid": "ep1", "content": "Alice works on Atlas"}], # citing+ [{"uuid": "ep1"}], # tombstone+ [{"uuid": "u1", "deleted_entities": ["alice"]}], # delete+ )+++async def test_a_hard_forget_blanks_the_dreams_copies_of_what_it_reached(+ dream_copies: AsyncMock,+) -> None:+ result = await forget(_hard_forget_driver(), ["u1"], hard=True)++ assert (result.deleted, result.failures) == (["u1"], [])+ assert result.dream_writes == [_DREAM_WRITE]+ dream_copies.assert_awaited_once()+ assert dream_copies.await_args is not None+ (scope,) = dream_copies.await_args.args+ kwargs = dream_copies.await_args.kwargs+ assert scope == SCOPE+ assert set(kwargs["reached"]) == {"u1", "ep1", "ep2"}+ assert list(kwargs["dream_writes"]) == [_DREAM_WRITE]+++async def test_a_soft_forget_leaves_them(dream_copies: AsyncMock) -> None:+ await forget(scripted(*SOFT, *NOTHING_DERIVED), ["u1"])++ dream_copies.assert_not_awaited()+++async def test_a_forget_that_found_nothing_leaves_them(+ dream_copies: AsyncMock,+) -> None:+ result = await forget(scripted([], [], [], [], [], []), ["u1"], hard=True)++ assert [f.code for f in result.failures] == [MemoryForgetFailureCode.NO_MATCH]+ dream_copies.assert_not_awaited()+++async def test_failing_to_blank_them_is_a_cleanup_error_on_each_fact(+ dream_copies: AsyncMock,+) -> None:+ """The fact is still purged and hidden; forgetting it again retries."""+ dream_copies.side_effect = ConnectionError("database down")++ result = await forget(_hard_forget_driver(), ["u1"], hard=True)++ assert result.deleted == ["u1"]+ assert [(f.uuid, f.code) for f in result.failures] == [("u1", CLEANUP)]+ assert "ConnectionError" in result.failures[0].reason+ assert "database down" not in result.failures[0].reason+++async def test_a_pass_still_running_is_no_failure(dream_copies: AsyncMock) -> None:+ """The pass is marked, and blanked in the statement that closes it+ (``dream/forget_copies.py``): nothing is left for the user to repeat."""+ dream_copies.return_value = ["p-running"]++ result = await forget(_hard_forget_driver(), ["u1"], hard=True)++ assert (result.deleted, result.failures) == (["u1"], [])diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_fake.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_fake.pynew file mode 100644index 0000000000..a84f986a05--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_fake.py@@ -0,0 +1,90 @@+"""The scripted driver the forget's unit tests run ``recall_forget.retract``+against, with the in-memory write lock (``conftest.lock_redis``): it+answers in order, the graph's two reads of dream markers apart. Shared by+``recall_forget_test.py``, ``recall_forget_writes_test.py`` and+``recall_forget_cascade_test.py``; not collected by pytest.+"""++from unittest.mock import AsyncMock, MagicMock, patch++from . import recall_forget, recall_reconcile+from .memory_model import ForgetResult, MemoryForgetFailureCode+from .scope import MemoryScope, write_lock_key++SCOPE = MemoryScope.for_user("user-abc")+LOCK = write_lock_key(SCOPE.group_id)+CLEANUP = MemoryForgetFailureCode.CLEANUP_ERROR+# The graph's two reads of dream markers, answered apart from the rest.+MARKER_READS = (recall_reconcile.MARKERS_QUERY, recall_reconcile.IN_FLIGHT_QUERY)+++def scripted(*results, markers: object = None, in_flight: object = 0) -> AsyncMock:+ """A driver whose queries return (or raise) ``results`` in order. The+ two reads of the graph's dream markers answer apart: the one every+ forget makes first ``markers`` (none by default), the one of the dream+ writes still in flight ``in_flight`` (none)."""+ driver = AsyncMock()+ answers = iter(results)++ async def answer(query: str, **params):+ if query == recall_reconcile.MARKERS_QUERY:+ value = [] if markers is None else markers+ elif query == recall_reconcile.IN_FLIGHT_QUERY:+ value = (+ in_flight+ if isinstance(in_flight, Exception)+ else [{"count": in_flight}]+ )+ else:+ value = next(answers)+ if isinstance(value, Exception):+ raise value+ return value, [], None++ driver.execute_query.side_effect = answer+ return driver+++async def forget(driver: AsyncMock, uuids: list[str], **kwargs) -> ForgetResult:+ with patch.object(recall_forget, "open_driver", MagicMock(return_value=driver)):+ return await recall_forget.retract(SCOPE, uuids, **kwargs)+++def own_calls(driver: AsyncMock) -> list:+ """The forget's own queries: every one but its reads of dream markers."""+ return [+ call+ for call in driver.execute_query.await_args_list+ if call.args[0] not in MARKER_READS+ ]+++def own_queries(driver: AsyncMock) -> list[str]:+ return [call.args[0] for call in own_calls(driver)]+++def own_call(driver: AsyncMock, index: int) -> tuple[str, dict]:+ """The ``index``-th of the forget's own queries."""+ call = own_calls(driver)[index]+ return call.args[0], call.kwargs+++def in_flight_reads(driver: AsyncMock) -> list[dict]:+ return [+ call.kwargs+ for call in driver.execute_query.await_args_list+ if call.args[0] == recall_reconcile.IN_FLIGHT_QUERY+ ]+++# lookup, retract, scrub facts, entity keys (none), redact+SOFT = (+ [{"uuid": "u1"}],+ [{"uuid": "u1"}],+ [{"ends": []}],+ [],+ [{"uuid": "ep1", "via": ["u1"]}],+)+# The cascade finding nothing derived: no earlier try, the episodes citing+# the forgotten fact, no derived fact, no derived dream episode.+NOTHING_DERIVED = ([], [{"uuid": "ep1", "via": ["u1"]}], [], [])diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_test.pyindex 4a7df16c38..8e34b815da 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_test.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_test.py@@ -2,18 +2,29 @@in-memory write lock (``conftest.lock_redis``).Pin the Cypher each forget mode issues, its order, the write lock it holds-around every write, and the per-uuid failure reporting. What hiding and-purging send is pinned in ``recall_hide_test.py`` and-``recall_orphans_test.py``; the live runs are+around every write, and the per-uuid failure reporting. What hiding, the+cascade and purging send is pinned in ``recall_hide_test.py``,+``recall_cascade_test.py`` and ``recall_orphans_test.py``, the forget's+side of the dream markers in ``recall_forget_writes_test.py`` and of the+cascade in ``recall_forget_cascade_test.py``; the live runs are``recall_forget_integration_test.py``, ``recall_hard_forget_integration_test.py``,-``recall_inflight_integration_test.py`` and ``recall_integration_test.py``.+``recall_cascade_integration_test.py``, ``recall_inflight_integration_test.py``+and ``recall_integration_test.py``."""from unittest.mock import AsyncMock, MagicMock, patchimport pytest-from . import recall_forget, recall_hide, recall_orphans, scope_lock+from . import (+ recall_cascade,+ recall_cascade_queries,+ recall_erase,+ recall_forget,+ recall_hide,+ recall_orphans,+ scope_lock,+)from .memory_model import (FORGET_BUSY_REASON,FORGET_NO_MATCH_REASON,@@ -21,93 +32,83 @@ from .memory_model import ()from .recall import FORGOTTEN_FACTfrom .recall_fake_redis import FakeRedis-from .scope import MemoryScope, write_lock_key--_SCOPE = MemoryScope.for_user("user-abc")-_LOCK = write_lock_key(_SCOPE.group_id)-_CLEANUP = MemoryForgetFailureCode.CLEANUP_ERROR---def _driver(*results) -> AsyncMock:- """A driver whose queries return (or raise) ``results`` in order."""- driver = AsyncMock()- driver.execute_query.side_effect = [- r if isinstance(r, Exception) else (r, [], None) for r in results- ]- return driver---async def _retract(driver: AsyncMock, uuids: list[str], **kwargs):- with patch.object(recall_forget, "open_driver", MagicMock(return_value=driver)):- return await recall_forget.retract(_SCOPE, uuids, **kwargs)---def _queries(driver: AsyncMock) -> list[str]:- return [call.args[0] for call in driver.execute_query.await_args_list]---def _call(driver: AsyncMock, index: int) -> tuple[str, dict]:- call = driver.execute_query.await_args_list[index]- return call.args[0], call.kwargs---# lookup, retract, scrub facts, entity keys (none), redact-_SOFT = ([{"uuid": "u1"}], [{"uuid": "u1"}], [{"ends": []}], [], [{"uuid": "ep1"}])+from .recall_forget_fake import (+ CLEANUP,+ LOCK,+ NOTHING_DERIVED,+ SCOPE,+ SOFT,+ forget,+ own_call,+ own_queries,+ scripted,+)class TestSoftRetract:@pytest.mark.asyncioasync def test_marks_the_edge_forgotten_then_hides_its_text(self) -> None:- driver = _driver(*_SOFT)+ driver = scripted(*SOFT, *NOTHING_DERIVED)- result = await _retract(driver, ["u1"])+ result = await forget(driver, ["u1"])assert (result.deleted, result.failures) == (["u1"], [])assert result.redacted_episodes == ["ep1"]driver.close.assert_awaited_once()- lookup, lookup_kwargs = _call(driver, 0)+ lookup, lookup_kwargs = own_call(driver, 0)assert "SET" not in lookup, "the lookup must be a read"- assert lookup_kwargs == {"uuids": ["u1"], "group_id": _SCOPE.group_id}- write, kwargs = _call(driver, 1)+ assert lookup_kwargs == {"uuids": ["u1"], "group_id": SCOPE.group_id}+ write, kwargs = own_call(driver, 1)assert "SET e.forgotten_at = coalesce(e.forgotten_at, $now)," in writeassert "e.expired_at = coalesce(e.expired_at, $now)," in writeassert "invalid_at" not in write, "a forget is not a world change"assert (kwargs["status"], kwargs["reason"]) == ("retracted", "user_signal")- assert _call(driver, 2) == (+ assert kwargs["hard"] is False, "no purge to come"+ assert own_call(driver, 2) == (recall_hide.SCRUB_FACTS_QUERY,{"uuids": ["u1"], "placeholder": FORGOTTEN_FACT},)- assert _call(driver, 4) == (+ assert own_call(driver, 4) == (recall_hide.REDACT_EPISODES_QUERY,{"uuids": ["u1"], "now": kwargs["now"]},)+ assert own_queries(driver)[5:] == [+ recall_cascade.EARLIER_QUERY,+ recall_hide.REDACT_EPISODES_QUERY,+ recall_cascade.DERIVED_FACTS_QUERY,+ recall_cascade.DERIVED_EPISODES_QUERY,+ ], "the cascade runs last, finding nothing derived"+ assert result.derived == []@pytest.mark.asyncioasync def test_reason_is_recorded(self) -> None:- driver = _driver(*_SOFT)+ driver = scripted(*SOFT, *NOTHING_DERIVED)- await _retract(driver, ["u1"], reason="settings_page")+ await forget(driver, ["u1"], reason="settings_page")- assert _call(driver, 1)[1]["reason"] == "settings_page"+ assert own_call(driver, 1)[1]["reason"] == "settings_page"@pytest.mark.asyncioasync def test_unknown_uuid_is_a_no_match_and_nothing_is_written(self) -> None:- driver = _driver([])+ driver = scripted([], []) # the lookup, then nothing names it- result = await _retract(driver, ["missing"])+ result = await forget(driver, ["missing"])- assert result.deleted == []+ assert result.deleted == [] and result.resumed == []assert [(f.uuid, f.code) for f in result.failures] == [("missing", MemoryForgetFailureCode.NO_MATCH)]assert result.failures[0].reason == FORGET_NO_MATCH_REASON- assert driver.execute_query.await_count == 1+ assert own_queries(driver) == [+ recall_forget._EXISTING_EDGES_QUERY,+ recall_cascade_queries.NAMED_ROOTS_QUERY,+ ], "reads only"@pytest.mark.asyncioasync def test_lookup_error_fails_every_uuid_with_its_reason(self) -> None:- driver = _driver(RuntimeError("Unknown function 'datetime'"))+ driver = scripted(RuntimeError("Unknown function 'datetime'"))- result = await _retract(driver, ["u1", "u2"])+ result = await forget(driver, ["u1", "u2"])assert result.deleted == []assert [f.uuid for f in result.failures] == ["u1", "u2"]@@ -120,52 +121,52 @@ class TestSoftRetract:@pytest.mark.asyncioasync def test_write_error_and_vanished_edge_are_told_apart(self) -> None:- driver = _driver(+ driver = scripted([{"uuid": "errored"}, {"uuid": "vanished"}],RuntimeError("boom"),[], # deleted between the lookup and the write)- result = await _retract(driver, ["errored", "vanished"])+ result = await forget(driver, ["errored", "vanished"])by_uuid = {f.uuid: f for f in result.failures}assert by_uuid["errored"].code == MemoryForgetFailureCode.QUERY_ERRORassert "boom" in by_uuid["errored"].reasonassert by_uuid["vanished"].code == MemoryForgetFailureCode.NO_MATCH- assert driver.execute_query.await_count == 3, "nothing to hide"+ assert len(own_queries(driver)) == 3, "nothing to hide"@pytest.mark.asyncioasync def test_a_failed_hide_is_a_cleanup_error_on_each_edge(self) -> None:- driver = _driver(+ driver = scripted([{"uuid": "u1"}, {"uuid": "u2"}],[{"uuid": "u1"}],[{"uuid": "u2"}],RuntimeError("down"),)- result = await _retract(driver, ["u1", "u2"])+ result = await forget(driver, ["u1", "u2"])assert result.deleted == ["u1", "u2"], "the facts themselves are forgotten"assert [(f.uuid, f.code) for f in result.failures] == [- ("u1", _CLEANUP),- ("u2", _CLEANUP),+ ("u1", CLEANUP),+ ("u2", CLEANUP),]assert "RuntimeError: down" in result.failures[0].reason@pytest.mark.asyncioasync def test_repeated_uuids_are_forgotten_once(self) -> None:- driver = _driver(*_SOFT)+ driver = scripted(*SOFT, *NOTHING_DERIVED)- result = await _retract(driver, ["u1", "u1"])+ result = await forget(driver, ["u1", "u1"])assert result.deleted == ["u1"] and result.failures == []- assert _call(driver, 0)[1]["uuids"] == ["u1"]+ assert own_call(driver, 0)[1]["uuids"] == ["u1"]@pytest.mark.asyncioasync def test_empty_request_opens_no_driver(self) -> None:open_driver = MagicMock()with patch.object(recall_forget, "open_driver", open_driver):- result = await recall_forget.retract(_SCOPE, [])+ result = await recall_forget.retract(SCOPE, [])assert result.deleted == [] and result.failures == []open_driver.assert_not_called()@@ -177,32 +178,34 @@ class TestWriteLock:self, lock_redis: FakeRedis) -> None:held: list[bool] = []- results = list(_SOFT)+ results = [[], *SOFT, *NOTHING_DERIVED, [{"count": 0}]]async def query(cypher: str, **params: object):- held.append(_LOCK in lock_redis.values)+ held.append(LOCK in lock_redis.values)return results.pop(0), [], Nonedriver = AsyncMock()driver.execute_query.side_effect = query- await _retract(driver, ["u1"])+ await forget(driver, ["u1"])- assert held == [True] * len(_SOFT)- assert _LOCK not in lock_redis.values, "released afterwards"+ assert held == [True] * (+ 2 + len(SOFT) + len(NOTHING_DERIVED)+ ), "reconcile, cascade and the read of writes in flight too"+ assert LOCK not in lock_redis.values, "released afterwards"@pytest.mark.asyncioasync def test_a_lock_held_for_the_whole_wait_fails_as_busy_and_writes_nothing(self, lock_redis: FakeRedis) -> None:- lock_redis.values[_LOCK] = "an ingestion's token"+ lock_redis.values[LOCK] = "an ingestion's token"open_driver = MagicMock()with (patch.object(recall_forget, "FORGET_LOCK_WAIT_SECONDS", 0),patch.object(recall_forget, "open_driver", open_driver),):- result = await recall_forget.retract(_SCOPE, ["u1", "u2"])+ result = await recall_forget.retract(SCOPE, ["u1", "u2"])assert result.deleted == []assert [(f.uuid, f.code) for f in result.failures] == [@@ -211,17 +214,17 @@ class TestWriteLock:]assert result.failures[0].reason == FORGET_BUSY_REASONopen_driver.assert_not_called()- assert lock_redis.values[_LOCK] == "an ingestion's token", "left alone"+ assert lock_redis.values[LOCK] == "an ingestion's token", "left alone"@pytest.mark.asyncioasync def test_redis_unreachable_forgets_without_the_lock(self, caplog: pytest.LogCaptureFixture) -> None:- driver = _driver(*_SOFT)+ driver = scripted(*SOFT, *NOTHING_DERIVED)down = AsyncMock(side_effect=ConnectionError("redis down"))with patch.object(scope_lock, "get_redis_async", down):- result = await _retract(driver, ["u1"])+ result = await forget(driver, ["u1"])assert (result.deleted, result.failures) == (["u1"], [])assert "write lock unavailable" in caplog.text@@ -230,43 +233,74 @@ class TestWriteLock:class TestHardRetract:@pytest.mark.asyncioasync def test_hides_everything_before_it_purges_anything(self) -> None:- driver = _driver(- *_SOFT[:4],- [{"uuid": "ep1"}, {"uuid": "ep2"}], # redact+ redacted = [{"uuid": "ep1", "via": ["u1"]}, {"uuid": "ep2", "via": ["u1"]}]+ driver = scripted(+ *SOFT[:4],+ redacted, # redact+ [], # the cascade: no earlier try,+ redacted, # the episodes citing the fact,+ [], # the dream's among them erased,+ [], # nothing derived+ [],[{"uuid": "ep1", "content": "Alice works on Atlas"}], # citing[{"uuid": "ep1"}], # tombstone[{"uuid": "u1", "deleted_entities": ["alice", "carol"]}], # delete)- result = await _retract(driver, ["u1"], hard=True)+ result = await forget(driver, ["u1"], hard=True)assert result.deleted == ["u1"] and result.failures == []+ write, kwargs = own_call(driver, 1)+ assert "e.hard_forgotten_at = CASE WHEN $hard" in write+ assert kwargs["hard"] is True, "stamped before the purge, for a late settle"assert result.tombstoned_episodes == ["ep1"]assert result.redacted_episodes == ["ep2"], "kept for another edge, hidden"assert result.deleted_entities == ["alice", "carol"]- assert _queries(driver)[4:] == [+ assert own_queries(driver)[4:] == [+ recall_hide.REDACT_EPISODES_QUERY,+ recall_cascade.EARLIER_QUERY,recall_hide.REDACT_EPISODES_QUERY,+ recall_erase.ERASE_DREAM_EPISODES_QUERY,+ recall_cascade.DERIVED_FACTS_QUERY,+ recall_cascade.DERIVED_EPISODES_QUERY,recall_orphans._CITING_EPISODES_QUERY,recall_orphans._TOMBSTONE_QUERY,recall_orphans._DELETE_EDGE_QUERY,- ]+ ], "the cascade runs before anything is purged"@pytest.mark.asyncioasync def test_a_failed_hide_stops_it_before_any_purge(self) -> None:- driver = _driver([{"uuid": "u1"}], [{"uuid": "u1"}], RuntimeError("down"))+ """The cascade still runs, erasing (``recall_forget_cascade_test.py``),+ but nothing is purged: a deleted edge could no longer hide its text."""+ driver = scripted(+ [{"uuid": "u1"}],+ [{"uuid": "u1"}],+ RuntimeError("down"), # the hide's scrub+ [], # the cascade: no earlier try,+ [{"uuid": "ep1", "via": ["u1"]}], # the episodes citing the fact,+ [], # the dream's among them erased,+ [], # nothing derived+ [],+ )- result = await _retract(driver, ["u1"], hard=True)+ result = await forget(driver, ["u1"], hard=True)assert result.deleted == [], "a deleted edge could no longer hide its text"- assert [(f.uuid, f.code) for f in result.failures] == [("u1", _CLEANUP)]- assert driver.execute_query.await_count == 3+ assert [(f.uuid, f.code) for f in result.failures] == [("u1", CLEANUP)]+ assert own_queries(driver)[3:] == [+ recall_cascade.EARLIER_QUERY,+ recall_hide.REDACT_EPISODES_QUERY,+ recall_erase.ERASE_DREAM_EPISODES_QUERY,+ recall_cascade.DERIVED_FACTS_QUERY,+ recall_cascade.DERIVED_EPISODES_QUERY,+ ], "the cascade, and no purge"@pytest.mark.asyncioasync def test_unmatched_delete_is_a_no_match(self) -> None:- driver = _driver(*_SOFT[:4], [], [], [], [])+ driver = scripted(*SOFT[:4], [], [], [], [], [], [], [], [])- result = await _retract(driver, ["u1"], hard=True)+ result = await forget(driver, ["u1"], hard=True)assert result.deleted == []assert [f.code for f in result.failures] == [MemoryForgetFailureCode.NO_MATCH]- assert driver.execute_query.await_count == 8+ assert len(own_queries(driver)) == 12diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_writes.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_writes.pynew file mode 100644index 0000000000..86277a8e9d--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_writes.py@@ -0,0 +1,75 @@+"""The dream writes a forget answers for (``recall_forget.py``).++Before anything else a forget completes the dream citation markers its+graph still holds (``recall_reconcile.reconcile``), so its cascade sees the+provenance of every dream write that landed. After its cascade it asks+whether a dream write that could still land cites anything it reached+(``recall_reconcile.in_flight``): a root, a derived fact it retracted or+walked through, an episode it hid. Where the graph's write lock holds, none+can; where it does not (Redis unreachable, a lost lease), one can land+after the forget, and only its own settle (``recall_landing.py``) retracts+it. Either way, when something may be missing, each fact the forget+retracted is a ``cleanup_error``, which asks for the forget again: the+retry reconciles what landed meanwhile, and reports the forget done once no+such write is left.+"""++import logging++from graphiti_core.driver.driver import GraphDriver++from .memory_model import ForgetResult, MemoryForgetFailure+from .recall_reconcile import in_flight, reconcile++logger = logging.getLogger(__name__)+++async def unreconciled(driver: GraphDriver, group_id: str) -> Exception | None:+ """Complete the graph's pending dream records before anything else; what+ went wrong, when some may still be missing: the error, or ``None``."""+ try:+ done = await reconcile(driver, group_id)+ except Exception as exc:+ logger.warning(+ f"Could not reconcile graph {group_id[:20]} before a forget",+ exc_info=True,+ )+ return exc+ if done.incomplete():+ return RuntimeError("dream writes that landed are still without a record")+ return None+++async def still_landing(+ driver: GraphDriver, group_id: str, result: ForgetResult, roots: list[str]+) -> Exception | None:+ """Why the forget is not done when a dream write that could still land+ cites something it reached (a root, a derived fact it retracted or+ walked through, an episode it hid), else ``None``."""+ if not roots:+ return None+ facts = [*roots, *result.derived, *result.passed]+ try:+ count = await in_flight(driver, facts, result.redacted_episodes)+ except Exception as exc:+ logger.warning(+ f"Could not read the dream writes in flight in graph {group_id[:20]}",+ exc_info=True,+ )+ return exc+ if not count:+ return None+ return RuntimeError(f"{count} dream write(s) citing what it reached may still land")+++def provenance_incomplete(+ result: ForgetResult, retracted: list[str], exc: Exception+) -> None:+ """A ``cleanup_error`` on each fact forgotten that has no failure yet: its+ cascade may have missed a dream fact whose record was still pending."""+ failed = {failure.uuid for failure in result.failures}+ result.failures.extend(+ MemoryForgetFailure.cleanup_error(uuid, exc)+ for uuid in retracted+ if uuid not in failed+ )diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_writes_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_writes_test.pynew file mode 100644index 0000000000..ffe031e1c8--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_forget_writes_test.py@@ -0,0 +1,110 @@+"""Unit tests for the forget's side of the dream citation markers+(``recall_forget_writes.py``), through ``recall_forget.retract``: it reads+the graph's markers before anything else and completes the landed ones+(``recall_reconcile.py``, pinned in ``recall_reconcile_test.py``), and+after its cascade asks for a retry (``cleanup_error``) while a write that+could still land cites what it reached. The live runs are+``recall_provenance_integration_test.py`` and+``recall_marker_race_integration_test.py``.+"""++from unittest.mock import AsyncMock, patch++import pytest++from . import recall_forget, recall_forget_writes, recall_reconcile+from .recall_forget_fake import (+ CLEANUP,+ NOTHING_DERIVED,+ SOFT,+ forget,+ in_flight_reads,+ scripted,+)+++class TestPendingDreamRecords:+ """Every forget completes the graph's pending dream records first+ (``recall_reconcile.py``, pinned in ``recall_reconcile_test.py``), so+ its cascade sees them."""++ @pytest.mark.asyncio+ async def test_it_reads_them_before_anything_else(self) -> None:+ driver = scripted(*SOFT, *NOTHING_DERIVED)++ await forget(driver, ["u1"])++ first = driver.execute_query.await_args_list[0]+ assert first.args[0] == recall_reconcile.MARKERS_QUERY++ @pytest.mark.asyncio+ async def test_a_failed_read_forgets_anyway_and_asks_for_a_retry(self) -> None:+ driver = scripted(*SOFT, *NOTHING_DERIVED, markers=RuntimeError("down"))++ result = await forget(driver, ["u1"])++ assert result.deleted == ["u1"], "the fact itself is forgotten"+ assert [(f.uuid, f.code) for f in result.failures] == [("u1", CLEANUP)]+ assert "RuntimeError: down" in result.failures[0].reason++ @pytest.mark.asyncio+ async def test_more_than_one_forget_takes_asks_for_a_retry(self) -> None:+ driver = AsyncMock()+ driver.execute_query.side_effect = [+ (r, [], None) for r in (*SOFT, *NOTHING_DERIVED)+ ]++ with patch.object(+ recall_forget_writes,+ "reconcile",+ AsyncMock(return_value=recall_reconcile.Reconciled(left=True)),+ ):+ result = await forget(driver, ["u1"])++ assert [(f.uuid, f.code) for f in result.failures] == [("u1", CLEANUP)]+ assert "without a record" in result.failures[0].reason++ @pytest.mark.asyncio+ async def test_a_write_in_flight_citing_what_it_reached_asks_for_a_retry(+ self,+ ) -> None:+ """Its write could land after the forget, and only its own settle+ would retract it. The forget reads what it reached: its roots, the+ derived facts it retracted or walked through, the episodes it hid."""++ async def cascade(driver, group_id, roots, now, result, *, erase) -> None:+ result.derived.append("d1")+ result.passed.append("p1")+ result.redacted_episodes.append("dream-ep")++ driver = scripted(*SOFT, in_flight=1)++ with patch.object(recall_forget, "cascade", cascade):+ result = await forget(driver, ["u1"])++ assert result.deleted == ["u1"], "the fact itself is forgotten"+ assert [(f.uuid, f.code) for f in result.failures] == [("u1", CLEANUP)]+ assert "may still land" in result.failures[0].reason+ [read] = in_flight_reads(driver)+ assert (read["facts"], read["episodes"], read["state"]) == (+ ["u1", "d1", "p1"],+ ["ep1", "dream-ep"],+ "pending",+ )++ @pytest.mark.asyncio+ async def test_a_failed_in_flight_read_asks_for_a_retry(self) -> None:+ driver = scripted(*SOFT, *NOTHING_DERIVED, in_flight=RuntimeError("down"))++ result = await forget(driver, ["u1"])++ assert [(f.uuid, f.code) for f in result.failures] == [("u1", CLEANUP)]+ assert "RuntimeError: down" in result.failures[0].reason++ @pytest.mark.asyncio+ async def test_nothing_forgotten_reads_no_write_in_flight(self) -> None:+ driver = scripted([], []) # the lookup, then nothing names it++ await forget(driver, ["missing"])++ assert in_flight_reads(driver) == []diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_hide.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_hide.pyindex 2f72e8ff05..ce343afbeb 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/recall_hide.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_hide.py@@ -148,16 +148,26 @@ WHERE member IN scrubbedSET c.summary = ''"""-# Every episode naming one of ``$uuids`` that the recall policy now hides,-# which after the retraction is all of them: the stamp and the read-side test-# cannot disagree. ``coalesce`` keeps the time of the first redaction.+# Every episode naming one of ``$uuids``, or hidden for one before, that the+# recall policy now hides, which after the retraction is all of them: the+# stamp and the read-side test cannot disagree. ``coalesce`` keeps the time+# of the first redaction. ``redacted_for`` keeps, once each, the forgotten+# facts it was hidden for: a hard forget's purge drops a deleted fact from+# ``entity_edges``, and a cascade resumed after it still finds the fact's own+# episodes through this (``recall_cascade.py``). ``via`` names the facts+# among ``$uuids`` it cites or was hidden for (the cascade follows it).REDACT_EPISODES_QUERY = (forgotten_facts_clause()+ f"""MATCH (ep:Episodic)-WHERE any(x IN coalesce(ep.entity_edges, []) WHERE x IN $uuids)+WHERE any(x IN coalesce(ep.entity_edges, []) + coalesce(ep.redacted_for, [])+ WHERE x IN $uuids)AND NOT ({recallable_episode_predicate("ep")})-SET ep.redacted_at = coalesce(ep.redacted_at, $now)-RETURN ep.uuid AS uuid+WITH ep, coalesce(ep.redacted_for, []) AS kept+SET ep.redacted_at = coalesce(ep.redacted_at, $now),+ ep.redacted_for = kept + [x IN coalesce(ep.entity_edges, [])+ WHERE x IN $uuids AND NOT x IN kept]+RETURN ep.uuid AS uuid,+ [x IN coalesce(ep.entity_edges, []) + kept WHERE x IN $uuids] AS via""")diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_hide_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_hide_test.pyindex 19230db11b..16f82e3c9c 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/recall_hide_test.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_hide_test.py@@ -151,7 +151,19 @@ class TestRedactQuery:live fact: the redaction and the read side share one predicate."""query = recall_hide.REDACT_EPISODES_QUERYassert query.startswith(forgotten_facts_clause())- assert "any(x IN coalesce(ep.entity_edges, []) WHERE x IN $uuids)" in query+ assert (+ "any(x IN coalesce(ep.entity_edges, []) + coalesce(ep.redacted_for, [])"+ in query+ )assert f"NOT ({recallable_episode_predicate('ep')})" in queryassert "SET ep.redacted_at = coalesce(ep.redacted_at, $now)" in queryassert "content" not in query, "the text stays for audit"++ def test_an_episode_remembers_the_forgotten_facts_it_was_hidden_for(+ self,+ ) -> None:+ """So a cascade resumed after a hard forget purged the fact, which+ drops it from ``entity_edges``, still finds the fact's episodes."""+ query = recall_hide.REDACT_EPISODES_QUERY+ assert "ep.redacted_for = kept + [x IN coalesce(ep.entity_edges, [])" in query+ assert "WHERE x IN $uuids AND NOT x IN kept]" in querydiff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_integration_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_integration_test.pyindex 2f1a7c9e23..8c400f1495 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/recall_integration_test.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_integration_test.py@@ -64,6 +64,8 @@ _LONG_AGO = "2025-01-01T00:00:00+00:00"# Only the forget's marker, or the pre-policy shape, makes a fact forgotten._EDGE_SHAPES = {("active", None, None, None, None): False,+ (None, None, None, None, None): False, # an edge from before MemoryFact+ (None, None, _LONG_AGO, None, None): True,("retracted", "user_signal", _LONG_AGO, None, _LONG_AGO): True,("retracted", None, None, None, None): False,("retracted", "user_signal", _LONG_AGO, None, None): False,@@ -275,23 +277,29 @@ async def test_the_forgotten_fact_cypher_and_python_agree(forgotten=forgotten,)+ forgotten = forgotten_fact_predicate("e")found = await rows(driver,"MATCH ()-[e:RELATES_TO]->() RETURN e.uuid AS uuid, "- f"CASE WHEN {forgotten_fact_predicate('e')} THEN true ELSE false END AS f",+ f"{forgotten} AS f, NOT ({forgotten}) AS kept",)in_cypher = {row["uuid"]: row["f"] for row in found}read_back = await EntityEdge.get_by_uuids(driver, uuids)assert in_cypher == {edge.uuid: is_forgotten(edge) for edge in read_back}assert [in_cypher[edge_uuid] for edge_uuid in uuids] == list(_EDGE_SHAPES.values())+ # Never null, so its negation holds on every live edge, one with no+ # status or reason set included.+ kept = {row["uuid"]: row["kept"] for row in found}+ assert kept == {edge_uuid: not f for edge_uuid, f in in_cypher.items()}+ assert (kept[uuids[0]], kept[uuids[1]]) == (True, True)# The search for a permanent delete ranks these alone: with every fact- # equally close (the stub embeds everything alike), six that are not- # forgotten take none of the slots, however few.- forgotten = {edge_uuid for edge_uuid, gone in in_cypher.items() if gone}+ # equally close (the stub embeds everything alike), those not forgotten+ # take none of the slots, however few.+ gone = {edge_uuid for edge_uuid, f in in_cypher.items() if f}every = await search_forgotten_facts(scope, "Atlas", limit=len(uuids))- assert {edge.uuid for edge in every} == forgotten+ assert {edge.uuid for edge in every} == gonetwo = await search_forgotten_facts(scope, "Atlas", limit=2)- assert len(two) == 2 and {edge.uuid for edge in two} <= forgotten+ assert len(two) == 2 and {edge.uuid for edge in two} <= gone@pytest.mark.integrationdiff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_landing.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_landing.pynew file mode 100644index 0000000000..7bbc1b2a43--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_landing.py@@ -0,0 +1,123 @@+"""A dream write that lands after a forget reached what it cites.++The graph's write lock keeps a forget from landing between a dream write's+check before ``add_episode`` (``recall_citations.rests_on_a_forget``) and its+record. Where the lock does not hold (Redis unreachable, or a holder that+lost its lease: ``scope_lock.py``), a forget can land in between, and its+cascade cannot reach the write's facts, which carry no record yet. So+whoever completes a write's citation marker, its writer right after the+record or ``recall_reconcile.reconcile``, settles the write first:+``settle`` looks up every source it cites and, when a forget reached one+since the pass read the graph, directly or up the derived facts no longer+live that it cites (``recall_sources.py``), runs a forget's cascade from it+(``recall_cascade.py``). The write's facts, now recorded, are retracted and+its dream episode hidden, erased when a hard forget reached the source.+Only then is the marker cleared. The writer settles from the citations it+holds, so it does even when its marker is already gone.++A cited derived fact no longer live is cascaded from itself, under the root+the source it rests on names: the cascade leaves it as it is and retracts+what rests on it, the write's facts among them. When the walk up from one+stopped at its bound (``recall_sources.ancestry``), the settle fails closed+and cascades from it under its own name.+"""++import logging+from datetime import datetime, timezone++from graphiti_core.driver.driver import GraphDriver+from pydantic import BaseModel, Field++from .memory_model import ForgetResult+from .recall_cascade import cascade+from .recall_citations import Citations+from .recall_sources import (+ Reach,+ ancestry,+ fact_reach,+ fact_states,+ reached_episodes,+ walkable,+)++logger = logging.getLogger(__name__)+++class Reached(BaseModel):+ """The sources a write cites that a forget reached, each mapped to the+ root its retractions name: facts, and hidden episodes (``seeds``), to+ cascade from softly or, for what a hard forget reached, erasing+ (``hard``)."""++ soft: dict[str, str] = Field(default_factory=dict)+ hard: dict[str, str] = Field(default_factory=dict)+ soft_seeds: dict[str, str] = Field(default_factory=dict)+ hard_seeds: dict[str, str] = Field(default_factory=dict)++ def any(self) -> bool:+ return bool(self.soft or self.hard or self.soft_seeds or self.hard_seeds)++ def fact(self, uuid: str, reach: Reach) -> None:+ (self.hard if reach.hard else self.soft)[uuid] = reach.root++ def seed(self, uuid: str, reach: Reach) -> None:+ (self.hard_seeds if reach.hard else self.soft_seeds)[uuid] = reach.root+++async def settle(driver: GraphDriver, group_id: str, citations: Citations) -> bool:+ """Cascade from each source in ``citations`` a forget reached since the+ pass read the graph, so what rests on it (the write just recorded) is+ retracted; True when nothing is left to do, False when a cascade stopped+ short and the write's marker must stay. Raises when a read fails."""+ reached = await reached_sources(driver, citations)+ if not reached.any():+ return True+ result = ForgetResult()+ now = datetime.now(timezone.utc).isoformat()+ for facts, seeds, erase in (+ (reached.hard, reached.hard_seeds, True),+ (reached.soft, reached.soft_seeds, False),+ ):+ if facts or seeds:+ await cascade(+ driver,+ group_id,+ list(facts),+ now,+ result,+ erase=erase,+ seeds=seeds,+ named=facts,+ )+ if result.failures:+ logger.warning(+ f"Could not settle a dream write in graph {group_id[:20]}: its "+ "cascade stopped short; its marker stays for reconcile"+ )+ return False+ logger.info(+ f"Settled a dream write in graph {group_id[:20]} that landed after a "+ f"forget of what it cites: {len(result.derived)} fact(s) retracted"+ )+ return True+++async def reached_sources(driver: GraphDriver, citations: Citations) -> Reached:+ """The sources in ``citations`` a forget reached, directly or through+ the derived facts no longer live among them, as ``Reached``."""+ reached = Reached()+ facts = await fact_states(driver, citations.fact_uuids)+ for uuid in dict.fromkeys(citations.fact_uuids):+ if (reach := fact_reach(uuid, facts.get(uuid))) is not None:+ reached.fact(uuid, reach)+ walk = await ancestry(driver, list(facts.values()))+ for uuid, reach in walk.reached.items():+ reached.fact(uuid, reach)+ if walk.unfinished:+ for row in facts.values():+ if walkable(row) and row["uuid"] not in walk.reached:+ reached.fact(row["uuid"], Reach(root=row["uuid"], hard=False))+ episodes = await reached_episodes(driver, citations.episode_uuids)+ for uuid, reach in episodes.items():+ reached.seed(uuid, reach)+ return reacheddiff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_landing_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_landing_test.pynew file mode 100644index 0000000000..bbad8e9d87--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_landing_test.py@@ -0,0 +1,186 @@+"""Unit tests for ``recall_landing.settle``: a dream write that landed after+a forget reached what it cites is cascaded from before its marker clears.+A forgotten fact softly, under the root its reason names; one a hard forget+reached (purged, still being purged, or erased by its cascade) erasing; a+hidden episode under the root it was hidden for, erasing when a hard forget+emptied it, reached that root, or it is gone; a cited derived fact no+longer live that rests on any of those, from itself under that root, and+from itself under its own name when the walk up stopped at its bound (it+fails closed). The walk is pinned in ``recall_sources_test.py``; the live+runs are in ``recall_marker_integration_test.py``,+``recall_marker_race_integration_test.py`` and+``recall_ancestry_integration_test.py``.+"""++from unittest.mock import patch++import pytest++from . import recall_landing, recall_sources+from .memory_model import MemoryForgetFailure+from .recall_citations import Citations+from .recall_sources_fake import episode, fact, sources+++class _Cascades(list):+ """The cascades ``settle`` ran: (roots, erase, seeds, named)."""++ def __init__(self, fail: bool = False) -> None:+ super().__init__()+ self.fail = fail++ async def __call__(+ self, driver, group_id, roots, now, result, *, erase, seeds, named+ ) -> None:+ self.append((roots, erase, seeds, named))+ result.derived.append("late-fact")+ if self.fail:+ result.failures.append(MemoryForgetFailure.derived_left(roots[0]))+++async def _settle(driver, citations: Citations, cascades: _Cascades) -> bool:+ with patch.object(recall_landing, "cascade", cascades):+ return await recall_landing.settle(driver, "user_a", citations)+++async def test_nothing_forgotten_settles_at_once() -> None:+ driver = sources({"f1": fact(live=True)}, {"ep1": episode()})+ cascades = _Cascades()++ settled = await _settle(+ driver, Citations(fact_uuids=["f1"], episode_uuids=["ep1"]), cascades+ )++ assert settled is True+ assert cascades == []+++async def test_a_forgotten_fact_is_cascaded_from_under_its_root() -> None:+ """``f1`` was retracted by a cascade from ``root``; ``f2`` is live."""+ driver = sources(+ {+ "f1": fact(forgotten=True, reason="derived_from_forgotten:root"),+ "f2": fact(live=True),+ }+ )+ cascades = _Cascades()++ settled = await _settle(driver, Citations(fact_uuids=["f1", "f2"]), cascades)++ assert settled is True+ assert cascades == [(["f1"], False, {}, {"f1": "root"})]+++async def test_a_fact_a_hard_forget_reached_is_cascaded_from_erasing() -> None:+ """``gone`` was purged, ``root`` is still being purged, ``d1`` was+ erased by a hard forget's cascade from ``r0``."""+ driver = sources(+ {+ "root": fact(forgotten=True, hard=True, reason="user_signal"),+ "d1": fact(forgotten=True, hard=True, reason="derived_from_forgotten:r0"),+ }+ )+ cascades = _Cascades()++ await _settle(driver, Citations(fact_uuids=["gone", "root", "d1"]), cascades)++ assert cascades == [+ (+ ["gone", "root", "d1"],+ True,+ {},+ {"gone": "gone", "root": "root", "d1": "r0"},+ )+ ]+++async def test_a_hidden_episode_seeds_under_the_root_it_was_hidden_for() -> None:+ """``ep1`` was hidden for ``d1``, which a cascade from ``root``+ retracted; ``tomb`` is emptied, ``orphan`` hidden for a purged fact and+ ``gone`` gone, all three erasing."""+ driver = sources(+ {"d1": fact(forgotten=True, reason="derived_from_forgotten:root")},+ {+ "ep1": episode(hidden=True, hidden_for=("d1",)),+ "ep2": episode(),+ "tomb": episode(hidden=True, hard=True),+ "orphan": episode(hidden=True, hidden_for=("purged",)),+ },+ )+ cascades = _Cascades()++ await _settle(+ driver,+ Citations(episode_uuids=["ep1", "ep2", "tomb", "orphan", "gone"]),+ cascades,+ )++ assert cascades == [+ ([], True, {"gone": "gone", "tomb": "tomb", "orphan": "purged"}, {}),+ ([], False, {"ep1": "root"}, {}),+ ]+++async def test_a_cited_derived_fact_no_longer_live_is_cascaded_from_itself() -> None:+ """``d`` was superseded after the pass read it, and ``f``, which it was+ derived from, forgotten: the cascade from ``d`` leaves it as it is and+ retracts what rests on it, the write's facts among them, under ``f``;+ ``e``'s source was purged, so it erases."""+ driver = sources(+ {+ "d": fact(facts=("f",)),+ "e": fact(facts=("purged",)),+ "f": fact(forgotten=True, reason="user_signal"),+ }+ )+ cascades = _Cascades()++ await _settle(driver, Citations(fact_uuids=["d", "e"]), cascades)++ assert cascades == [+ (["e"], True, {}, {"e": "purged"}),+ (["d"], False, {}, {"d": "f"}),+ ]+++async def test_a_walk_stopped_at_its_bound_fails_closed() -> None:+ """Its sources could not all be read: the cited fact is cascaded from+ under its own name, so the write's facts are retracted all the same."""+ driver = sources({"d": fact(facts=("p1", "p2", "p3"))})+ cascades = _Cascades()++ with patch.object(recall_sources, "CASCADE_MAX_ITEMS", 2):+ await _settle(driver, Citations(fact_uuids=["d"]), cascades)++ assert cascades == [(["d"], False, {}, {"d": "d"})]+++async def test_hard_sources_are_cascaded_from_first() -> None:+ driver = sources({"f1": fact(forgotten=True)})+ cascades = _Cascades()++ await _settle(driver, Citations(fact_uuids=["gone", "f1"]), cascades)++ assert [(roots, erase) for roots, erase, _, _ in cascades] == [+ (["gone"], True),+ (["f1"], False),+ ]+++async def test_a_cascade_that_stops_short_keeps_the_marker(+ caplog: pytest.LogCaptureFixture,+) -> None:+ driver = sources({"f1": fact(forgotten=True)})++ settled = await _settle(driver, Citations(fact_uuids=["f1"]), _Cascades(fail=True))++ assert settled is False+ assert "its marker stays for reconcile" in caplog.textdiff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_marker_crash_integration_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_marker_crash_integration_test.pynew file mode 100644index 0000000000..1a9c78f21a--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_marker_crash_integration_test.py@@ -0,0 +1,190 @@+"""Dream writes whose writer crashed or whose ``add_episode`` raised, on a+live FalkorDB (``recall_reconcile.py``, ``migrations/dream_markers.py``).++- A crashed writer's placed episode is never recalled; its marker holds up+ a forget of what it cites (``cleanup_error``) for a day, then expires,+ and the operator command deletes it with the episode.+- A write whose ``add_episode`` raised marks its marker aborted; reconcile+ completes it when graphiti saved the episode, even in part, and+ otherwise drops it with the placed episode (through the production+ worker, its model failing once the episode is placed).++Run with FalkorDB reachable (see ``conftest.py``)::++ poetry run pytest -m integration backend/copilot/graphiti/recall_marker_crash_integration_test.py+"""++from collections.abc import AsyncIterator+from typing import Any+from unittest.mock import AsyncMock++import pytest+import pytest_asyncio+from pytest_mock import MockerFixture++from backend.copilot.dream import apply+from backend.copilot.dream.schemas import ConsolidatedFact, DreamOperations++from .falkordb_driver import AutoGPTFalkorDriver+from .marked_write import placed+from .memory_model import MemoryForgetFailureCode+from .migrations import dream_markers+from .recall import forgotten_facts_clause, recallable_episode_predicate+from .recall_cascade_fixtures import FLOUR, SUPPLIES, derivation, dream, gather+from .recall_citations import Citations+from .recall_derivation import MARKER_LABEL, abort, mark+from .recall_forget import retract+from .recall_integration_fixtures import (+ BuildClient,+ episode_row,+ ingest_facts,+ model_of,+ patch_recall_boundaries,+ rows,+ stop_ingestion_workers,+)+from .recall_marker_fixtures import age, land, markers, placed_payload, seed_root+from .recall_reconcile import Reconciled, reconcile++[email protected](autouse=True)+def boundaries(mocker, scope_graph, stub_graphiti_client):+ patch_recall_boundaries(mocker, scope_graph[0], stub_graphiti_client)+++async def test_a_crashed_writer_holds_forgets_up_for_a_day_then_is_resolved(+ scope_graph,+) -> None:+ driver, scope = scope_graph+ gid = scope.group_id+ await seed_root(driver, gid, "x-root")+ marker = await mark(+ driver, gid, "x-episode", "dream_crash", Citations(fact_uuids=["x-root"])+ )+ await placed(driver, gid, placed_payload("dream_crash"), "x-episode")+ [shown] = await rows(+ driver,+ forgotten_facts_clause() + "MATCH (ep:Episodic {uuid: 'x-episode'}) "+ f"RETURN {recallable_episode_predicate('ep')} AS recallable",+ )++ first = await retract(scope, ["x-root"])+ waiting = await reconcile(driver, gid)+ await age(driver, marker)+ expired = await reconcile(driver, gid)+ second = await retract(scope, ["x-root"])+ listed = await dream_markers.list_markers(driver, gid)+ resolved = await dream_markers.resolve_graph(+ driver, gid, dream_markers.Selection(expired=True), apply=True+ )++ assert shown == {"recallable": False}, "a placed episode is never recalled"+ assert [(f.uuid, f.code) for f in first.failures] == [+ ("x-root", MemoryForgetFailureCode.CLEANUP_ERROR)+ ]+ assert "may still land" in first.failures[0].reason+ assert (waiting, expired) == (Reconciled(waiting=1), Reconciled(expired=1))+ assert second.failures == [], "past the bound it holds nothing up"+ assert [(m.uuid, m.state, m.saved, m.landed) for m in listed] == [+ (marker, "expired", False, False)+ ]+ assert resolved == dream_markers.Resolved(deleted=1)+ assert await markers(driver) == []+ assert await episode_row(driver, "x-episode") == {}, "deleted with its marker"+++async def test_an_aborted_write_graphiti_saved_in_part_is_completed(+ scope_graph,+) -> None:+ driver, scope = scope_graph+ gid, cited = scope.group_id, Citations(fact_uuids=["s-root"])+ await seed_root(driver, gid, "s-root")+ marker = await mark(driver, gid, "s-episode", "dream_saved", cited)+ await land(+ driver,+ gid,+ episode="s-episode",+ name="dream_saved",+ edge="s-fact",+ content="Saved",+ missing=("s-never-saved",),+ )+ await abort(driver, marker)++ done = await reconcile(driver, gid)+ forgotten = await retract(scope, ["s-root"])++ assert done == Reconciled(completed=1), "an episode was saved: completed"+ assert (await derivation(driver, "s-fact"))["facts"] == ["s-root"]+ assert (forgotten.failures, forgotten.derived) == ([], ["s-fact"])+ assert await markers(driver) == []+++@pytest_asyncio.fixture(loop_scope="function")+async def dream_apply(mocker: MockerFixture) -> AsyncIterator[None]:+ """apply's Postgres side (the dream session and its summary) stubbed."""+ mocker.patch.object(apply, "_create_dream_session", AsyncMock(return_value="s"))+ mocker.patch.object(apply, "_write_dream_summary_message", AsyncMock())+ mocker.patch(+ "backend.copilot.dream.registry.ensure_dream_system_scheduled",+ AsyncMock(return_value=None),+ )+ yield+ await stop_ingestion_workers()+++def _model_down(build: BuildClient, mocker: MockerFixture) -> BuildClient:+ """``build``, its model failing every call."""++ def built(driver: AutoGPTFalkorDriver, responses: dict[str, dict]) -> Any:+ client = build(driver, responses)+ failing = AsyncMock(side_effect=RuntimeError("model down"))+ mocker.patch.object(model_of(client), "_generate_response", failing)+ return client++ return built+++async def test_a_write_whose_add_episode_raised_is_dropped_with_its_episode(+ scope_graph, stub_graphiti_client, dream_apply, mocker: MockerFixture+) -> None:+ """Through the worker: graphiti's model fails once the episode is placed."""+ driver, scope = scope_graph+ said, edges = await ingest_facts(+ driver, scope, stub_graphiti_client, [FLOUR], session_id="s-1"+ )+ write = ConsolidatedFact(+ content=SUPPLIES[2],+ confidence=0.9,+ source_fact_uuids=[edges[FLOUR[2]]],+ source_episode_uuids=[said],+ )+ read = await gather(scope)++ stats = await dream(+ driver,+ scope,+ _model_down(stub_graphiti_client, mocker),+ DreamOperations(writes=[write]),+ read,+ SUPPLIES,+ "p-abort",+ )+ [left] = await rows(+ driver,+ f"MATCH (m:{MARKER_LABEL}) MATCH (ep:Episodic {{uuid: m.episode_uuid}}) "+ "RETURN m.state AS state, ep.uuid AS episode, ep.write_pending AS pending",+ )+ done = await reconcile(driver, scope.group_id)++ assert (stats["failed_writes"], stats["provenance_pending"]) == (1, 0)+ assert (left["state"], left["pending"]) == ("aborted", True)+ assert done == Reconciled(dropped=1)+ assert await markers(driver) == []+ assert await episode_row(driver, left["episode"]) == {}diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_marker_fixtures.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_marker_fixtures.pynew file mode 100644index 0000000000..87706b00af--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_marker_fixtures.py@@ -0,0 +1,148 @@+"""Seeding and reads for the live dream-marker tests+(``recall_marker_integration_test.py``,+``recall_marker_crash_integration_test.py``,+``recall_marker_race_integration_test.py``,+``recall_ancestry_integration_test.py``): a user's fact and the episode+that stated it, a write landing as graphiti saves one, the markers a graph+holds, and a forget run to the end while graphiti saves a dream write.+Not collected by pytest.+"""++from collections.abc import Awaitable, Callable, Iterator+from contextlib import contextmanager+from datetime import datetime, timezone+from typing import Any+from unittest.mock import patch++from graphiti_core.driver.driver import GraphProvider+from graphiti_core.driver.falkordb_driver import FalkorDriverSession+from graphiti_core.models.edges.edge_db_queries import get_entity_edge_save_bulk_query+from graphiti_core.nodes import EpisodeType++from .fact_attributes import merging_save+from .falkordb_driver import AutoGPTFalkorDriver+from .memory_model import ForgetResult+from .recall_derivation import MARKER_LABEL+from .recall_forget import retract+from .recall_integration_fixtures import rows+from .scope import MemoryScope++# Long past ``recall_reconcile.MARKER_EXPIRY_SECONDS``.+LONG_AGO = "2000-01-01T00:00:00+00:00"+# graphiti's bulk save of a write's facts (``add_episode``), as the driver+# runs it: a merge of the map (``fact_attributes.merging_save``).+FACT_SAVE = merging_save(get_entity_edge_save_bulk_query(GraphProvider.FALKORDB))+++def saves_facts(query: Any) -> bool:+ """Whether ``query``, one statement or a transaction's, is graphiti's+ bulk save of a write's facts."""+ text = query if isinstance(query, str) else " ".join(q for q, _ in query)+ return FACT_SAVE in text+++async def seed_root(driver: AutoGPTFalkorDriver, group_id: str, root: str) -> None:+ """A user's fact ``root`` and the episode that stated it."""+ await driver.execute_query(+ """+ CREATE (a:Entity {uuid: $root + '-a', name: 'Root A', group_id: $gid}),+ (b:Entity {uuid: $root + '-b', name: 'Root B', group_id: $gid}),+ (:Episodic {uuid: $root + '-episode', name: 'user turn',+ group_id: $gid, content: 'The root source sentence',+ entity_edges: [$root]}),+ (a)-[:RELATES_TO {uuid: $root, group_id: $gid,+ fact: 'The root source sentence', name: 'rel',+ status: 'active', scope: 'real:global',+ episodes: [$root + '-episode'], created_at: $now}]->(b)+ """,+ gid=group_id,+ root=root,+ now=datetime.now(timezone.utc).isoformat(),+ )+++async def land(+ driver: AutoGPTFalkorDriver,+ group_id: str,+ *,+ episode: str,+ name: str,+ edge: str,+ content: str,+ missing: tuple[str, ...] = (),+) -> None:+ """A write landing as graphiti saves one: its episode saved over by+ uuid (every property replaced), then the fact it produced. ``missing``+ are facts the episode lists that were never saved."""+ await driver.execute_query(+ """+ MERGE (ep:Episodic {uuid: $episode})+ SET ep = {uuid: $episode, name: $name, group_id: $gid, content: $content,+ source: 'json', source_description: 'dream-pass consolidation',+ entity_edges: [$edge] + $missing}+ CREATE (a:Entity {uuid: $edge + '-a', name: $edge + ' A', group_id: $gid}),+ (b:Entity {uuid: $edge + '-b', name: $edge + ' B', group_id: $gid}),+ (a)-[:RELATES_TO {uuid: $edge, group_id: $gid, fact: $content,+ name: 'derived', status: 'active',+ scope: 'real:global', episodes: [$episode],+ created_at: $now, fact_embedding: [0.1, 0.2]}]->(b)+ """,+ gid=group_id,+ episode=episode,+ name=name,+ edge=edge,+ content=content,+ missing=list(missing),+ now=datetime.now(timezone.utc).isoformat(),+ )+++def placed_payload(name: str) -> dict[str, Any]:+ """What ``marked_write.placed`` is handed for a dream write ``name``."""+ return {+ "name": name,+ "episode_body": '{"content": "A dream fact"}',+ "source": EpisodeType.json,+ "source_description": "dream-pass consolidation",+ "reference_time": datetime.now(timezone.utc),+ }+++async def markers(driver: AutoGPTFalkorDriver) -> list[dict[str, Any]]:+ """Every marker's uuid and state."""+ return await rows(+ driver,+ f"MATCH (m:{MARKER_LABEL}) RETURN m.uuid AS uuid, m.state AS state "+ "ORDER BY uuid",+ )+++async def age(driver: AutoGPTFalkorDriver, marker: str) -> None:+ """Make ``marker`` look written long ago."""+ await driver.execute_query(+ f"MATCH (m:{MARKER_LABEL} {{uuid: $uuid}}) SET m.created_at = $long_ago",+ uuid=marker,+ long_ago=LONG_AGO,+ )+++@contextmanager+def forget_mid_save(+ scope: MemoryScope,+ root: str,+ hard: bool,+ unfence: Callable[[], Awaitable[None]],+) -> Iterator[list[ForgetResult]]:+ """Forget ``root`` to the end right before graphiti saves the dream+ write's facts, once ``unfence`` has taken the lock's protection away."""+ original = FalkorDriverSession.run+ forgotten: list[ForgetResult] = []++ async def run(session: FalkorDriverSession, query: Any, **params: Any) -> Any:+ if not forgotten and saves_facts(query):+ await unfence()+ forgotten.append(await retract(scope, [root], hard=hard))+ return await original(session, query, **params)++ with patch.object(FalkorDriverSession, "run", run):+ yield forgottendiff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_marker_integration_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_marker_integration_test.pynew file mode 100644index 0000000000..c6f68b766d--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_marker_integration_test.py@@ -0,0 +1,212 @@+"""The dream citation marker on a live FalkorDB (``recall_derivation.py``,+``recall_reconcile.py``, ``recall_landing.py``, ``migrations/dream_markers.py``):+Codex's second-pass attacks on it, as regressions.++- A marker names its write's episode by uuid: a user's episode sharing the+ dream episode's name is never stamped, and a hard forget of what the+ dream cited leaves it whole.+- A pending marker is never deleted for its age: past a day it expires, no+ longer holds forgets up, and is kept; a write landing after that is+ settled on landing, by its writer or by the next reconcile: retracted+ and, its root purged, erased. So is one settling while a hard forget is+ still purging its root.+- A write whose marker an operator resolved is settled by its writer when+ it lands.++Crashed and aborted writers are ``recall_marker_crash_integration_test.py``;+the young race, a forget while a write is in flight where the lock does not+hold, is ``recall_marker_race_integration_test.py``.++Run with FalkorDB reachable (see ``conftest.py``)::++ poetry run pytest -m integration backend/copilot/graphiti/recall_marker_integration_test.py+"""++from typing import Any+from unittest.mock import patch++import pytest++from . import recall_forget+from .marked_write import placed+from .memory_model import MemoryForgetFailureCode+from .migrations import dream_markers+from .recall import FORGOTTEN_FACT+from .recall_cascade_fixtures import derivation+from .recall_cascade_walk import derived_reason+from .recall_citations import Citations+from .recall_derivation import mark, record+from .recall_forget import retract+from .recall_integration_fixtures import (+ edge_row,+ episode_row,+ live_facts,+ patch_recall_boundaries,+)+from .recall_marker_fixtures import age, land, markers, placed_payload, seed_root+from .recall_reconcile import Reconciled, reconcile++[email protected](autouse=True)+def boundaries(mocker, scope_graph, stub_graphiti_client):+ patch_recall_boundaries(mocker, scope_graph[0], stub_graphiti_client)+++async def test_a_user_episode_named_like_the_dream_episode_is_never_touched(+ scope_graph,+) -> None:+ """Codex's collision: both were stamped, then both bodies erased."""+ driver, scope = scope_graph+ gid, shared = scope.group_id, "dream_same_name"+ await seed_root(driver, gid, "c-root")+ user_says = "A sentence supplied independently by the user"+ await land(+ driver,+ gid,+ episode="c-user",+ name=shared,+ edge="c-user-fact",+ content=user_says,+ )+ await land(+ driver,+ gid,+ episode="c-dream",+ name=shared,+ edge="c-dream-fact",+ content="Dream",+ )+ await mark(driver, gid, "c-dream", shared, Citations(fact_uuids=["c-root"]))++ reconciled = await reconcile(driver, gid)+ result = await retract(scope, ["c-root"], hard=True)++ assert reconciled == Reconciled(completed=1)+ assert (await derivation(driver, "c-user"))["facts"] is None, "never stamped"+ assert (await derivation(driver, "c-dream"))["facts"] == ["c-root"]+ assert (result.failures, result.derived) == ([], ["c-dream-fact"])+ assert (await episode_row(driver, "c-user"))["content"] == user_says+ assert (await episode_row(driver, "c-dream"))["content"] == ""+ assert "c-user-fact" in await live_facts(driver)++[email protected]("settled_by", ["writer", "reconcile"])+async def test_an_old_marker_is_kept_and_its_late_write_settled_on_landing(+ scope_graph, settled_by: str+) -> None:+ """Codex's slow write: its marker was deleted after an hour, and the+ write that landed later kept a live fact no forget could reach."""+ driver, scope = scope_graph+ gid, cited = scope.group_id, Citations(fact_uuids=["o-root"])+ await seed_root(driver, gid, "o-root")+ marker = await mark(driver, gid, "o-episode", "dream_old", cited)+ await age(driver, marker)++ expired = await reconcile(driver, gid)+ forgotten = await retract(scope, ["o-root"], hard=True)+ kept = await markers(driver)+ await land(+ driver,+ gid,+ episode="o-episode",+ name="dream_old",+ edge="o-fact",+ content="Late",+ )+ if settled_by == "writer":+ assert await record(driver, gid, marker, "o-episode", ["o-fact"], cited)+ else:+ assert await reconcile(driver, gid) == Reconciled(completed=1)++ assert expired == Reconciled(expired=1)+ assert forgotten.failures == [], "an expired marker holds no forget up"+ assert kept == [{"uuid": marker, "state": "expired"}], "never deleted for age"+ assert "o-fact" not in await live_facts(driver), "settled, before any retry"+ fact = await edge_row(driver, "o-fact")+ assert (fact["status"], fact["reason"]) == ("retracted", derived_reason("o-root"))+ assert (fact["fact"], fact["fact_redacted"]) == (FORGOTTEN_FACT, ""), "erased"+ assert (await episode_row(driver, "o-episode"))["content"] == ""+ assert await markers(driver) == []+ retried = await retract(scope, ["o-root"], hard=True)+ assert (retried.failures, retried.resumed) == ([], ["o-root"])+++async def test_a_write_settled_while_a_hard_forget_purges_is_erased(+ scope_graph,+) -> None:+ """Where the lock does not hold, a write can land and settle after a+ hard forget retracted its root and before it purged it: the root's+ ``hard_forgotten_at`` makes that settle erase, as one after the purge+ would."""+ driver, scope = scope_graph+ gid, cited = scope.group_id, Citations(fact_uuids=["h-root"])+ await seed_root(driver, gid, "h-root")+ marker = await mark(driver, gid, "h-episode", "dream_purging", cited)+ purge = recall_forget.purge++ async def landing_then_purge(*args: Any) -> None:+ await land(+ driver,+ gid,+ episode="h-episode",+ name="dream_purging",+ edge="h-fact",+ content="Late",+ )+ assert await record(driver, gid, marker, "h-episode", ["h-fact"], cited)+ await purge(*args)++ with patch.object(recall_forget, "purge", landing_then_purge):+ forgotten = await retract(scope, ["h-root"], hard=True)++ assert [f.code for f in forgotten.failures] == [+ MemoryForgetFailureCode.CLEANUP_ERROR+ ], "its marker was there when the forget looked"+ assert "h-fact" not in await live_facts(driver)+ fact = await edge_row(driver, "h-fact")+ assert (fact["fact"], fact["fact_redacted"]) == (FORGOTTEN_FACT, "")+ assert await edge_row(driver, "h-root") == {}, "purged all the same"+++async def test_a_write_whose_marker_was_resolved_is_settled_by_its_writer(+ scope_graph,+) -> None:+ driver, scope = scope_graph+ gid, cited = scope.group_id, Citations(fact_uuids=["r-root"])+ await seed_root(driver, gid, "r-root")+ marker = await mark(driver, gid, "r-episode", "dream_resolved", cited)+ await placed(driver, gid, placed_payload("dream_resolved"), "r-episode")++ refused = await dream_markers.resolve_graph(+ driver, gid, dream_markers.Selection(uuids={marker}), apply=True+ )+ resolved = await dream_markers.resolve_graph(+ driver, gid, dream_markers.Selection(uuids={marker}, force=True), apply=True+ )+ forgotten = await retract(scope, ["r-root"])+ await land(+ driver,+ gid,+ episode="r-episode",+ name="dream_resolved",+ edge="r-fact",+ content="Late",+ )+ recorded = await record(driver, gid, marker, "r-episode", ["r-fact"], cited)++ assert (refused.refused, refused.deleted) == (1, 0), "young: only with --force"+ assert resolved == dream_markers.Resolved(deleted=1)+ assert forgotten.failures == []+ assert recorded, "settled from the citations its writer holds"+ assert "r-fact" not in await live_facts(driver)+ fact = await edge_row(driver, "r-fact")+ assert (fact["reason"], fact["fact_redacted"]) == (derived_reason("r-root"), "Late")+ assert (await derivation(driver, "r-fact"))["facts"] == ["r-root"]diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_marker_race_integration_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_marker_race_integration_test.pynew file mode 100644index 0000000000..d6749b8cac--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_marker_race_integration_test.py@@ -0,0 +1,218 @@+"""A forget landing while a dream write is in flight, where the graph's write+lock does not hold, on a live FalkorDB through ``dream/apply.py`` and the+production ingestion worker (``recall_forget_writes.py``,+``recall_landing.py``).++The lock admits two writers at once when Redis cannot be reached (both go+ahead without it) or when a holder loses its lease (its key expires while+it writes, and the forget takes the lock). Codex's race: the forget+reported success while the write was still in flight, and the write then+landed a live derived fact resting on a purged root.++Here the forget runs to the end right before graphiti saves the write's+facts (its episode is saved, its facts are not), so it cannot reach them:+it reports ``cleanup_error``, since a write citing its root may still land.+The write lands, and its writer settles it before clearing its marker: its+fact is retracted (erased, for a hard forget) a few statements after+graphiti saved it, the dream episode hidden. A read in those few+statements can see the fact live; a writer that died in between leaves its+marker, which the next reconcile (the forget's retry, which the+``cleanup_error`` asks for) completes and settles. Forgetting again reports+the forget done.++Run with FalkorDB reachable (see ``conftest.py``)::++ poetry run pytest -m integration backend/copilot/graphiti/recall_marker_race_integration_test.py+"""++from collections.abc import AsyncIterator, Awaitable, Callable+from unittest.mock import AsyncMock++import pytest+import pytest_asyncio+from pytest_mock import MockerFixture+from redis.asyncio import Redis++from backend.copilot.dream import apply+from backend.copilot.dream.schemas import ConsolidatedFact, DreamOperations++from . import marked_write, scope_lock+from .memory_model import MemoryForgetFailureCode+from .recall import FORGOTTEN_FACT+from .recall_cascade_fixtures import FLOUR, SUPPLIES, derivation, dream, gather+from .recall_cascade_walk import derived_reason+from .recall_derivation import MARKER_LABEL+from .recall_forget import retract+from .recall_integration_fixtures import (+ count,+ edge_row,+ episode_row,+ ingest_facts,+ live_facts,+ patch_recall_boundaries,+ rows,+ sentence_properties,+ stop_ingestion_workers,+)+from .recall_marker_fixtures import forget_mid_save+from .scope import MemoryScope, write_lock_key++_MARKERS = f"MATCH (m:{MARKER_LABEL}) RETURN count(m) AS c"++[email protected](autouse=True)+def boundaries(mocker, scope_graph, stub_graphiti_client):+ patch_recall_boundaries(mocker, scope_graph[0], stub_graphiti_client)+++@pytest_asyncio.fixture(loop_scope="function")+async def dream_apply(mocker: MockerFixture) -> AsyncIterator[None]:+ """apply's Postgres side (the dream session and its summary) stubbed."""+ mocker.patch.object(apply, "_create_dream_session", AsyncMock(return_value="s"))+ mocker.patch.object(apply, "_write_dream_summary_message", AsyncMock())+ mocker.patch(+ "backend.copilot.dream.registry.ensure_dream_system_scheduled",+ AsyncMock(return_value=None),+ )+ yield+ await stop_ingestion_workers()+++def _unfencing(+ overlap: str, scope: MemoryScope, redis: Redis, mocker: MockerFixture+) -> Callable[[], Awaitable[None]]:+ """What takes the lock's protection away: Redis down for every writer+ from the start, or the writer's lease lost right before the forget."""+ if overlap == "redis_down":+ down = AsyncMock(side_effect=ConnectionError("redis down"))+ mocker.patch.object(scope_lock, "get_redis_async", down)++ async def unfence() -> None:+ if overlap == "lease_lost":+ key = write_lock_key(scope.group_id)+ assert await redis.get(key), "the writer held the lock"+ await redis.delete(key)++ return unfence+++async def _dreamed(driver, kind: str) -> str:+ """The one fact or episode carrying a dream's record."""+ match = "()-[x:RELATES_TO]->()" if kind == "fact" else "(x:Episodic)"+ [row] = await rows(+ driver,+ f"MATCH {match} WHERE x.derived_from_facts IS NOT NULL RETURN x.uuid AS uuid",+ )+ return row["uuid"]+++ ("overlap", "hard"), [("redis_down", True), ("lease_lost", False)]+)+async def test_a_forget_while_a_write_is_in_flight_and_the_write_landing(+ scope_graph,+ stub_graphiti_client,+ dream_apply,+ live_lock: Redis,+ mocker: MockerFixture,+ overlap: str,+ hard: bool,+) -> None:+ driver, scope = scope_graph+ said, edges = await ingest_facts(+ driver, scope, stub_graphiti_client, [FLOUR], session_id="s-1"+ )+ flour = edges[FLOUR[2]]+ write = ConsolidatedFact(+ content=SUPPLIES[2],+ confidence=0.9,+ source_fact_uuids=[flour],+ source_episode_uuids=[said],+ )+ read = await gather(scope)+ unfence = _unfencing(overlap, scope, live_lock, mocker)++ with forget_mid_save(scope, flour, hard, unfence) as forgotten:+ stats = await dream(+ driver,+ scope,+ stub_graphiti_client,+ DreamOperations(writes=[write]),+ read,+ SUPPLIES,+ f"p-{overlap}",+ )++ [first] = forgotten+ assert [(f.uuid, f.code) for f in first.failures] == [+ (flour, MemoryForgetFailureCode.CLEANUP_ERROR)+ ], "a write citing the root may still land"+ assert "may still land" in first.failures[0].reason+ assert (stats["provenance_pending"], stats["failed_writes"]) == (0, 0)+ supplies = await _dreamed(driver, "fact")+ assert await live_facts(driver) == {}, "its writer settled it on landing"+ record = await derivation(driver, supplies)+ assert (record["facts"], record["reason"]) == ([flour], derived_reason(flour))+ fact = await edge_row(driver, supplies)+ assert fact["fact"] == FORGOTTEN_FACT+ assert (fact["fact_redacted"] == "") is hard, "erased only for a hard forget"+ episode = await episode_row(driver, await _dreamed(driver, "episode"))+ assert episode["redacted_at"] is not None, "the dream episode is hidden"+ assert await count(driver, _MARKERS) == 0+ if hard:+ assert await sentence_properties(driver, SUPPLIES[2]) == set()+ retried = await retract(scope, [flour], hard=hard)+ assert retried.failures == [], "forgetting again reports it done"+++async def test_a_writer_dying_after_its_write_landed_is_settled_by_the_retry(+ scope_graph,+ stub_graphiti_client,+ dream_apply,+ live_lock: Redis,+ mocker: MockerFixture,+) -> None:+ """The residual window: the writer dies between graphiti's save and its+ record, so its fact is live until the next reconcile, which the+ forget's ``cleanup_error`` asks for: the retry completes the marker,+ settles the write and erases the fact."""+ driver, scope = scope_graph+ said, edges = await ingest_facts(+ driver, scope, stub_graphiti_client, [FLOUR], session_id="s-1"+ )+ flour = edges[FLOUR[2]]+ write = ConsolidatedFact(+ content=SUPPLIES[2],+ confidence=0.9,+ source_fact_uuids=[flour],+ source_episode_uuids=[said],+ )+ read = await gather(scope)+ unfence = _unfencing("redis_down", scope, live_lock, mocker)+ mocker.patch.object(marked_write, "recorded", AsyncMock())++ with forget_mid_save(scope, flour, True, unfence) as forgotten:+ await dream(+ driver,+ scope,+ stub_graphiti_client,+ DreamOperations(writes=[write]),+ read,+ SUPPLIES,+ "p-dies",+ )+ live = await live_facts(driver)+ retried = await retract(scope, [flour], hard=True)++ [first] = forgotten+ assert [f.code for f in first.failures] == [MemoryForgetFailureCode.CLEANUP_ERROR]+ assert list(live.values()) == [SUPPLIES[2]], "live until the next reconcile"+ assert (retried.failures, retried.resumed) == ([], [flour])+ assert await live_facts(driver) == {}+ fact = await edge_row(driver, await _dreamed(driver, "fact"))+ assert (fact["fact"], fact["fact_redacted"]) == (FORGOTTEN_FACT, "")+ assert await count(driver, _MARKERS) == 0diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_provenance_integration_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_provenance_integration_test.pynew file mode 100644index 0000000000..0743397c3d--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_provenance_integration_test.py@@ -0,0 +1,240 @@+"""A dream write's provenance is durable before its fact exists, on a live+FalkorDB whose own Redis holds the graph's write lock, through+``dream/apply.py`` and the production ingestion worker+(``recall_derivation.py``, ``recall_reconcile.py``,+``provenance_pending.py``).++Codex's probe: the record written after the write fails. Before the marker,+the pass reported success, the derived fact was live with no record, and+forgetting what it was derived from left it live. Now the marker written+before the write keeps the citations, the pass reports the write as+``provenance_pending``, and the next forget completes the record before it+cascades, even one that was waiting for the write's lock; the reaper's sweep+completes it without a forget. A marker that cannot be written drops the+write.++Run with FalkorDB reachable (see ``conftest.py``)::++ poetry run pytest -m integration backend/copilot/graphiti/recall_provenance_integration_test.py+"""++import asyncio+from collections.abc import AsyncIterator, Iterator+from contextlib import contextmanager+from typing import Any+from unittest.mock import AsyncMock, patch++import pytest+import pytest_asyncio+from graphiti_core.driver.falkordb_driver import FalkorDriverSession+from pytest_mock import MockerFixture++from backend.copilot.dream import apply+from backend.copilot.dream.schemas import ConsolidatedFact, DreamOperations+from backend.data import redis_client++from .falkordb_driver import AutoGPTFalkorDriver+from .memory_model import ForgetResult+from .provenance_pending import PENDING_KEY, Swept, sweep_pending+from .recall_cascade_fixtures import FLOUR, SUPPLIES, derivation, dream, gather+from .recall_derivation import MARK_QUERY, MARKER_LABEL, RECORD_EPISODE_QUERY+from .recall_forget import retract+from .recall_integration_fixtures import (+ count,+ ingest_facts,+ live_facts,+ patch_recall_boundaries,+ rows,+ stop_ingestion_workers,+)+from .recall_marker_fixtures import saves_facts+from .scope import MemoryScope++_MARKERS = f"MATCH (m:{MARKER_LABEL}) RETURN count(m) AS c"++[email protected](autouse=True)+def boundaries(mocker, scope_graph, stub_graphiti_client):+ patch_recall_boundaries(mocker, scope_graph[0], stub_graphiti_client)+++@pytest_asyncio.fixture(loop_scope="function")+async def dream_apply(mocker: MockerFixture) -> AsyncIterator[None]:+ """apply's Postgres side (the dream session and its summary) stubbed."""+ mocker.patch.object(apply, "_create_dream_session", AsyncMock(return_value="s"))+ mocker.patch.object(apply, "_write_dream_summary_message", AsyncMock())+ mocker.patch(+ "backend.copilot.dream.registry.ensure_dream_system_scheduled",+ AsyncMock(return_value=None),+ )+ yield+ await stop_ingestion_workers()+++@contextmanager+def _failing(query: str, times: int | None = None) -> Iterator[None]:+ """Every run of ``query`` raises while the block is open, or only its+ first ``times`` runs."""+ original = AutoGPTFalkorDriver.execute_query+ runs = [0]++ async def failing(self, cypher_query_, **params):+ if cypher_query_ == query:+ runs[0] += 1+ if times is None or runs[0] <= times:+ raise RuntimeError("injected failure")+ return await original(self, cypher_query_, **params)++ with patch.object(AutoGPTFalkorDriver, "execute_query", failing):+ yield+++async def _said(driver, scope: MemoryScope, build) -> tuple[str, str, DreamOperations]:+ """The user's flour fact and chat turn, and a consolidation citing both."""+ said, edges = await ingest_facts(driver, scope, build, [FLOUR], session_id="s-1")+ flour = edges[FLOUR[2]]+ write = ConsolidatedFact(+ content=SUPPLIES[2],+ confidence=0.9,+ source_fact_uuids=[flour],+ source_episode_uuids=[said],+ )+ return flour, said, DreamOperations(writes=[write])+++async def _supplies(driver) -> str | None:+ """The consolidation's fact, forgotten or not; None if never written."""+ found = await rows(+ driver,+ "MATCH ()-[e:RELATES_TO]->() "+ "WHERE coalesce(e.fact_redacted, e.fact) = $sentence RETURN e.uuid AS uuid",+ sentence=SUPPLIES[2],+ )+ return found[0]["uuid"] if found else None+++async def _pending() -> set[str]:+ redis = await redis_client.get_redis_async()+ return set(redis.sets.get(PENDING_KEY, set()))+++async def test_a_record_that_failed_is_reconciled_before_the_next_forget(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ driver, scope = scope_graph+ flour, _, ops = await _said(driver, scope, stub_graphiti_client)+ read = await gather(scope)++ with _failing(RECORD_EPISODE_QUERY):+ stats = await dream(+ driver, scope, stub_graphiti_client, ops, read, SUPPLIES, "p-fail"+ )++ supplies = await _supplies(driver)+ assert supplies is not None and supplies in await live_facts(driver)+ assert (stats["consolidated_count"], stats["provenance_pending"]) == (1, 1)+ assert (await derivation(driver, supplies))["facts"] is None+ assert await count(driver, _MARKERS) == 1+ assert await _pending() == {scope.group_id}++ result = await retract(scope, [flour])++ assert (result.derived, result.failures) == ([supplies], [])+ assert await live_facts(driver) == {}+ assert (await derivation(driver, supplies))["facts"] == [flour]+ assert await count(driver, _MARKERS) == 0+++async def test_a_marker_that_cannot_be_written_drops_the_write(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ driver, scope = scope_graph+ _, _, ops = await _said(driver, scope, stub_graphiti_client)+ read = await gather(scope)++ with _failing(MARK_QUERY):+ stats = await dream(+ driver, scope, stub_graphiti_client, ops, read, SUPPLIES, "p-nomark"+ )++ assert (stats["failed_writes"], stats["provenance_pending"]) == (1, 0)+ assert await _supplies(driver) is None+ dreamed = (+ "MATCH (ep:Episodic) WHERE ep.name STARTS WITH 'dream_' RETURN count(ep) AS c"+ )+ assert await count(driver, dreamed) == 0+ assert await count(driver, _MARKERS) == 0+++async def test_the_reaper_sweep_completes_a_record_without_a_forget(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ driver, scope = scope_graph+ flour, said, ops = await _said(driver, scope, stub_graphiti_client)+ read = await gather(scope)+ with _failing(RECORD_EPISODE_QUERY):+ await dream(driver, scope, stub_graphiti_client, ops, read, SUPPLIES, "p-r")++ swept = await sweep_pending()++ assert swept == Swept(graphs=1, completed=1)+ supplies = await _supplies(driver)+ assert supplies is not None+ record = await derivation(driver, supplies)+ assert (record["facts"], record["episodes"]) == ([flour], [said])+ assert await count(driver, _MARKERS) == 0+ assert await _pending() == set()+++@contextmanager+def _forget_while_saving(+ scope: MemoryScope, flour: str+) -> Iterator[tuple[list[asyncio.Future], list[bool]]]:+ """Start a forget of ``flour`` right before graphiti saves the dream+ write's edges, while the worker holds the graph's write lock, and note+ whether it is still waiting for the lock 0.3 s later."""+ original = FalkorDriverSession.run+ started: list[asyncio.Future] = []+ waited: list[bool] = []++ async def run(session: FalkorDriverSession, query: Any, **params: Any) -> Any:+ if not started and saves_facts(query):+ started.append(asyncio.ensure_future(retract(scope, [flour])))+ await asyncio.sleep(0.3)+ waited.append(not started[0].done())+ return await original(session, query, **params)++ with patch.object(FalkorDriverSession, "run", run):+ yield started, waited+++async def test_a_forget_waiting_for_a_write_whose_record_fails_retracts_it(+ scope_graph, stub_graphiti_client, dream_apply+) -> None:+ driver, scope = scope_graph+ flour, _, ops = await _said(driver, scope, stub_graphiti_client)+ read = await gather(scope)++ with (+ _failing(RECORD_EPISODE_QUERY, times=1),+ _forget_while_saving(scope, flour) as (started, waited),+ ):+ stats = await dream(+ driver, scope, stub_graphiti_client, ops, read, SUPPLIES, "p-race"+ )+ result: ForgetResult = await asyncio.wait_for(started[0], 30)++ assert waited == [True], "the forget waited for the write's lock"+ supplies = await _supplies(driver)+ assert stats["provenance_pending"] == 1+ assert (result.derived, result.failures) == ([supplies], [])+ assert await live_facts(driver) == {}+ assert await count(driver, _MARKERS) == 0diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_reconcile.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_reconcile.pynew file mode 100644index 0000000000..9437828bc8--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_reconcile.py@@ -0,0 +1,256 @@+"""Complete, or resolve, the dream citation markers a graph still holds.++The ingestion worker writes a dream write's citation marker before the+write, naming the uuid the write's episode is written under+(``marked_write.py``), and deletes it once the write is recorded and settled+(``recall_derivation.record``). A marker left behind is in one of these+states, and ``reconcile`` acts on each:++- its write landed: graphiti saved the episode and every fact the episode+ lists (the record failed after the write, or the writer died). It is+ completed as the writer would have: the marker's citations recorded on+ the episode, found by its uuid in its graph, and on the facts the episode+ produced or merged into, the write settled (``recall_landing.settle``),+ then the marker deleted;+- ``aborted``: the writer's ``add_episode`` raised. An episode graphiti+ saved, even in part, is completed as above. Otherwise the marker is+ deleted, with the episode its writer placed (``write_pending``, never+ saved), in one statement that first confirms no saved episode has its+ uuid (``drop_unlanded``);+- ``pending``, not landed: its write is under way, or its writer died+ before graphiti saved it. It is ``waiting`` for+ ``MARKER_EXPIRY_SECONDS``, and while it is, a forget of anything it cites+ reports ``cleanup_error`` (``in_flight``): its write could still land+ after the forget, and only its own settle would retract it. Past the+ bound it is marked ``expired`` and no longer holds forgets up, but it is+ never deleted for its age: should the write land after all, its writer+ records and settles it, and a marker kept is completed as soon as its+ episode is there;+- ``expired``: as pending, past the bound. An operator lists and resolves+ these with ``migrations/dream_markers.py``.++Markers whose write landed come first, then aborted, pending and expired+ones, so waiting writes never starve a landed one: at most+``RECONCILE_MAX_MARKERS`` a call, and ``left`` says a landed one remains.++Every forget runs ``reconcile`` before anything else, holding the graph's+write lock (``recall_forget.py``); the dream reaper runs it for the graphs a+failed record was noted in (``provenance_pending.py``), and the derivation+backfill for every graph it writes. Idempotent: a marker completed twice+stamps the same record.+"""++import logging+from datetime import datetime, timedelta, timezone+from typing import Any++from graphiti_core.driver.driver import GraphDriver+from pydantic import BaseModel++from .recall_citations import Citations+from .recall_derivation import ABORTED, EXPIRED, MARKER_LABEL, PENDING, record++logger = logging.getLogger(__name__)++# How long a pending marker whose write has not landed holds up the forgets+# of what it cites before it is marked expired (an add_episode takes minutes+# at most while the lock holds; this bounds a writer that died).+MARKER_EXPIRY_SECONDS = 24 * 3600+# Markers one call looks at; more are left for the next.+RECONCILE_MAX_MARKERS = 200+# The order markers are taken in: landed (or aborted and saved), aborted and+# not saved, pending and not landed, expired.+LANDED, DROPPABLE, UNLANDED, KEPT = 0, 1, 2, 3+++class Reconciled(BaseModel):+ """One call's markers: ``completed`` (recorded, settled, deleted),+ ``unsettled`` (landed, but the record or its settle failed: kept),+ ``dropped`` (aborted, no episode saved), ``waiting`` (could still land:+ pending within the bound, or aborted with an episode saved since it was+ read), ``expired``, and whether a landed one was ``left``."""++ completed: int = 0+ unsettled: int = 0+ dropped: int = 0+ waiting: int = 0+ expired: int = 0+ left: bool = False++ def incomplete(self) -> bool:+ """A write that landed may still be without its record."""+ return self.left or self.unsettled > 0+++async def reconcile(+ driver: GraphDriver, group_id: str, *, limit: int = RECONCILE_MAX_MARKERS+) -> Reconciled:+ """Complete or resolve up to ``limit`` of the markers graph ``group_id``+ holds, landed ones first. Raises when a read fails: what it did stays."""+ markers = _rows(+ await driver.execute_query(MARKERS_QUERY, group_id=group_id, limit=limit + 1)+ )+ done = Reconciled(left=len(markers) > limit and markers[limit]["rank"] == LANDED)+ now = datetime.now(timezone.utc)+ for marker in markers[:limit]:+ await _resolve(driver, group_id, marker, now, done)+ if done.completed or done.dropped or done.unsettled:+ logger.info(+ f"Reconciled graph {group_id[:20]}: {done.completed} dream "+ f"record(s) completed, {done.unsettled} not, {done.dropped} "+ "aborted write(s) resolved"+ )+ return done+++async def _resolve(+ driver: GraphDriver,+ group_id: str,+ marker: dict[str, Any],+ now: datetime,+ done: Reconciled,+) -> None:+ """Act on one marker as its state says (the module docstring)."""+ if marker["rank"] == LANDED:+ if await complete(driver, group_id, marker):+ done.completed += 1+ else:+ done.unsettled += 1+ elif marker["state"] == ABORTED:+ if await drop_unlanded(driver, marker["uuid"]):+ done.dropped += 1+ else:+ done.waiting += 1+ elif marker["state"] == EXPIRED:+ done.expired += 1+ elif age_seconds(marker["created_at"], now) < MARKER_EXPIRY_SECONDS:+ done.waiting += 1+ else:+ await driver.execute_query(+ EXPIRE_QUERY, uuid=marker["uuid"], state=EXPIRED, now=now.isoformat()+ )+ done.expired += 1+++async def complete(driver: GraphDriver, group_id: str, marker: dict[str, Any]) -> bool:+ """Record, settle and delete ``marker`` (a ``MARKERS_QUERY`` row) whose+ write landed; False when that stopped short (the marker stays)."""+ touched = _rows(+ await driver.execute_query(TOUCHED_FACTS_QUERY, episodes=[marker["episode"]])+ )+ citations = Citations(+ fact_uuids=marker["facts"] or [], episode_uuids=marker["episodes"] or []+ )+ return await record(+ driver,+ group_id,+ marker["uuid"],+ marker["episode"],+ [row["uuid"] for row in touched],+ citations,+ )+++async def drop_unlanded(driver: GraphDriver, marker: str) -> bool:+ """Delete ``marker`` and the episode its writer placed, when no saved+ episode has its uuid; True when it was deleted."""+ rows = _rows(await driver.execute_query(DROP_UNLANDED_QUERY, uuid=marker))+ return bool(rows and rows[0]["dropped"])+++async def in_flight(driver: GraphDriver, facts: list[str], episodes: list[str]) -> int:+ """How many dream writes that could still land (``pending`` markers+ within ``MARKER_EXPIRY_SECONDS``) cite one of ``facts`` or+ ``episodes``."""+ if not (facts or episodes):+ return 0+ cutoff = datetime.now(timezone.utc) - timedelta(seconds=MARKER_EXPIRY_SECONDS)+ rows = _rows(+ await driver.execute_query(+ IN_FLIGHT_QUERY,+ facts=facts,+ episodes=episodes,+ state=PENDING,+ cutoff=cutoff.isoformat(),+ )+ )+ return int(rows[0]["count"]) if rows else 0+++def age_seconds(created_at: object, now: datetime) -> float:+ """How long ago a marker was written; unreadable reads as long ago."""+ try:+ written = datetime.fromisoformat(str(created_at))+ except ValueError:+ return float("inf")+ if written.tzinfo is None:+ written = written.replace(tzinfo=timezone.utc)+ return (now - written).total_seconds()+++def _rows(result: Any) -> list[dict[str, Any]]:+ return result[0] if result else []+++# Every marker, with the rank its state gives it: whether graphiti saved its+# episode (found by uuid in the graph, no longer ``write_pending``) and every+# fact that episode lists (graphiti saves the episode first, its facts last).+MARKERS_QUERY = f"""+MATCH (m:{MARKER_LABEL})+OPTIONAL MATCH (ep:Episodic {{uuid: m.episode_uuid}})+WHERE ep.group_id = $group_id+WITH m, ep, coalesce(ep.entity_edges, []) AS listed+UNWIND (CASE WHEN size(listed) = 0 THEN [null] ELSE listed END) AS listed_uuid+OPTIONAL MATCH ()-[e:RELATES_TO {{uuid: listed_uuid}}]->()+WITH m, ep, size(listed) AS wanted, count(e) AS found+WITH m, coalesce(m.state, '{PENDING}') AS state,+ ep IS NOT NULL AND ep.write_pending IS NULL AS saved,+ ep IS NOT NULL AND ep.write_pending IS NULL AND found = wanted AS landed+WITH m, state, saved, CASE+ WHEN landed OR (saved AND state = '{ABORTED}') THEN {LANDED}+ WHEN state = '{ABORTED}' THEN {DROPPABLE}+ WHEN state = '{PENDING}' THEN {UNLANDED}+ ELSE {KEPT} END AS rank+RETURN m.uuid AS uuid, m.episode_uuid AS episode,+ coalesce(m.derived_from_facts, []) AS facts,+ coalesce(m.derived_from_episodes, []) AS episodes,+ m.created_at AS created_at, state, saved, rank+ORDER BY rank, created_at+LIMIT $limit+"""++# A marker whose write graphiti never saved, with the episode its writer+# placed: only when every episode under its uuid is still ``write_pending``+# (or there is none), tested in the statement that deletes it.+DROP_UNLANDED_QUERY = f"""+MATCH (m:{MARKER_LABEL} {{uuid: $uuid}})+OPTIONAL MATCH (ep:Episodic {{uuid: m.episode_uuid}})+WITH m, collect(ep) AS placed+WHERE all(ep IN placed WHERE ep.write_pending IS NOT NULL)+FOREACH (ep IN placed | DETACH DELETE ep)+DELETE m+RETURN count(m) AS dropped+"""++EXPIRE_QUERY = f"""+MATCH (m:{MARKER_LABEL} {{uuid: $uuid}})+WHERE coalesce(m.state, '{PENDING}') = '{PENDING}'+SET m.state = $state, m.expired_at = $now+"""++IN_FLIGHT_QUERY = f"""+MATCH (m:{MARKER_LABEL})+WHERE coalesce(m.state, $state) = $state+ AND m.created_at > $cutoff+ AND (any(x IN coalesce(m.derived_from_facts, []) WHERE x IN $facts)+ OR any(x IN coalesce(m.derived_from_episodes, []) WHERE x IN $episodes))+RETURN count(m) AS count+"""++# The facts an episode produced or merged into: graphiti lists it among+# their ``episodes`` (an edge it only invalidated does not).+TOUCHED_FACTS_QUERY = """+MATCH ()-[e:RELATES_TO]->()+WHERE any(x IN coalesce(e.episodes, []) WHERE x IN $episodes)+RETURN e.uuid AS uuid+"""diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_reconcile_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_reconcile_test.pynew file mode 100644index 0000000000..274699f9bb--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_reconcile_test.py@@ -0,0 +1,238 @@+"""Unit tests for ``recall_reconcile``: the dream citation markers a graph+still holds, each acted on as its state says. One whose write landed (found+by its episode's uuid, never its name) is recorded, settled and deleted; an+aborted one without a saved episode is deleted with the episode its writer+placed, the check and the delete in one statement; a pending one not landed+waits, then expires, and is never deleted for its age; an expired one is+kept. ``in_flight`` counts the writes that could still land. The reaper's+sweep is in ``provenance_pending_test.py``; the live runs are+``recall_provenance_integration_test.py``,+``recall_marker_integration_test.py`` and+``recall_marker_crash_integration_test.py``.+"""++from datetime import datetime, timedelta, timezone+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from . import recall_reconcile+from .recall_citations import Citations++_NOW = datetime.now(timezone.utc)+++def _marker(+ uuid: str,+ rank: int,+ *,+ state: str = "pending",+ age: timedelta = timedelta(0),+ saved: bool = False,+) -> dict:+ return {+ "uuid": uuid,+ "episode": f"episode-{uuid}",+ "facts": ["f1"],+ "episodes": ["ep1"],+ "created_at": (_NOW - age).isoformat(),+ "state": state,+ "saved": saved,+ "rank": rank,+ }+++def _driver(*results) -> MagicMock:+ driver = MagicMock()+ driver.execute_query = AsyncMock(side_effect=[(r, [], None) for r in results])+ return driver+++def _queries(driver: MagicMock) -> list[str]:+ return [call.args[0] for call in driver.execute_query.await_args_list]+++class TestReconcile:+ @pytest.mark.asyncio+ async def test_a_graph_with_no_marker_reads_once(self) -> None:+ driver = _driver([])++ done = await recall_reconcile.reconcile(driver, "user_a")++ assert done == recall_reconcile.Reconciled()+ [read] = driver.execute_query.await_args_list+ assert read.args[0] == recall_reconcile.MARKERS_QUERY+ assert read.kwargs == {"group_id": "user_a", "limit": 201}++ @pytest.mark.asyncio+ async def test_a_landed_marker_is_recorded_settled_and_deleted_by_uuid(+ self,+ ) -> None:+ driver = _driver([_marker("m1", recall_reconcile.LANDED)], [{"uuid": "e1"}])+ record = AsyncMock(return_value=True)++ with patch.object(recall_reconcile, "record", record):+ done = await recall_reconcile.reconcile(driver, "user_a")++ assert done == recall_reconcile.Reconciled(completed=1)+ touched = driver.execute_query.await_args_list[1]+ assert touched.args[0] == recall_reconcile.TOUCHED_FACTS_QUERY+ assert touched.kwargs == {"episodes": ["episode-m1"]}+ record.assert_awaited_once_with(+ driver,+ "user_a",+ "m1",+ "episode-m1",+ ["e1"],+ Citations(fact_uuids=["f1"], episode_uuids=["ep1"]),+ )++ @pytest.mark.asyncio+ async def test_one_whose_record_or_settle_stops_short_is_kept(self) -> None:+ driver = _driver([_marker("m1", recall_reconcile.LANDED)], [])++ with patch.object(recall_reconcile, "record", AsyncMock(return_value=False)):+ done = await recall_reconcile.reconcile(driver, "user_a")++ assert done == recall_reconcile.Reconciled(unsettled=1)+ assert done.incomplete()++ @pytest.mark.asyncio+ async def test_an_aborted_one_whose_write_never_landed_is_dropped(self) -> None:+ """With the episode its writer placed, when graphiti never saved it."""+ driver = _driver(+ [_marker("m1", recall_reconcile.DROPPABLE, state="aborted")],+ [{"dropped": 1}],+ )++ done = await recall_reconcile.reconcile(driver, "user_a")++ assert done == recall_reconcile.Reconciled(dropped=1)+ drop = driver.execute_query.await_args_list[1]+ assert (drop.args[0], drop.kwargs) == (+ recall_reconcile.DROP_UNLANDED_QUERY,+ {"uuid": "m1"},+ )++ @pytest.mark.asyncio+ async def test_an_aborted_one_whose_episode_was_saved_since_is_kept(+ self,+ ) -> None:+ """The drop tests it again in its own statement: an episode graphiti+ saved after the read keeps the marker, for the next reconcile to+ complete."""+ driver = _driver(+ [_marker("m1", recall_reconcile.DROPPABLE, state="aborted")],+ [{"dropped": 0}],+ )++ done = await recall_reconcile.reconcile(driver, "user_a")++ assert done == recall_reconcile.Reconciled(waiting=1)++ @pytest.mark.asyncio+ async def test_a_pending_one_not_landed_waits_within_the_bound(self) -> None:+ driver = _driver(+ [_marker("m1", recall_reconcile.UNLANDED, age=timedelta(hours=23))]+ )++ done = await recall_reconcile.reconcile(driver, "user_a")++ assert done == recall_reconcile.Reconciled(waiting=1)+ assert _queries(driver) == [recall_reconcile.MARKERS_QUERY]++ @pytest.mark.asyncio+ @pytest.mark.parametrize("created_at", ["long ago", None])+ async def test_past_the_bound_it_expires_and_is_never_deleted(+ self, created_at: str | None+ ) -> None:+ old = _marker("m1", recall_reconcile.UNLANDED, age=timedelta(hours=25))+ stale = {**old, "created_at": created_at} if created_at != "long ago" else old+ driver = _driver([stale], [])++ done = await recall_reconcile.reconcile(driver, "user_a")++ assert done == recall_reconcile.Reconciled(expired=1)+ expire = driver.execute_query.await_args_list[1]+ assert expire.args[0] == recall_reconcile.EXPIRE_QUERY+ assert (expire.kwargs["uuid"], expire.kwargs["state"]) == ("m1", "expired")+ assert "DELETE" not in recall_reconcile.EXPIRE_QUERY++ @pytest.mark.asyncio+ async def test_an_expired_one_is_kept(self) -> None:+ driver = _driver(+ [+ _marker(+ "m1", recall_reconcile.KEPT, state="expired", age=timedelta(days=9)+ )+ ]+ )++ done = await recall_reconcile.reconcile(driver, "user_a")++ assert done == recall_reconcile.Reconciled(expired=1)+ assert _queries(driver) == [recall_reconcile.MARKERS_QUERY]++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ ("next_rank", "left"),+ [(0, True), (2, False), (3, False)],+ ids=["a landed one", "one waiting", "one expired"],+ )+ async def test_left_only_when_a_landed_one_remains(+ self, next_rank: int, left: bool+ ) -> None:+ """Landed markers come first, so what is beyond the limit is landed+ only when every one taken was."""+ rows = [_marker("m1", recall_reconcile.KEPT), _marker("m2", next_rank)]+ driver = _driver(rows)++ done = await recall_reconcile.reconcile(driver, "user_a", limit=1)++ assert done.left is left+ assert driver.execute_query.await_args_list[0].kwargs["limit"] == 2+++class TestTheMarkersRead:+ def test_finds_the_episode_by_uuid_in_the_graph_never_by_name(self) -> None:+ query = recall_reconcile.MARKERS_QUERY+ assert "OPTIONAL MATCH (ep:Episodic {uuid: m.episode_uuid})" in query+ assert "WHERE ep.group_id = $group_id" in query+ assert "episode_name" not in query and "ep.name" not in query++ def test_landed_means_graphiti_saved_the_episode_and_every_fact(self) -> None:+ query = recall_reconcile.MARKERS_QUERY+ assert "ep.write_pending IS NULL AND found = wanted AS landed" in query+ assert "ORDER BY rank, created_at" in query++ def test_a_marker_goes_only_while_no_saved_episode_has_its_uuid(self) -> None:+ query = recall_reconcile.DROP_UNLANDED_QUERY+ assert "WHERE all(ep IN placed WHERE ep.write_pending IS NOT NULL)" in query+ assert query.index("WHERE all(") < query.index("DELETE m")+ assert "FOREACH (ep IN placed | DETACH DELETE ep)" in query+++class TestInFlight:+ @pytest.mark.asyncio+ async def test_counts_the_pending_writes_citing_what_it_is_given(self) -> None:+ driver = _driver([{"count": 2}])+ expiry = timedelta(seconds=recall_reconcile.MARKER_EXPIRY_SECONDS)+ before = datetime.now(timezone.utc)++ count = await recall_reconcile.in_flight(driver, ["f1"], ["ep1"])++ after = datetime.now(timezone.utc)+ assert count == 2+ [read] = driver.execute_query.await_args_list+ assert read.args[0] == recall_reconcile.IN_FLIGHT_QUERY+ cutoff = datetime.fromisoformat(read.kwargs["cutoff"])+ assert before - expiry <= cutoff <= after - expiry+ assert (read.kwargs["facts"], read.kwargs["episodes"]) == (["f1"], ["ep1"])+ assert read.kwargs["state"] == "pending"++ @pytest.mark.asyncio+ async def test_nothing_to_cite_reads_nothing(self) -> None:+ driver = _driver()++ assert await recall_reconcile.in_flight(driver, [], []) == 0+ driver.execute_query.assert_not_awaited()diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_sources.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_sources.pynew file mode 100644index 0000000000..d08a0587a2--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_sources.py@@ -0,0 +1,214 @@+"""Whether a forget reached what a dream write cites: directly, or through+the derived facts no longer live that it cites.++A cited fact a forget reached is forgotten, retracted by a forget's cascade+too (its reason names the root the user forgot), or gone: a hard forget+purged it (the pass read it, so it existed). A hard forget reached a fact+that is gone, one it is still purging (``hard_forgotten_at``, stamped when+it retracts the fact) and a derived fact its cascade erased (blank audit+copy). A cited episode a forget reached is hidden, and a tombstone, one+that is gone, or one hidden for a fact a hard forget reached, was emptied+by a hard forget or soon will be. Each leads to the root the user forgot+(``Reach``), hard when a hard forget reached it.++A forget's cascade walks down through a derived fact that is no longer+live (superseded, contradicted) without retracting it, and retracts what+rests on it (``recall_cascade.py``). A write citing such a fact rests on+its sources just as those do, and carries their content. So ``ancestry``+walks up from each cited derived fact no longer live, through the records+(``derived_from_facts``, ``derived_from_episodes``), level by level, as far+as derived facts no longer live go: the cascade's walk-through rule,+upward. A derived fact carries a record and no user's episode states it+(every episode it names carries a record). A source a forget reached ends+its branch, reached; a live fact, a user's fact or a recallable episode+ends it clean. Bounded like the cascade: at most ``CASCADE_MAX_ROUNDS``+levels up and ``CASCADE_MAX_ITEMS`` facts read. A walk stopped by either+with facts left to read is ``unfinished``, and its callers fail closed: the+check before a write drops it (``recall_citations.rests_on_a_forget``), the+settle after one lands retracts it (``recall_landing.settle``).+"""++from typing import Any++from graphiti_core.driver.driver import GraphDriver+from pydantic import BaseModel, Field++from .recall import (+ forgotten_fact_predicate,+ forgotten_facts_clause,+ live_fact_predicate,+ recallable_episode_predicate,+)+from .recall_cascade import CASCADE_MAX_ITEMS, CASCADE_MAX_ROUNDS+from .recall_cascade_walk import DERIVED_FROM_FORGOTTEN++_PREFIX = f"{DERIVED_FROM_FORGOTTEN}:"+++class Reach(BaseModel):+ """Where a source a forget reached leads: the ``root`` the user forgot,+ and whether a ``hard`` forget reached it."""++ root: str+ hard: bool+++class Ancestry(BaseModel):+ """Each cited derived fact no longer live that rests on a source a forget+ reached, and where that source leads; ``unfinished`` when the walk+ stopped at its bound with facts left to read."""++ reached: dict[str, Reach] = Field(default_factory=dict)+ unfinished: bool = False++ def add(self, cited: str, reach: Reach) -> None:+ """File ``reach`` for ``cited``, a hard one over a soft one."""+ known = self.reached.get(cited)+ if known is None or (reach.hard and not known.hard):+ self.reached[cited] = reach+++async def fact_states(driver: GraphDriver, uuids: list[str]) -> dict[str, Any]:+ """``FACT_STATES_QUERY``'s row for each of ``uuids`` still in the graph."""+ return {row["uuid"]: row for row in await _read(driver, FACT_STATES_QUERY, uuids)}+++def walkable(row: dict[str, Any]) -> bool:+ """A derived fact no longer live and not forgotten: the cascade walks+ through one, and a write citing one rests on what it rests on."""+ return bool(row["derived"]) and not (row["live"] or row["forgotten"])+++def fact_reach(uuid: str, row: dict[str, Any] | None) -> Reach | None:+ """Where fact ``uuid`` leads when a forget reached it (its ``row`` None:+ gone, purged by a hard forget), else None."""+ if row is None:+ return Reach(root=uuid, hard=True)+ if not row["forgotten"]:+ return None+ return Reach(root=_root(uuid, row), hard=bool(row["hard"]))+++async def reached_episodes(driver: GraphDriver, uuids: list[str]) -> dict[str, Reach]:+ """Each of ``uuids`` a forget reached (hidden, or gone), and where it+ leads: the root it was hidden for (the first fact in its+ ``redacted_for``, or that fact's root), else itself."""+ rows = await _read(driver, EPISODE_STATES_QUERY, uuids)+ hidden = [row for row in rows if row["hidden"]]+ firsts = [row["hidden_for"][0] for row in hidden if row["hidden_for"]]+ facts = await fact_states(driver, firsts)+ found = {row["uuid"] for row in rows}+ reached = {+ uuid: Reach(root=uuid, hard=True)+ for uuid in dict.fromkeys(uuids)+ if uuid not in found+ }+ reached.update({row["uuid"]: _episode_reach(row, facts) for row in hidden})+ return reached+++async def ancestry(driver: GraphDriver, cited: list[dict[str, Any]]) -> Ancestry:+ """Walk up from the walkable facts among ``cited`` (``fact_states``+ rows) to the sources a forget reached (the module docstring). Each+ source is read once, for the first cited fact that reaches it: either+ caller needs one, since the check drops the whole write, and the+ settle's cascade from any cited fact reaches every fact the write made,+ whose record names them all."""+ found = Ancestry()+ frontier = [row for row in cited if walkable(row)]+ origin = {row["uuid"]: row["uuid"] for row in frontier}+ seen = set(origin)+ read = 0+ for _ in range(CASCADE_MAX_ROUNDS):+ if not frontier:+ return found+ parents = _named(frontier, origin, "facts", seen)+ episodes = _named(frontier, origin, "episodes", set())+ read += len(parents)+ if read > CASCADE_MAX_ITEMS:+ found.unfinished = True+ return found+ states = await fact_states(driver, list(parents))+ for uuid, cited_uuid in parents.items():+ if (reach := fact_reach(uuid, states.get(uuid))) is not None:+ found.add(cited_uuid, reach)+ for uuid, reach in (await reached_episodes(driver, list(episodes))).items():+ found.add(episodes[uuid], reach)+ seen.update(parents)+ frontier = [states[u] for u in parents if u in states and walkable(states[u])]+ origin.update({row["uuid"]: parents[row["uuid"]] for row in frontier})+ found.unfinished = bool(frontier)+ return found+++def _named(+ frontier: list[dict[str, Any]], origin: dict[str, str], key: str, seen: set[str]+) -> dict[str, str]:+ """The sources ``frontier``'s records name under ``key`` (not ``seen``),+ each mapped to the cited fact it was reached from."""+ named: dict[str, str] = {}+ for row in frontier:+ for uuid in row[key]:+ if uuid not in seen and uuid not in named:+ named[uuid] = origin[row["uuid"]]+ return named+++def _episode_reach(row: dict[str, Any], facts: dict[str, Any]) -> Reach:+ """Where a hidden episode leads; hard when it is a tombstone or a hard+ forget reached the fact it was hidden for."""+ first = row["hidden_for"][0] if row["hidden_for"] else None+ root = _root(first, facts[first]) if first in facts else first or row["uuid"]+ purged = first is not None and (first not in facts or bool(facts[first]["hard"]))+ return Reach(root=root, hard=bool(row["hard"]) or purged)+++def _root(uuid: str, fact: dict[str, Any]) -> str:+ """The root a forgotten fact's retraction names, else the fact itself."""+ reason = fact.get("reason") or ""+ return reason.removeprefix(_PREFIX) if reason.startswith(_PREFIX) else uuid+++async def _read(+ driver: GraphDriver, query: str, uuids: list[str]+) -> list[dict[str, Any]]:+ """``query``'s rows for ``uuids``, once each; none read for none."""+ if not uuids:+ return []+ result = await driver.execute_query(query, uuids=list(dict.fromkeys(uuids)))+ return result[0] if result else []+++# Each of ``$uuids`` still in the graph: whether a forget reached it, and a+# hard one (purging it, or its cascade erased it), its reason (a cascade's+# names the root), whether it is live, whether it is derived (it carries a+# record and no user's episode states it: every episode it names carries+# one), and what its record names.+FACT_STATES_QUERY = f"""+MATCH ()-[e:RELATES_TO]->()+WHERE e.uuid IN $uuids+OPTIONAL MATCH (stated:Episodic)+WHERE stated.uuid IN coalesce(e.episodes, [])+ AND stated.derived_from_facts IS NULL+WITH e, count(stated) AS independent+RETURN e.uuid AS uuid, {forgotten_fact_predicate("e")} AS forgotten,+ e.hard_forgotten_at IS NOT NULL+ OR coalesce(e.fact_redacted, '-') = '' AS hard,+ e.expiration_reason AS reason, {live_fact_predicate("e")} AS live,+ e.derived_from_facts IS NOT NULL AND independent = 0 AS derived,+ coalesce(e.derived_from_facts, []) AS facts,+ coalesce(e.derived_from_episodes, []) AS episodes+"""++# Each of ``$uuids`` still in the graph, whether the recall policy hides it,+# whether a hard forget emptied it, and the forgotten facts it was hidden for.+EPISODE_STATES_QUERY = (+ forgotten_facts_clause()+ + f"""+MATCH (ep:Episodic)+WHERE ep.uuid IN $uuids+RETURN ep.uuid AS uuid, NOT ({recallable_episode_predicate("ep")}) AS hidden,+ ep.hard_deleted_at IS NOT NULL AS hard,+ coalesce(ep.redacted_for, []) AS hidden_for+"""+)diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_sources_fake.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_sources_fake.pynew file mode 100644index 0000000000..03777cba20--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_sources_fake.py@@ -0,0 +1,71 @@+"""An in-memory answer to the source-state reads of ``recall_sources.py``,+for the unit tests of the walk up the records, the check before a dream+write and the settle after one lands. Not collected by pytest.+"""++from typing import Any+from unittest.mock import AsyncMock, MagicMock++from . import recall_sources+++def fact(+ *,+ forgotten: bool = False,+ hard: bool = False,+ reason: str | None = None,+ live: bool = False,+ derived: bool = True,+ facts: tuple[str, ...] = (),+ episodes: tuple[str, ...] = (),+) -> dict[str, Any]:+ """A fact's ``FACT_STATES_QUERY`` row, without its uuid: a derived fact+ no longer live unless told otherwise, naming ``facts`` and+ ``episodes``."""+ return {+ "forgotten": forgotten,+ "hard": hard,+ "reason": reason,+ "live": live,+ "derived": derived,+ "facts": list(facts),+ "episodes": list(episodes),+ }+++def episode(+ *, hidden: bool = False, hard: bool = False, hidden_for: tuple[str, ...] = ()+) -> dict[str, Any]:+ """An episode's ``EPISODE_STATES_QUERY`` row, without its uuid."""+ return {"hidden": hidden, "hard": hard, "hidden_for": list(hidden_for)}+++def sources(+ facts: dict[str, dict[str, Any]], episodes: dict[str, dict[str, Any]] | None = None+) -> MagicMock:+ """A driver answering the two state reads from ``facts`` and+ ``episodes`` by uuid (a uuid in neither is gone), and every other query+ with nothing."""+ known = episodes or {}++ async def answer(query: str, **params: Any) -> tuple[list, list, None]:+ if query == recall_sources.FACT_STATES_QUERY:+ rows = [{"uuid": u, **facts[u]} for u in params["uuids"] if u in facts]+ elif query == recall_sources.EPISODE_STATES_QUERY:+ rows = [{"uuid": u, **known[u]} for u in params["uuids"] if u in known]+ else:+ rows = []+ return rows, [], None++ driver = MagicMock()+ driver.execute_query = AsyncMock(side_effect=answer)+ return driver+++def reads(driver: MagicMock, query: str) -> list[list[str]]:+ """The uuids each read of ``query`` asked for, in order."""+ return [+ call.kwargs["uuids"]+ for call in driver.execute_query.await_args_list+ if call.args[0] == query+ ]diff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_sources_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_sources_test.pynew file mode 100644index 0000000000..5e14100066--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_sources_test.py@@ -0,0 +1,213 @@+"""Unit tests for ``recall_sources``: where a source a forget reached leads,+and the walk up the records from a cited derived fact no longer live+(``ancestry``), the cascade's walk-through rule upward, bounded like the+cascade. The live runs are in ``recall_ancestry_integration_test.py``.+"""++from unittest.mock import patch++import pytest++from . import recall_sources+from .recall_cascade import CASCADE_MAX_ROUNDS+from .recall_sources import Reach, ancestry, fact_reach, fact_states+from .recall_sources_fake import episode, fact, reads, sources+++async def _walk(driver, cited: list[str]) -> recall_sources.Ancestry:+ """The walk from the cited facts, as the check and the settle start it."""+ return await ancestry(driver, list((await fact_states(driver, cited)).values()))+++class TestTheWalk:+ @pytest.mark.asyncio+ async def test_a_derived_fact_no_longer_live_leads_to_the_fact_it_rests_on(+ self,+ ) -> None:+ """``d`` was superseded after the pass read it; ``f`` was forgotten."""+ driver = sources(+ {"d": fact(facts=("f",)), "f": fact(forgotten=True, reason="user_signal")}+ )++ walk = await _walk(driver, ["d"])++ assert walk.reached == {"d": Reach(root="f", hard=False)}+ assert not walk.unfinished++ @pytest.mark.asyncio+ async def test_it_goes_up_level_by_level_and_names_the_root(self) -> None:+ """``f`` was retracted by a cascade from ``root``."""+ driver = sources(+ {+ "d1": fact(facts=("d2",)),+ "d2": fact(facts=("f",)),+ "f": fact(forgotten=True, reason="derived_from_forgotten:root"),+ }+ )++ walk = await _walk(driver, ["d1"])++ assert walk.reached == {"d1": Reach(root="root", hard=False)}+ assert reads(driver, recall_sources.FACT_STATES_QUERY) == [+ ["d1"],+ ["d2"],+ ["f"],+ ]++ @pytest.mark.asyncio+ async def test_a_source_a_hard_forget_reached_leads_there_erasing(self) -> None:+ """``gone`` and ``purged`` were purged; ``h`` is still being purged;+ a hard reach wins over the soft one ``f`` gives."""+ driver = sources(+ {+ "d1": fact(facts=("gone",)),+ "d2": fact(facts=("h",)),+ "d3": fact(facts=("f", "purged")),+ "h": fact(forgotten=True, hard=True),+ "f": fact(forgotten=True),+ }+ )++ walk = await _walk(driver, ["d1", "d2", "d3"])++ assert walk.reached == {+ "d1": Reach(root="gone", hard=True),+ "d2": Reach(root="h", hard=True),+ "d3": Reach(root="purged", hard=True),+ }++ @pytest.mark.asyncio+ async def test_a_source_two_cited_facts_share_is_read_once_for_the_first(+ self,+ ) -> None:+ """Either caller needs one: the check drops the write, and the+ settle's cascade from any cited fact reaches every fact the write+ made, whose record names them all."""+ driver = sources(+ {+ "d1": fact(facts=("f",)),+ "d2": fact(facts=("f",)),+ "f": fact(forgotten=True),+ }+ )++ walk = await _walk(driver, ["d1", "d2"])++ assert walk.reached == {"d1": Reach(root="f", hard=False)}+ assert reads(driver, recall_sources.FACT_STATES_QUERY) == [["d1", "d2"], ["f"]]++ @pytest.mark.asyncio+ async def test_a_hidden_source_episode_leads_to_the_root_it_was_hidden_for(+ self,+ ) -> None:+ driver = sources(+ {+ "d1": fact(episodes=("turn",)),+ "d2": fact(episodes=("tomb",)),+ "f": fact(forgotten=True),+ },+ {+ "turn": episode(hidden=True, hidden_for=("f",)),+ "tomb": episode(hidden=True, hard=True),+ },+ )++ walk = await _walk(driver, ["d1", "d2"])++ assert walk.reached == {+ "d1": Reach(root="f", hard=False),+ "d2": Reach(root="tomb", hard=True),+ }++ @pytest.mark.asyncio+ async def test_a_live_fact_a_users_fact_or_a_recallable_episode_ends_it_clean(+ self,+ ) -> None:+ """A live source would have been retracted with its own source; a+ fact a user states, and what rests on it, a forget passes over."""+ driver = sources(+ {+ "d": fact(facts=("live", "users"), episodes=("turn",)),+ "live": fact(live=True, facts=("f",)),+ "users": fact(derived=False, facts=("f",)),+ "f": fact(forgotten=True),+ },+ {"turn": episode()},+ )++ walk = await _walk(driver, ["d"])++ assert (walk.reached, walk.unfinished) == ({}, False)+ assert reads(driver, recall_sources.FACT_STATES_QUERY) == [+ ["d"],+ ["live", "users"],+ ]++ @pytest.mark.asyncio+ async def test_only_a_derived_fact_no_longer_live_starts_it(self) -> None:+ """A live fact, one already forgotten (the check before the walk+ reads that) and a user's own fact read once, and nothing more."""+ driver = sources(+ {+ "live": fact(live=True, facts=("f",)),+ "gone-already": fact(forgotten=True, facts=("f",)),+ "users": fact(derived=False, facts=("f",)),+ }+ )++ walk = await _walk(driver, ["live", "gone-already", "users"])++ assert walk.reached == {}+ assert driver.execute_query.await_count == 1+++class TestTheBound:+ @pytest.mark.asyncio+ async def test_a_chain_deeper_than_the_cascades_rounds_is_unfinished(+ self,+ ) -> None:+ depth = CASCADE_MAX_ROUNDS + 1+ chain = {f"d{i}": fact(facts=(f"d{i + 1}",)) for i in range(depth)}+ driver = sources({**chain, f"d{depth}": fact(forgotten=True)})++ walk = await _walk(driver, ["d0"])++ assert (walk.reached, walk.unfinished) == ({}, True)+ assert len(reads(driver, recall_sources.FACT_STATES_QUERY)) == 1 + (+ CASCADE_MAX_ROUNDS+ )++ @pytest.mark.asyncio+ async def test_more_sources_than_the_cascades_budget_is_unfinished(+ self,+ ) -> None:+ driver = sources({"d": fact(facts=("p1", "p2", "p3"))})++ with patch.object(recall_sources, "CASCADE_MAX_ITEMS", 2):+ walk = await _walk(driver, ["d"])++ assert walk.unfinished+ assert reads(driver, recall_sources.FACT_STATES_QUERY) == [["d"]]+++class TestReach:+ def test_a_fact_gone_forgotten_or_still_there(self) -> None:+ assert fact_reach("gone", None) == Reach(root="gone", hard=True)+ forgotten = {"uuid": "f", **fact(forgotten=True, reason="user_signal")}+ assert fact_reach("f", forgotten) == Reach(root="f", hard=False)+ assert fact_reach("d", {"uuid": "d", **fact()}) is None++ def test_the_fact_read_carries_the_cascades_walk_through_rule(self) -> None:+ query = recall_sources.FACT_STATES_QUERY+ assert "stated.derived_from_facts IS NULL" in query+ assert (+ "e.derived_from_facts IS NOT NULL AND independent = 0 AS derived" in query+ )+ assert "e.hard_forgotten_at IS NOT NULL" in query++ def test_the_reads_write_nothing(self) -> None:+ for query in (+ recall_sources.FACT_STATES_QUERY,+ recall_sources.EPISODE_STATES_QUERY,+ ):+ assert "SET" not in query and "DELETE" not in querydiff --git a/autogpt_platform/backend/backend/copilot/graphiti/recall_test.py b/autogpt_platform/backend/backend/copilot/graphiti/recall_test.pyindex f490aaad93..335a90177e 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/recall_test.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/recall_test.py@@ -180,6 +180,7 @@ class TestRecallableEpisodePredicate:def test_cypher(self) -> None:assert recall.recallable_episode_predicate("ep", "gone") == ("ep.redacted_at IS NULL"+ " AND ep.write_pending IS NULL"" AND none(x IN coalesce(ep.entity_edges, []) WHERE x IN gone)")@@ -190,19 +191,22 @@ class TestRecallableEpisodePredicate:assert clause.rstrip().endswith("WITH collect(forgotten_fact.uuid) AS gone")@pytest.mark.parametrize(- ("entity_edges", "redacted", "recallable"),+ ("entity_edges", "redacted", "pending", "recallable"),[- (["e1", "e2"], False, True),- (["e1", "gone-1"], False, False), # one forgotten fact hides it all- ([], True, False), # stamped by a forget- ([], False, True),+ (["e1", "e2"], False, False, True),+ (["e1", "gone-1"], False, False, False), # one forgotten fact hides it+ ([], True, False, False), # stamped by a forget+ ([], False, True, False), # a dream write still being written+ ([], False, False, True),],)def test_is_recallable_episode(- self, entity_edges: list[str], redacted: bool, recallable: bool+ self, entity_edges: list[str], redacted: bool, pending: bool, recallable: bool) -> None:assert (- recall.is_recallable_episode(entity_edges, {"gone-1"}, redacted=redacted)+ recall.is_recallable_episode(+ entity_edges, {"gone-1"}, redacted=redacted, write_pending=pending+ )is recallable)diff --git a/autogpt_platform/backend/backend/copilot/graphiti/types.py b/autogpt_platform/backend/backend/copilot/graphiti/types.pyindex 9ba2858ee5..810995886d 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/types.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/types.py@@ -24,8 +24,10 @@ from .memory_model import ExtractedStatus, SourceKind# Edge properties only the platform writes, never graphiti's extraction: a# forget's marker and audit copies (``recall_forget.py``, ``recall_hide.py``),# the ratification stamp (``dream/ratification.py``), the web-verification-# stamp, kept for a verification path that does not exist yet, and the recall-# stamps (``recall_stamp.py``). No extraction model declares one+# stamp, kept for a verification path that does not exist yet, the recall+# stamps (``recall_stamp.py``) and the dream's derivation record+# (``recall_derivation.py``: an invented one would make a user's fact look+# derived, for a forget's cascade to take). No extraction model declares one# (``types_test.py``), so ``extracted_attributes`` drops any the model# invents; a stored one still reads back from the edge.POLICY_EDGE_FIELDS = frozenset(@@ -38,6 +40,8 @@ POLICY_EDGE_FIELDS = frozenset("recall_count","last_recalled_at","prev_recalled_at",+ "derived_from_facts",+ "derived_from_episodes",})diff --git a/autogpt_platform/backend/backend/copilot/inference/complete.py b/autogpt_platform/backend/backend/copilot/inference/complete.pyindex 32c208d4fc..e893e7be48 100644--- a/autogpt_platform/backend/backend/copilot/inference/complete.py+++ b/autogpt_platform/backend/backend/copilot/inference/complete.py@@ -130,7 +130,9 @@ async def _call_provider_sync(try:response = await _with_forced_tool_fallback(send, request)except Exception as exc:- raise InferenceError(f"LLM call failed: {type(exc).__name__}: {exc}") from exc+ raise InferenceError(+ f"LLM call failed: {type(exc).__name__}: {exc}", code="call_failed"+ ) from excif not isinstance(response, ProviderResponse):# ``call_provider`` returns a ``BatchSubmissionRef`` only fordiff --git a/autogpt_platform/backend/backend/copilot/inference/context.py b/autogpt_platform/backend/backend/copilot/inference/context.pyindex bf3c95f94e..53933b63fc 100644--- a/autogpt_platform/backend/backend/copilot/inference/context.py+++ b/autogpt_platform/backend/backend/copilot/inference/context.py@@ -125,6 +125,16 @@ class InferenceContext(BaseModel):trace_id: str | None = None+# How a background call failed, naming it without quoting the model.+InferenceErrorCode = Literal[+ "empty_content",+ "non_json_content",+ "schema_mismatch",+ "call_failed",+ "inference_failed",+]++class InferenceError(RuntimeError):"""A background call produced no usable answer.@@ -132,8 +142,17 @@ class InferenceError(RuntimeError):(empty content, unparseable JSON, a schema mismatch). Callers retain itfor telemetry and apply their own charging policy; failed dream phasesare absorbed. ``None`` means no call completed.++ ``code`` identifies the failure without quoting model content in durable records."""- def __init__(self, message: str, usage: InferenceUsage | None = None) -> None:+ def __init__(+ self,+ message: str,+ usage: InferenceUsage | None = None,+ *,+ code: InferenceErrorCode = "inference_failed",+ ) -> None:super().__init__(message)self.usage = usage+ self.code: InferenceErrorCode = codediff --git a/autogpt_platform/backend/backend/copilot/tools/graphiti_forget.py b/autogpt_platform/backend/backend/copilot/tools/graphiti_forget.pyindex 6d2934d228..286631f720 100644--- a/autogpt_platform/backend/backend/copilot/tools/graphiti_forget.py+++ b/autogpt_platform/backend/backend/copilot/tools/graphiti_forget.py@@ -53,6 +53,16 @@ _BUSY_RETRY_SECONDS = 5# size threshold (base.py) and losing all detail to truncation._MAX_FAILURE_DETAIL = 5+# What a soft forget did, told with its result: a model that read only "N+# memory edge(s) retracted" told a user who had asked for a permanent delete+# that it was permanent.+SOFT_FORGET_NOTE = (+ "This is not a permanent deletion: they are hidden from recall, from the "+ "memory context chats start with and from dreams, and their text is kept "+ "for audit. To delete them permanently, call tool:memory_forget_confirm "+ "with the same UUIDs and hard_delete=true."+)+logger = logging.getLogger(__name__)@@ -261,11 +271,18 @@ class MemoryForgetConfirmTool(BaseTool):session_id=session.session_id,)- mode = "permanently deleted" if hard_delete else "retracted from memory"return MemoryForgetConfirmResponse(- message=_build_confirm_message(len(result.deleted), mode, result.failures),+ message=_build_confirm_message(+ len(result.deleted),+ hard_delete,+ result.failures,+ len(result.derived),+ len(result.resumed),+ ),session_id=session.session_id,deleted_uuids=result.deleted,+ derived_uuids=result.derived,+ resumed_uuids=result.resumed,failed_uuids=[f.uuid for f in result.failures],failures=result.failures,)@@ -285,19 +302,42 @@ async def _retract_once_more_if_busy(def _build_confirm_message(- deleted_count: int, mode: str, failures: list[MemoryForgetFailure]+ deleted_count: int,+ hard: bool,+ failures: list[MemoryForgetFailure],+ derived_count: int = 0,+ resumed_count: int = 0,) -> str:"""Human/model-readable summary that spells out *why* edges failed.A bare "N failed" gives the model nothing to act on (SECRT-2371); listingeach UUID with its reason lets it retry, hard-delete, or tell the user.+ The facts the dream had derived from the forgotten ones, retracted with+ them (``graphiti/recall_cascade.py``), are counted so the user hears of+ them too, and so are the edges an earlier hard forget had already erased,+ whose derived facts this forget went on to retract and erase. A soft+ forget that retracted anything says it was not a permanent deletion, what+ it did instead, and how to delete permanently (``SOFT_FORGET_NOTE``).Only the first ``_MAX_FAILURE_DETAIL`` reasons are inlined: a large batchfailing wholesale (e.g. a driver outage) would otherwise push the tooloutput past the persist-and-summarize threshold and lose *all* detail. Thefull per-UUID list stays available in the structured ``failures`` field."""+ mode = "permanently deleted" if hard else "retracted from memory"summary = f"{deleted_count} memory edge(s) {mode}."+ if derived_count:+ summary = (+ f"{deleted_count} memory edge(s) {mode}; {derived_count} fact(s) "+ "derived from them retracted too."+ )+ if resumed_count:+ summary += (+ f" {resumed_count} memory edge(s) had already been erased; what "+ "was derived from them was retracted and erased."+ )+ if deleted_count and not hard:+ summary += f" {SOFT_FORGET_NOTE}"if not failures:return summaryshown = failures[:_MAX_FAILURE_DETAIL]diff --git a/autogpt_platform/backend/backend/copilot/tools/graphiti_forget_test.py b/autogpt_platform/backend/backend/copilot/tools/graphiti_forget_test.pyindex a539a0eb02..225db763b8 100644--- a/autogpt_platform/backend/backend/copilot/tools/graphiti_forget_test.py+++ b/autogpt_platform/backend/backend/copilot/tools/graphiti_forget_test.py@@ -23,6 +23,7 @@ from backend.copilot.graphiti.scope import MemoryScopefrom backend.copilot.model import ChatSessionfrom backend.copilot.tools.graphiti_forget import (_MAX_FAILURE_DETAIL,+ SOFT_FORGET_NOTE,MemoryForgetConfirmTool,MemoryForgetSearchTool,_build_confirm_message,@@ -52,9 +53,12 @@ def lock_redis(mocker) -> FakeRedis:def _mock_driver(*results) -> AsyncMock:"""A FalkorDB driver whose queries return ``results`` in order, for- driving the real ``retract`` from the confirm tool."""+ driving the real ``retract`` from the confirm tool, after the read of+ pending dream records every forget makes first (none here)."""driver = AsyncMock()- driver.execute_query.side_effect = [(r, [], None) for r in results]+ driver.execute_query.side_effect = [+ r if isinstance(r, Exception) else (r, [], None) for r in ([], *results)+ ]return driver@@ -304,6 +308,26 @@ class TestForgetConfirmModes:assert isinstance(response, MemoryForgetConfirmResponse)assert response.message == "1 memory edge(s) permanently deleted."+ @pytest.mark.asyncio+ async def test_the_facts_derived_from_it_are_reported(self) -> None:+ """The user hears of the dream's facts the forget retracted too."""+ result = ForgetResult(deleted=["e1"], derived=["d1", "d2", "d3"])+ session = ChatSession.new("user-abc", dry_run=False)+ with (+ patch(f"{_MODULE}.is_enabled_for_user", _enabled),+ patch(f"{_MODULE}.retract", AsyncMock(return_value=result)),+ ):+ response = await MemoryForgetConfirmTool()._execute(+ "user-abc", session, uuids=["e1"]+ )++ assert isinstance(response, MemoryForgetConfirmResponse)+ assert response.derived_uuids == ["d1", "d2", "d3"]+ assert response.message == (+ "1 memory edge(s) retracted from memory; "+ "3 fact(s) derived from them retracted too. " + SOFT_FORGET_NOTE+ )+@pytest.mark.asyncioasync def test_unavailable_graph_is_an_error_response(self) -> None:session = ChatSession.new("user-abc", dry_run=False)@@ -331,7 +355,7 @@ class TestForgetFailuresAreActionable:"""End to end: a soft delete that matches nothing must return theper-UUID reason in both the structured `failures` field and thehuman-readable message — not a bare "0 invalidated, 1 failed"."""- driver = _mock_driver([]) # the edge lookup finds nothing+ driver = _mock_driver([], []) # the lookup finds nothing, nothing names itsession = ChatSession.new("user-abc", dry_run=False)with (patch(f"{_MODULE}.is_enabled_for_user", _enabled),@@ -360,10 +384,16 @@ class TestForgetFailuresAreActionable:success count and the per-UUID failure detail."""driver = _mock_driver([{"uuid": "kept"}], # lookup: only "kept" exists+ [], # nothing names "gone" as a purged root[{"uuid": "kept"}], # retract "kept"[], # scrub its sentence[], # find the entities to scrub (none)[], # redact its episodes+ [], # the cascade: no earlier try,+ [], # no episode citing it,+ [], # nothing derived from it+ [],+ [{"count": 0}], # no dream write still in flight cites it)session = ChatSession.new("user-abc", dry_run=False)with (@@ -389,14 +419,13 @@ class TestForgetFailuresAreActionable:async def test_confirm_tool_reports_a_failed_clean_up(self) -> None:"""Retracted, but the episode redaction failed: the model is told thefact is forgotten and that the clean-up did not finish."""- driver = AsyncMock()- driver.execute_query.side_effect = [- ([{"uuid": "u1"}], [], None), # lookup- ([{"uuid": "u1"}], [], None), # retract- ([], [], None), # scrub its sentence- ([], [], None), # find the entities to scrub (none)+ driver = _mock_driver(+ [{"uuid": "u1"}], # lookup+ [{"uuid": "u1"}], # retract+ [], # scrub its sentence+ [], # find the entities to scrub (none)RuntimeError("down"), # redact its episodes- ]+ )session = ChatSession.new("user-abc", dry_run=False)with (patch(f"{_MODULE}.is_enabled_for_user", _enabled),@@ -425,8 +454,48 @@ class TestBuildConfirmMessage:threshold and lose all detail)."""def test_no_failures_returns_summary_only(self) -> None:- message = _build_confirm_message(3, "retracted from memory", [])- assert message == "3 memory edge(s) retracted from memory."+ message = _build_confirm_message(3, True, [])+ assert message == "3 memory edge(s) permanently deleted."++ def test_the_facts_derived_from_them_are_counted(self) -> None:+ message = _build_confirm_message(1, True, [], 3)+ assert message == (+ "1 memory edge(s) permanently deleted; "+ "3 fact(s) derived from them retracted too."+ )++ def test_edges_already_erased_are_told_apart(self) -> None:+ message = _build_confirm_message(0, False, [], 2, 1)+ assert message == (+ "0 memory edge(s) retracted from memory; "+ "2 fact(s) derived from them retracted too. "+ "1 memory edge(s) had already been erased; what was derived from "+ "them was retracted and erased."+ )++ def test_a_soft_forget_says_it_is_not_a_permanent_deletion(self) -> None:+ """A model told a user who had asked for a permanent delete that a soft+ forget was permanent. A soft forget's message says it was not, what it+ did instead and how to delete permanently; a hard forget's says none+ of that, and neither does a soft forget that retracted nothing."""+ soft = _build_confirm_message(1, False, [], 2)+ assert soft == (+ "1 memory edge(s) retracted from memory; 2 fact(s) derived from "+ "them retracted too. This is not a permanent deletion: they are "+ "hidden from recall, from the memory context chats start with and "+ "from dreams, and their text is kept for audit. To delete them "+ "permanently, call tool:memory_forget_confirm with the same UUIDs "+ "and hard_delete=true."+ )++ hard = _build_confirm_message(1, True, [], 2)+ assert hard == (+ "1 memory edge(s) permanently deleted; "+ "2 fact(s) derived from them retracted too."+ )++ nothing = _build_confirm_message(0, False, [])+ assert nothing == "0 memory edge(s) retracted from memory."def test_caps_inlined_detail_and_notes_remainder(self) -> None:overflow = _MAX_FAILURE_DETAIL + 4@@ -439,7 +508,7 @@ class TestBuildConfirmMessage:for i in range(overflow)]- message = _build_confirm_message(0, "retracted from memory", failures)+ message = _build_confirm_message(0, False, failures)# Full count is reported, but only the first N reasons are inlined.assert f"{overflow} failed" in messagediff --git a/autogpt_platform/backend/backend/copilot/tools/models.py b/autogpt_platform/backend/backend/copilot/tools/models.pyindex 69eea2c192..d2cb6c5cc9 100644--- a/autogpt_platform/backend/backend/copilot/tools/models.py+++ b/autogpt_platform/backend/backend/copilot/tools/models.py@@ -1413,6 +1413,12 @@ class MemoryForgetConfirmResponse(ToolResponseBase):type: ResponseType = ResponseType.MEMORY_FORGET_CONFIRMdeleted_uuids: list[str] = Field(default_factory=list)+ # Facts the dream derived from the deleted ones, retracted with them+ # (``graphiti/recall_cascade.py``).+ derived_uuids: list[str] = Field(default_factory=list)+ # Requested uuids already erased by an earlier hard forget whose cascade+ # this one went on with (``graphiti/recall_forget.py``).+ resumed_uuids: list[str] = Field(default_factory=list)failed_uuids: list[str] = Field(default_factory=list)failures: list[MemoryForgetFailure] = Field(default_factory=list)diff --git a/autogpt_platform/backend/backend/data/db.py b/autogpt_platform/backend/backend/data/db.pyindex 0b75ab5717..586a50b780 100644--- a/autogpt_platform/backend/backend/data/db.py+++ b/autogpt_platform/backend/backend/data/db.py@@ -117,6 +117,20 @@ async def transaction(timeout: timedelta = TRANSACTION_TIMEOUT):yield tx+@asynccontextmanager+async def bounded_transaction(statement_timeout_ms: int):+ """A transaction each of whose statements the database gives up on+ after *statement_timeout_ms* (``statement_timeout``, for this transaction+ only), cancelling it there, where a caller's own deadline only stops+ waiting for it."""+ async with transaction() as tx:+ await tx.query_raw(+ "SELECT set_config('statement_timeout', $1, true)",+ str(statement_timeout_ms),+ )+ yield tx++def get_database_schema() -> str:"""Extract database schema from DATABASE_URL."""parsed_url = urlparse(DATABASE_URL)diff --git a/autogpt_platform/backend/backend/data/db_manager.py b/autogpt_platform/backend/backend/data/db_manager.pyindex 458089d39f..9e40cc6a7f 100644--- a/autogpt_platform/backend/backend/data/db_manager.py+++ b/autogpt_platform/backend/backend/data/db_manager.py@@ -764,6 +764,8 @@ class DatabaseManager(AppService):get_next_sequence = _(chat_db.get_next_sequence)update_tool_message_content = _(chat_db.update_tool_message_content)update_message_content_by_sequence = _(chat_db.update_message_content_by_sequence)+ add_dream_narrative = _(chat_db.add_dream_narrative)+ blank_dream_narratives = _(chat_db.blank_dream_narratives)update_chat_message_stamps = _(chat_db.update_chat_message_stamps)update_chat_message_tool_calls = _(chat_db.update_chat_message_tool_calls)update_chat_session_title = _(chat_db.update_chat_session_title)@@ -793,6 +795,7 @@ class DatabaseManager(AppService):list_expired_dream_passes = _(dream_pass_db.list_expired_dream_passes)list_dream_pass_cleanups = _(dream_pass_db.list_dream_pass_cleanups)delete_old_dream_passes = _(dream_pass_db.delete_old_dream_passes)+ erase_dream_passes = _(dream_pass_db.erase_dream_passes)# ============ Morning Briefing ============ ## Exposed so the Prisma-less scheduler process can compose, store and@@ -1286,6 +1289,8 @@ class DatabaseManagerAsyncClient(AppServiceClient):get_next_sequence = d.get_next_sequenceupdate_tool_message_content = d.update_tool_message_contentupdate_message_content_by_sequence = d.update_message_content_by_sequence+ add_dream_narrative = d.add_dream_narrative+ blank_dream_narratives = d.blank_dream_narrativesupdate_chat_message_stamps = d.update_chat_message_stampsupdate_chat_message_tool_calls = d.update_chat_message_tool_callsupdate_chat_session_title = d.update_chat_session_title@@ -1311,3 +1316,4 @@ class DatabaseManagerAsyncClient(AppServiceClient):list_expired_dream_passes = d.list_expired_dream_passeslist_dream_pass_cleanups = d.list_dream_pass_cleanupsdelete_old_dream_passes = d.delete_old_dream_passes+ erase_dream_passes = d.erase_dream_passesdiff --git a/autogpt_platform/backend/backend/data/db_manager_test.py b/autogpt_platform/backend/backend/data/db_manager_test.pyindex ed8d2a663f..cbd4c3c478 100644--- a/autogpt_platform/backend/backend/data/db_manager_test.py+++ b/autogpt_platform/backend/backend/data/db_manager_test.py@@ -68,6 +68,9 @@ _DREAM_PASS_METHODS = ("list_expired_dream_passes","list_dream_pass_cleanups","delete_old_dream_passes",+ "erase_dream_passes",+ "add_dream_narrative",+ "blank_dream_narratives",)diff --git a/autogpt_platform/backend/backend/data/dream_pass.py b/autogpt_platform/backend/backend/data/dream_pass.pyindex 206eff3216..ee3b7621d3 100644--- a/autogpt_platform/backend/backend/data/dream_pass.py+++ b/autogpt_platform/backend/backend/data/dream_pass.py@@ -10,7 +10,9 @@ forward and merges its JSON columns in the database.A pass that has reached a terminal status (complete, errored, cancelled,expired, skipped) is final: no update writes it, the way``job_status.mark_complete`` never rewrites a finished job, but for the-cleanup after it marking itself finished.+cleanup after it marking itself finished, and a hard forget marking it and+blanking the dream's text in it (``erase_dream_passes``). Its read set+(``readUuids``) only ever grows, and outlives its input bundle.Three queries work across users: the reaper lists open passes whose leaselapsed (``list_expired_dream_passes``, on the status and lease expiry index)@@ -25,10 +27,18 @@ from datetime import datetimeimport prisma.modelsfrom prisma.types import DreamPassCreateInput, DreamPassWhereInput-from backend.data.db import execute_raw_with_schema+from backend.copilot.dream.pass_errors import PASS_ERROR_CODES, PASS_ERROR_STAGES+from backend.data.db import (+ bounded_transaction,+ execute_raw_with_schema,+ query_raw_with_schema,+)from backend.data.dream_pass_models import (CLOSED_STATUSES,+ DREAM_TEXT_FIELDS,+ ERASED_TEXT,OPEN_STATUSES,+ UNKNOWN_READS,DreamPassDraft,DreamPassRecord,DreamPassUpdate,@@ -49,6 +59,81 @@ DELETE FROM {schema_prefix}"DreamPass" WHERE "id" IN ()"""+# A hard forget's marks, for one user's scope ($1, $2), in one statement+# over every pass of the scope: no page or limit can leave one out. A pass is+# marked when it read what the forget reached (its read set, matched against+# the reached uuids, $4, by a hash join), may have read anything (its read+# set holds UNKNOWN_READS, $10: a pass from before read sets were recorded),+# may still be writing (it is open, $5, or closed with its cleanup pending: a+# stop from outside closes the row while the pass may still be applying), or+# wrote what the forget erased ($3, the passes its erased dream writes name:+# a pass's own write is not in its read set). A pass marked by an earlier+# forget is returned again, so a forget repeated after its narratives' blank+# failed finishes it, but not written again: it was blanked when marked+# closed, or as it closed. One marked now that is closed as this statement+# writes it (its status read from the row being written, so one closing+# meanwhile is not missed) is blanked now; an open one when it closes+# (``dream_pass_update.py``). Every pass of the scope keeps its error only+# when it is one of the fixed codes ($8, $9), marked or not. Returns the ids+# marked, now or before.+ERASE_SQL = """+WITH reached AS (+ SELECT DISTINCT uuid FROM unnest($4::text[]) AS reached(uuid)+),+read_reached AS (+ SELECT DISTINCT pass."id"+ FROM {schema_prefix}"DreamPass" AS pass,+ unnest(pass."readUuids") AS read(uuid)+ WHERE pass."userId" = $1 AND pass."scopeKey" = $2+ AND read.uuid IN (SELECT uuid FROM reached)+),+target AS (+ SELECT "id",+ ("erasedAt" IS NOT NULL+ OR "status"::text = ANY($5::text[])+ OR "cleanupPendingAt" IS NOT NULL+ OR "id" IN (SELECT unnest($3::text[]))+ OR "id" IN (SELECT "id" FROM read_reached)+ OR $10::text = ANY("readUuids")) AS reached+ FROM {schema_prefix}"DreamPass"+ WHERE "userId" = $1 AND "scopeKey" = $2+),+written AS (+ UPDATE {schema_prefix}"DreamPass" AS pass SET+ "erasedAt" = CASE WHEN target.reached+ THEN COALESCE(pass."erasedAt", CURRENT_TIMESTAMP AT TIME ZONE 'UTC')+ ELSE pass."erasedAt" END,+ "phaseOutputs" = CASE+ WHEN target.reached AND pass."erasedAt" IS NULL+ AND NOT (pass."status"::text = ANY($5::text[]))+ THEN {schema_prefix}"dream_text_blanked"(+ pass."phaseOutputs", $6::text[], $7::text)+ ELSE pass."phaseOutputs" END,+ "operations" = CASE+ WHEN target.reached AND pass."erasedAt" IS NULL+ AND NOT (pass."status"::text = ANY($5::text[]))+ THEN {schema_prefix}"dream_text_blanked"(+ pass."operations", $6::text[], $7::text)+ ELSE pass."operations" END,+ "error" = {schema_prefix}"dream_error_blanked"(+ pass."error", $8::text[], $9::text[], $7::text),+ "updatedAt" = CURRENT_TIMESTAMP AT TIME ZONE 'UTC'+ FROM target+ WHERE pass."id" = target."id"+ AND ((target.reached AND pass."erasedAt" IS NULL)+ OR pass."error" IS DISTINCT FROM {schema_prefix}"dream_error_blanked"(+ pass."error", $8::text[], $9::text[], $7::text))+ RETURNING pass."id"+)+SELECT "id" FROM target WHERE reached+"""++# The erase's own deadline in the database, just under the store's for the+# call (``copilot/dream/store.RECORD_WRITE_TIMEOUT_SECONDS``): the statement+# is cancelled there before the caller stops waiting for it, and the forget+# reports a cleanup error to repeat.+ERASE_STATEMENT_TIMEOUT_MS = 9_000+async def create_dream_pass(draft: DreamPassDraft) -> DreamPassRecord:data: DreamPassCreateInput = {@@ -63,6 +148,8 @@ async def create_dream_pass(draft: DreamPassDraft) -> DreamPassRecord:"startedAt": draft.started_at,"leaseToken": draft.lease_token,"leaseExpiresAt": draft.lease_expires_at,+ # Nothing read yet: the column's default is UNKNOWN_READS.+ "readUuids": [],}if draft.cost_context is not None:data["costContext"] = SafeJson(draft.cost_context)@@ -203,6 +290,36 @@ async def list_dream_pass_cleanups(return [DreamPassRecord.from_db(row) for row in rows]+async def erase_dream_passes(+ user_id: str, scope_key: str, *, reached: list[str], named: list[str]+) -> list[str]:+ """Mark, for *user_id*'s hard forget, every pass of the scope that read+ what it reached (*reached*, uuids) or may have read anything (from before+ read sets were recorded), may still be writing, or wrote what it erased+ (*named*, pass ids), blanking those already closed, and keep every+ error of the scope only as a fixed code (``ERASE_SQL``,+ ``copilot/dream/forget_copies.py``). The ids of the passes marked, now or+ by an earlier forget, the open ones included; repeating it writes nothing+ more. Raises when the statement runs past ``ERASE_STATEMENT_TIMEOUT_MS``+ or a lock it waits on outlasts it."""+ async with bounded_transaction(ERASE_STATEMENT_TIMEOUT_MS) as tx:+ rows = await query_raw_with_schema(+ ERASE_SQL,+ user_id,+ scope_key,+ named,+ reached,+ [status.value for status in OPEN_STATUSES],+ list(DREAM_TEXT_FIELDS),+ ERASED_TEXT,+ list(PASS_ERROR_STAGES),+ list(PASS_ERROR_CODES),+ UNKNOWN_READS,+ client=tx,+ )+ return [row["id"] for row in rows]++async def delete_old_dream_passes(created_before: datetime, limit: int = 1000) -> int:"""Delete at most *limit* closed passes, of every user, created before*created_before*, and say how many went. An open pass is never deleted,diff --git a/autogpt_platform/backend/backend/data/dream_pass_blank_test.py b/autogpt_platform/backend/backend/data/dream_pass_blank_test.pynew file mode 100644index 0000000000..ab102d0a48--- /dev/null+++ b/autogpt_platform/backend/backend/data/dream_pass_blank_test.py@@ -0,0 +1,614 @@+"""A hard forget's writes to the DreamPass table against a real database+(``copilot/dream/forget_copies.py``): which of a scope's passes+``erase_dream_passes`` marks and blanks (a pass from before read sets were+recorded with any forget in its scope), never another user's or scope's,+and returns again once marked; every error kept only as a fixed code; its+statement timeout; its cost over 3,000 passes; and every transition that+closes a marked pass blanking it in the same statement+(``dream_pass_update.py``), one that leaves it open or an unmarked pass+untouched, the read set only growing, and none of them touching a pass's+usage attribution."""++import asyncio+import json+import time+import uuid+from collections.abc import AsyncIterator, Awaitable, Callable+from datetime import datetime, timezone+from typing import Any++import pytest+from prisma.enums import (+ DreamPassPhase,+ DreamPassRoute,+ DreamPassStatus,+ DreamPassTrigger,+)+from prisma.models import DreamPass, User++from backend.copilot.dream.pass_errors import UNRECORDED+from backend.copilot.dream.pass_record import (+ cancelled,+ cleanup_finished,+ expired,+ failed,+ reaped,+)+from backend.copilot.dream.schemas import (+ ConsolidatedFact,+ ConsolidationOutput,+ DreamOperations,+)+from backend.data import db, dream_pass+from backend.util.json import SafeJson++from .dream_pass import (+ ERASE_STATEMENT_TIMEOUT_MS,+ create_dream_pass,+ erase_dream_passes,+ get_dream_pass,+ update_dream_pass,+)+from .dream_pass_models import (+ ERASED_TEXT,+ UNKNOWN_READS,+ DreamPassApplied,+ DreamPassDraft,+ DreamPassOperations,+ DreamPassUpdate,+ DreamPhaseOutputs,+)++pytestmark = pytest.mark.asyncio(loop_scope="session")++_NOW = datetime.now(timezone.utc).replace(microsecond=0)+_SENTENCE = "Alice runs the Atlas launch"+_FORGOTTEN = "fact-forgotten"+++async def make_user() -> AsyncIterator[Callable[[], Awaitable[str]]]:+ created: list[str] = []++ async def make() -> str:+ user_id = str(uuid.uuid4())+ await User.prisma().create(+ data={+ "id": user_id,+ "email": f"dream-blank-{user_id}@example.com",+ "topUpConfig": SafeJson({}),+ "timezone": "UTC",+ }+ )+ created.append(user_id)+ return user_id++ yield make+ await User.prisma().delete_many(where={"id": {"in": created}})+++def _outputs(content: str, cites: str = _FORGOTTEN) -> DreamPhaseOutputs:+ write = ConsolidatedFact(content=content, confidence=0.8, source_fact_uuids=[cites])+ return DreamPhaseOutputs(+ consolidate=ConsolidationOutput(facts=[write]),+ sanitize=DreamOperations(writes=[write], summary_for_user=content),+ )+++def _operations(content: str) -> DreamPassOperations:+ return DreamPassOperations(+ planned=DreamOperations(summary_for_user=content),+ applied=DreamPassApplied(consolidated_count=1, summary_for_user=content),+ )+++async def _pass(+ owner: str,+ scope_key: str,+ *,+ status: DreamPassStatus = DreamPassStatus.RUNNING,+ cites: str = _FORGOTTEN,+ cost_context: dict[str, Any] | None = None,+) -> str:+ """A pass that read ``cites`` and consolidated ``_SENTENCE`` citing it,+ then moved to ``status`` (closed with no cleanup pending, for a closed+ status)."""+ draft = DreamPassDraft(+ id=str(uuid.uuid4()),+ user_id=owner,+ scope_key=scope_key,+ route=DreamPassRoute.SYNC,+ trigger=DreamPassTrigger.CRON,+ started_at=_NOW,+ cost_context=cost_context,+ )+ await create_dream_pass(draft)+ assert await update_dream_pass(+ draft.id,+ DreamPassUpdate(+ read_uuids=[cites],+ phase_outputs=_outputs(_SENTENCE, cites),+ operations=_operations(_SENTENCE),+ ),+ )+ if status != DreamPassStatus.RUNNING:+ await DreamPass.prisma().update(+ where={"id": draft.id},+ data={"status": status, "phase": DreamPassPhase.DONE},+ )+ return draft.id+++async def _legacy_pass(owner: str, scope_key: str) -> str:+ """A closed pass written as a release without read sets wrote one: its+ insert names no read set, so the column's default applies, as it does to+ every row that existed when the column was added."""+ pass_id = str(uuid.uuid4())+ await db.execute_raw_with_schema(+ 'INSERT INTO {schema_prefix}"DreamPass" ("id", "userId", "scopeKey", '+ '"route", "trigger", "status", "phase", "updatedAt") '+ "VALUES ($1, $2, $3, 'SYNC', 'CRON', 'RUNNING', 'GATHER', now())",+ pass_id,+ owner,+ scope_key,+ )+ assert await update_dream_pass(+ pass_id,+ DreamPassUpdate(+ phase_outputs=_outputs(_SENTENCE, "fact-unrelated"),+ operations=_operations(_SENTENCE),+ ),+ )+ await DreamPass.prisma().update(+ where={"id": pass_id},+ data={"status": DreamPassStatus.COMPLETE, "phase": DreamPassPhase.DONE},+ )+ return pass_id+++async def _text(pass_id: str) -> str:+ row = await get_dream_pass(pass_id)+ assert row is not None+ return row.phase_outputs.model_dump_json() + row.operations.model_dump_json()+++async def test_a_closed_pass_that_read_a_reached_uuid_is_marked_and_blanked(+ make_user,+):+ owner = await make_user()+ pass_id = await _pass(owner, owner, status=DreamPassStatus.COMPLETE)++ marked = await erase_dream_passes(owner, owner, reached=[_FORGOTTEN], named=[])++ row = await get_dream_pass(pass_id)+ assert marked == [pass_id] and row is not None and row.erased_at is not None+ assert _SENTENCE not in await _text(pass_id)+ assert row.phase_outputs.consolidate is not None+ [write] = row.phase_outputs.consolidate.facts+ assert (write.content, write.source_fact_uuids) == (ERASED_TEXT, [_FORGOTTEN])+ assert row.operations.applied is not None+ assert row.operations.applied.consolidated_count == 1, "counts stay"+ assert row.status is DreamPassStatus.COMPLETE, "nothing else moves"+++async def test_an_open_pass_is_marked_and_kept_until_it_closes(make_user):+ """Its next phase reads its row; whatever it names, it is marked."""+ owner = await make_user()+ running = await _pass(owner, owner, cites="fact-unrelated")++ assert await erase_dream_passes(owner, owner, reached=[_FORGOTTEN], named=[]) == [+ running+ ]++ row = await get_dream_pass(running)+ assert row is not None and row.erased_at is not None+ assert _SENTENCE in await _text(running)+++async def test_a_named_pass_or_one_whose_cleanup_is_pending_is_marked(make_user):+ """A dream write the forget erased names its pass, which may cite nothing+ else it reached; a pass closed from outside with its cleanup pending may+ still be applying."""+ owner = await make_user()+ named = await _pass(+ owner, owner, status=DreamPassStatus.COMPLETE, cites="fact-unrelated"+ )+ pending = await _pass(+ owner, owner, status=DreamPassStatus.CANCELLED, cites="fact-unrelated"+ )+ await DreamPass.prisma().update(+ where={"id": pending}, data={"cleanupPendingAt": _NOW}+ )++ marked = await erase_dream_passes(owner, owner, reached=[_FORGOTTEN], named=[named])++ assert sorted(marked) == sorted([named, pending])+ for pass_id in (named, pending):+ assert _SENTENCE not in await _text(pass_id)+++async def test_a_pass_not_reached_another_scopes_or_another_users_is_left(+ make_user,+):+ owner, stranger = await make_user(), await make_user()+ unreached = await _pass(+ owner, owner, status=DreamPassStatus.COMPLETE, cites="fact-unrelated"+ )+ other_scope = await _pass(owner, f"{owner}:expert", status=DreamPassStatus.COMPLETE)+ theirs = await _pass(stranger, stranger, status=DreamPassStatus.COMPLETE)+ their_open = await _pass(stranger, stranger)++ assert await erase_dream_passes(owner, owner, reached=[_FORGOTTEN], named=[]) == []++ for pass_id in (unreached, other_scope, theirs, their_open):+ row = await get_dream_pass(pass_id)+ assert row is not None and row.erased_at is None+ assert _SENTENCE in await _text(pass_id)+++async def test_a_pass_from_before_read_sets_goes_with_any_forget_in_its_scope(+ make_user,+):+ """What such a pass read is unknown, so it may rest on anything: any hard+ forget in its scope marks it and blanks its text, whatever it reached and+ whatever its writes cite. One with a known read set holding none of it+ is left, and so is an unknown one of another scope or user."""+ owner, stranger = await make_user(), await make_user()+ legacy = await _legacy_pass(owner, owner)+ other_scope = await _legacy_pass(owner, f"{owner}:expert")+ theirs = await _legacy_pass(stranger, stranger)+ known = await _pass(+ owner, owner, status=DreamPassStatus.COMPLETE, cites="fact-unrelated"+ )+ row = await get_dream_pass(legacy)+ assert row is not None and row.read_uuids == [UNKNOWN_READS]++ marked = await erase_dream_passes(owner, owner, reached=[_FORGOTTEN], named=[])++ row = await get_dream_pass(legacy)+ assert marked == [legacy] and row is not None and row.erased_at is not None+ assert _SENTENCE not in await _text(legacy)+ for pass_id in (other_scope, theirs, known):+ row = await get_dream_pass(pass_id)+ assert row is not None and row.erased_at is None+ assert _SENTENCE in await _text(pass_id)+++async def test_a_row_whose_writer_names_no_read_set_may_have_read_anything(+ make_user,+):+ """The column's default, which every row that existed when the column+ was added reads, and every row a release without it writes during a+ rollout: ``UNKNOWN_READS``."""+ owner = await make_user()++ row = await get_dream_pass(await _legacy_pass(owner, owner))++ assert row is not None and row.read_uuids == [UNKNOWN_READS]+++async def test_a_pass_this_release_creates_has_read_nothing_until_it_gathers(+ make_user,+):+ """Its create states its read set, so it never reads ``UNKNOWN_READS``:+ gathering adds what it read, and only that reaches it once it closes."""+ owner = await make_user()+ draft = DreamPassDraft(+ id=str(uuid.uuid4()),+ user_id=owner,+ scope_key=owner,+ route=DreamPassRoute.SYNC,+ trigger=DreamPassTrigger.CRON,+ started_at=_NOW,+ )++ created = await create_dream_pass(draft)+ assert created.read_uuids == []+ assert await update_dream_pass(draft.id, DreamPassUpdate(read_uuids=["fact-a"]))+ await DreamPass.prisma().update(+ where={"id": draft.id},+ data={"status": DreamPassStatus.COMPLETE, "phase": DreamPassPhase.DONE},+ )++ row = await get_dream_pass(draft.id)+ assert row is not None and row.read_uuids == ["fact-a"]+ assert await erase_dream_passes(owner, owner, reached=[_FORGOTTEN], named=[]) == []+++async def test_every_error_of_the_scope_is_kept_only_as_a_fixed_code(make_user):+ """An error the store did not write (it records codes only) may quote+ anything and names no uuid: every pass of the scope, reached or not and+ whatever its status, keeps its error only when it is one of the fixed+ codes, its stage when that is one; another scope's error stays. The+ review's probe, ``apply: SecretToken123``, looks like a code and is not."""+ owner = await make_user()+ errored = DreamPassStatus.ERRORED+ old = await _pass(owner, owner, status=errored, cites="x")+ coded = await _pass(owner, owner, status=errored, cites="x")+ lookalike = await _pass(owner, owner, status=errored, cites="x")+ bare = await _pass(owner, owner, status=errored, cites="x")+ stopped = await _pass(owner, owner, status=DreamPassStatus.CANCELLED, cites="x")+ elsewhere = await _pass(owner, f"{owner}:expert", status=errored, cites="x")+ quoted = f"recombine: LLM returned non-JSON content — first 200 chars: {_SENTENCE}"+ for pass_id, error in (+ (old, quoted),+ (coded, "consolidate: schema_mismatch"),+ (lookalike, "apply: SecretToken123"),+ (bare, f"1 validation error: input_value='{_SENTENCE}'"),+ (stopped, f"cancelled by an admin: {_SENTENCE}"),+ (elsewhere, quoted),+ ):+ await DreamPass.prisma().update(where={"id": pass_id}, data={"error": error})++ assert await erase_dream_passes(owner, owner, reached=[_FORGOTTEN], named=[]) == []++ errors = {+ pass_id: (row.error if (row := await get_dream_pass(pass_id)) else None)+ for pass_id in (old, coded, lookalike, bare, stopped, elsewhere)+ }+ assert errors == {+ old: f"recombine: {ERASED_TEXT}",+ coded: "consolidate: schema_mismatch",+ lookalike: f"apply: {ERASED_TEXT}",+ bare: ERASED_TEXT,+ stopped: ERASED_TEXT,+ elsewhere: quoted,+ }+++async def test_an_error_that_is_not_a_fixed_code_is_never_recorded(make_user):+ owner = await make_user()+ pass_id = await _pass(owner, owner)++ assert await update_dream_pass(+ pass_id, failed(f"recombine: the model said {_SENTENCE}", None, _NOW)+ )++ row = await get_dream_pass(pass_id)+ assert row is not None and row.error == UNRECORDED+++async def test_every_pass_of_the_scope_is_reached_however_many(make_user):+ """The review's 1,001 passes: the affected one is the oldest, behind a+ thousand newer, and no page or limit leaves it out."""+ owner = await make_user()+ oldest = await _pass(owner, owner, status=DreamPassStatus.COMPLETE)+ await DreamPass.prisma().create_many(+ data=[+ {+ "id": str(uuid.uuid4()),+ "userId": owner,+ "scopeKey": owner,+ "route": DreamPassRoute.SYNC,+ "trigger": DreamPassTrigger.ADMIN,+ "status": DreamPassStatus.SKIPPED,+ "phase": DreamPassPhase.DONE,+ "readUuids": [],+ }+ for _ in range(1000)+ ]+ )++ assert await erase_dream_passes(owner, owner, reached=[_FORGOTTEN], named=[]) == [+ oldest+ ]+ assert _SENTENCE not in await _text(oldest)+++async def test_a_pass_marked_before_is_returned_and_not_written_again(make_user):+ """The round-2 review's retry: a later forget, whatever it reached,+ returns every pass marked before so its narrative is blanked again, and+ leaves the row as it was (it was blanked as it closed or was marked)."""+ owner = await make_user()+ pass_id = await _pass(owner, owner, status=DreamPassStatus.COMPLETE)+ assert await erase_dream_passes(owner, owner, reached=[_FORGOTTEN], named=[]) == [+ pass_id+ ]+ first = await get_dream_pass(pass_id)++ again = await erase_dream_passes(owner, owner, reached=["fact-other"], named=[])++ row = await get_dream_pass(pass_id)+ assert again == [pass_id]+ assert first is not None and row is not None+ assert row == first, "not even updatedAt moves"+++async def test_the_read_set_only_grows_and_outlives_the_bundle(make_user):+ owner = await make_user()+ pass_id = await _pass(owner, owner, cites="fact-a")++ assert await update_dream_pass(+ pass_id, DreamPassUpdate(read_uuids=["fact-c", "fact-a", "episode-b"])+ )+ assert await update_dream_pass(pass_id, _CLOSES["applied"])++ row = await get_dream_pass(pass_id)+ assert row is not None and row.read_uuids == ["episode-b", "fact-a", "fact-c"]+++async def test_an_erase_waiting_on_a_locked_row_gives_up_at_its_timeout(+ make_user, monkeypatch+):+ """The database cancels the statement itself, just under the store's+ deadline, rather than leaving it running after the caller gave up; the+ forget reports a cleanup error to repeat."""+ monkeypatch.setattr(dream_pass, "ERASE_STATEMENT_TIMEOUT_MS", 300)+ owner = await make_user()+ pass_id = await _pass(owner, owner, status=DreamPassStatus.COMPLETE)++ async with db.transaction() as tx:+ await db.query_raw_with_schema(+ 'SELECT 1 FROM {schema_prefix}"DreamPass" WHERE "id" = $1 FOR UPDATE',+ pass_id,+ client=tx,+ )+ started = time.monotonic()+ with pytest.raises(Exception, match="statement timeout"):+ await asyncio.wait_for(+ erase_dream_passes(owner, owner, reached=[_FORGOTTEN], named=[]), 10+ )+ assert time.monotonic() - started < 5++ row = await get_dream_pass(pass_id)+ assert row is not None and row.erased_at is None, "nothing was written"+++async def test_the_erase_over_3000_passes_and_1000_reached_uuids_is_in_time(+ make_user,+):+ """The round-2 review's cost check, by the gather's caps: 3,000 closed+ passes of a scope, each with ~10 KB of phase output and 550 uuids read,+ against 1,000 reached uuids, first read by none of them (the selection's+ cost), then one read by every pass (all 3,000 marked and blanked in one+ statement), well inside the statement's timeout."""+ owner = await make_user()+ shared = "fact-read-by-every-pass"+ facts = [+ {+ "content": "A sample retained dream sentence. " * 12,+ "confidence": 0.9,+ "source_fact_uuids": [str(uuid.uuid4())],+ }+ for _ in range(20)+ ]+ await db.execute_raw_with_schema(+ 'INSERT INTO {schema_prefix}"DreamPass" ("id", "userId", "scopeKey", '+ '"route", "trigger", "status", "phase", "phaseOutputs", "readUuids") '+ "SELECT gen_random_uuid()::text, $1, $1, 'SYNC', 'CRON', 'COMPLETE', "+ "'DONE', $2::jsonb, ARRAY(SELECT gen_random_uuid()::text "+ "FROM generate_series(1, 549) WHERE pass.n > 0) || $3::text "+ "FROM generate_series(1, 3000) AS pass(n)",+ owner,+ json.dumps({"consolidate": {"facts": facts}}),+ shared,+ )+ budget = ERASE_STATEMENT_TIMEOUT_MS / 1000 / 2+ unread = [str(uuid.uuid4()) for _ in range(1000)]++ started = time.monotonic()+ assert await erase_dream_passes(owner, owner, reached=unread, named=[]) == []+ selection = time.monotonic() - started+ started = time.monotonic()+ marked = await erase_dream_passes(+ owner, owner, reached=[*unread[1:], shared], named=[]+ )+ all_marked = time.monotonic() - started++ assert len(marked) == 3000+ assert selection < budget and all_marked < budget, (selection, all_marked)+ blanked = await DreamPass.prisma().count(+ where={"userId": owner, "erasedAt": {"not": None}}+ )+ assert blanked == 3000+++_CLOSES: dict[str, DreamPassUpdate] = {+ "applied": DreamPassUpdate(+ status=DreamPassStatus.COMPLETE,+ phase=DreamPassPhase.DONE,+ operations=DreamPassOperations(+ applied=DreamPassApplied(summary_for_user=f"I noticed {_SENTENCE}.")+ ),+ completed_at=_NOW,+ ),+ "failed": failed("recombine: non_json_content", None, _NOW, marked=True),+ "expired": expired("expired: forced", not_updated_since=None),+ "reaped": reaped("recombine: reaped", not_updated_since=_NOW.replace(year=2100)),+}++[email protected]("close", [*sorted(_CLOSES), "cancelled"])+async def test_every_transition_that_closes_a_marked_pass_blanks_it(make_user, close):+ """The pass's own end (applied or failed, the update's own text+ included), a cancel, an expiry and the reaper's close: each blanks the+ marked row in the statement that closes it."""+ owner = await make_user()+ pass_id = await _pass(owner, owner, cites="fact-unrelated")+ await erase_dream_passes(owner, owner, reached=[_FORGOTTEN], named=[])+ await DreamPass.prisma().update(+ where={"id": pass_id}, data={"error": f"consolidate: said {_SENTENCE}"}+ )+ update = _CLOSES.get(close) or cancelled("cancelled: by_admin", owner_user_id=owner)++ assert await update_dream_pass(pass_id, update)++ row = await get_dream_pass(pass_id)+ assert row is not None and row.status is not DreamPassStatus.RUNNING+ assert _SENTENCE not in await _text(pass_id)+ assert _SENTENCE not in (row.error or "")+++async def test_the_cleanups_end_blanks_a_marked_pass_again(make_user):+ owner = await make_user()+ pass_id = await _pass(owner, owner, status=DreamPassStatus.COMPLETE)+ await DreamPass.prisma().update(+ where={"id": pass_id},+ data={"cleanupPendingAt": _NOW, "erasedAt": _NOW},+ )++ assert await update_dream_pass(pass_id, cleanup_finished())++ assert _SENTENCE not in await _text(pass_id)+++async def test_an_update_that_leaves_a_marked_pass_open_keeps_its_text(make_user):+ """Its next phase reads what it wrote."""+ owner = await make_user()+ pass_id = await _pass(owner, owner)+ await erase_dream_passes(owner, owner, reached=[_FORGOTTEN], named=[])++ assert await update_dream_pass(+ pass_id,+ DreamPassUpdate(+ phase=DreamPassPhase.RECOMBINE,+ phase_outputs=DreamPhaseOutputs(+ consolidate=ConsolidationOutput(+ facts=[+ ConsolidatedFact(+ content=f"{_SENTENCE}, again",+ confidence=0.8,+ source_fact_uuids=["x"],+ )+ ]+ )+ ),+ ),+ )++ assert f"{_SENTENCE}, again" in await _text(pass_id)+++async def test_an_unmarked_pass_closes_as_it_did(make_user):+ owner = await make_user()+ pass_id = await _pass(owner, owner)++ assert await update_dream_pass(pass_id, _CLOSES["applied"])++ row = await get_dream_pass(pass_id)+ assert row is not None and row.erased_at is None+ assert _SENTENCE in await _text(pass_id)+++async def test_a_hard_forget_leaves_a_passs_usage_attribution(make_user):+ """``costContext`` is the billing attribution a pass started under, not+ the dream's words: the erase of a closed pass, the blanking of a marked+ pass as it closes and the cleanup's end that blanks it again all leave+ it as it was."""+ owner = await make_user()+ attribution = {"user_id": owner, "trial_id": "trial-1", "generation": 3}+ closed = await _pass(+ owner, owner, status=DreamPassStatus.COMPLETE, cost_context=attribution+ )+ running = await _pass(owner, owner, cost_context=attribution)++ marked = await erase_dream_passes(owner, owner, reached=[_FORGOTTEN], named=[])+ assert await update_dream_pass(running, _CLOSES["failed"])+ assert await update_dream_pass(running, cleanup_finished())++ assert sorted(marked) == sorted([closed, running])+ for pass_id in (closed, running):+ row = await get_dream_pass(pass_id)+ assert row is not None and row.erased_at is not None+ assert _SENTENCE not in await _text(pass_id)+ assert row.cost_context == attributiondiff --git a/autogpt_platform/backend/backend/data/dream_pass_counts_test.py b/autogpt_platform/backend/backend/data/dream_pass_counts_test.pynew file mode 100644index 0000000000..0c2338a5d0--- /dev/null+++ b/autogpt_platform/backend/backend/data/dream_pass_counts_test.py@@ -0,0 +1,99 @@+"""What a pass's apply dropped, through a DreamPass row against a real+database: the record the sync route writes (``pass_record.outcome``) reads+back through ``dream_pass_result_from_row`` with its drop counts, and a row+written before them reads none dropped."""++import uuid+from collections.abc import AsyncIterator+from datetime import datetime, timezone++import pytest+from prisma.enums import DreamPassRoute, DreamPassTrigger+from prisma.models import DreamPass as PrismaDreamPass+from prisma.models import User++from backend.copilot.dream.pass_record import dream_pass_result_from_row, outcome+from backend.copilot.dream.schemas import DreamPassResult+from backend.util.json import SafeJson++from .dream_pass import create_dream_pass, get_dream_pass, update_dream_pass+from .dream_pass_models import DreamPassDraft++pytestmark = pytest.mark.asyncio(loop_scope="session")++_NOW = datetime.now(timezone.utc).replace(microsecond=0)+++async def pass_id() -> AsyncIterator[str]:+ """A new pass of a throwaway user; deleting the user afterwards cascades+ to it."""+ user_id = str(uuid.uuid4())+ await User.prisma().create(+ data={+ "id": user_id,+ "email": f"dream-pass-counts-{user_id}@example.com",+ "topUpConfig": SafeJson({}),+ "timezone": "UTC",+ }+ )+ draft = DreamPassDraft(+ id=str(uuid.uuid4()),+ user_id=user_id,+ scope_key=user_id,+ route=DreamPassRoute.SYNC,+ trigger=DreamPassTrigger.CRON,+ started_at=_NOW,+ )+ await create_dream_pass(draft)+ yield draft.id+ await User.prisma().delete(where={"id": user_id})+++async def test_an_applied_pass_reads_back_what_it_dropped(pass_id):+ row = await get_dream_pass(pass_id)+ assert row is not None+ result = DreamPassResult(+ user_id=row.user_id,+ pass_id=pass_id,+ completed_at=_NOW,+ consolidated_count=2,+ dropped_forgotten=1,+ uncited_writes_dropped=3,+ cross_scope_citations_dropped=4,+ failed_writes=5,+ provenance_pending=6,+ )++ assert await update_dream_pass(pass_id, outcome(result, None))++ row = await get_dream_pass(pass_id)+ assert row is not None+ read_back = dream_pass_result_from_row(row)+ assert (+ read_back.consolidated_count,+ read_back.dropped_forgotten,+ read_back.uncited_writes_dropped,+ read_back.cross_scope_citations_dropped,+ read_back.failed_writes,+ read_back.provenance_pending,+ ) == (2, 1, 3, 4, 5, 6)+++async def test_a_row_applied_before_the_drop_counts_reads_none_dropped(pass_id):+ await PrismaDreamPass.prisma().update(+ where={"id": pass_id},+ data={"operations": SafeJson({"applied": {"consolidated_count": 2}})},+ )++ row = await get_dream_pass(pass_id)+ assert row is not None and row.operations.applied is not None+ applied = row.operations.applied+ assert (+ applied.consolidated_count,+ applied.dropped_forgotten,+ applied.uncited_writes_dropped,+ applied.cross_scope_citations_dropped,+ applied.failed_writes,+ applied.provenance_pending,+ ) == (2, 0, 0, 0, 0, 0)diff --git a/autogpt_platform/backend/backend/data/dream_pass_lease_test.py b/autogpt_platform/backend/backend/data/dream_pass_lease_test.pyindex 07eae462a6..104410e53f 100644--- a/autogpt_platform/backend/backend/data/dream_pass_lease_test.py+++ b/autogpt_platform/backend/backend/data/dream_pass_lease_test.py@@ -24,6 +24,7 @@ from prisma.models import DreamPass as PrismaDreamPassfrom prisma.models import Userfrom backend.copilot.dream.fetch import DreamInput+from backend.copilot.dream.pass_errors import pass_errorfrom backend.copilot.dream.pass_record import (cancelled,cleanup_finished,@@ -54,6 +55,11 @@ from .dream_pass_models import (pytestmark = pytest.mark.asyncio(loop_scope="session")+# The stops' errors, as the cancel, the guard and the reaper write them.+_BY_ADMIN = pass_error("cancelled", "by_admin")+_STALLED = pass_error("expired", "stalled")+_REAPED = pass_error("gather", "reaped")+# Postgres keeps TIMESTAMP(3): whole milliseconds read back exactly._NOW = datetime.now(timezone.utc).replace(microsecond=0)# Lapses far older than any other test's rows, so the global scans that@@ -142,7 +148,7 @@ async def test_a_close_that_leaves_a_cleanup_marks_the_row_and_keeps_its_lease(it finished empties the mark and the lease."""pass_id = await _new_pass_with_a_bundle(owner)- assert await update_dream_pass(pass_id, cancelled("testing", owner_user_id=owner))+ assert await update_dream_pass(pass_id, cancelled(_BY_ADMIN, owner_user_id=owner))row = await _row(pass_id)assert row.status is DreamPassStatus.CANCELLED@@ -176,7 +182,7 @@ async def test_the_reaper_lists_open_passes_whose_lease_lapsed_oldest_first(owneawait _new_pass(owner, lease_expires_at=_NOW + timedelta(hours=1))await _new_pass(owner, lease_expires_at=None)closed = await _new_pass(owner, lease_expires_at=_LONG_AGO)- await update_dream_pass(closed, expired("stale", not_updated_since=None))+ await update_dream_pass(closed, expired(_STALLED, not_updated_since=None))# A closed row marked for its cleanup still carries its lapsed lease.assert (await _row(closed)).lease_expires_at == _LONG_AGO@@ -225,11 +231,11 @@ async def test_the_reapers_expiry_marks_the_row_until_its_cleanup_finishes(ownerassert not await update_dream_pass(pass_id, cleanup_finished())assert await update_dream_pass(- pass_id, reaped("lapsed", not_updated_since=row.updated_at)+ pass_id, reaped(_REAPED, not_updated_since=row.updated_at))closed = await _row(pass_id)- assert (closed.status, closed.error) == (DreamPassStatus.EXPIRED, "lapsed")+ assert (closed.status, closed.error) == (DreamPassStatus.EXPIRED, _REAPED)assert (closed.lease_token, closed.lease_expires_at) == ("tok", None)assert closed.cleanup_pending_at is not Noneassert closed.cancel_generation == 1@@ -242,7 +248,7 @@ async def test_the_reapers_expiry_marks_the_row_until_its_cleanup_finishes(ownerassert (done.cleanup_pending_at, done.lease_token) == (None, None)assert (done.status, done.error, done.cancel_generation) == (DreamPassStatus.EXPIRED,- "lapsed",+ _REAPED,1,)cleanups = await list_dream_pass_cleanups(_NOW, limit=1000)@@ -257,7 +263,7 @@ async def test_the_cleanup_scan_lists_closed_marked_rows_longest_pending_first(pass_id = await _new_pass(owner, lease_expires_at=_LONG_AGO)row = await _row(pass_id)assert await update_dream_pass(- pass_id, reaped("lapsed", not_updated_since=row.updated_at)+ pass_id, reaped(_REAPED, not_updated_since=row.updated_at))await PrismaDreamPass.prisma().update(where={"id": pass_id},@@ -287,14 +293,14 @@ async def test_the_reapers_scan_of_cleanups_goes_on_from_where_it_stopped(tied = []for _ in range(3):pass_id = await _new_pass(owner, lease_expires_at=_LONG_AGO)- await update_dream_pass(pass_id, cancelled("testing", owner_user_id=owner))+ await update_dream_pass(pass_id, cancelled(_BY_ADMIN, owner_user_id=owner))await PrismaDreamPass.prisma().update(where={"id": pass_id}, data={"cleanupPendingAt": mark})tied.append(pass_id)tied.sort()later = await _new_pass(owner, lease_expires_at=_LONG_AGO)- await update_dream_pass(later, cancelled("testing", owner_user_id=owner))+ await update_dream_pass(later, cancelled(_BY_ADMIN, owner_user_id=owner))await PrismaDreamPass.prisma().update(where={"id": later}, data={"cleanupPendingAt": mark + timedelta(seconds=1)})@@ -328,7 +334,7 @@ async def test_the_cleanup_scan_lists_a_row_once_its_cleanup_is_due(owner):ids = {}for name, (marked, lease_expires_at) in shapes.items():pass_id = await _new_pass(owner, lease_expires_at=_LONG_AGO)- await update_dream_pass(pass_id, cancelled("testing", owner_user_id=owner))+ await update_dream_pass(pass_id, cancelled(_BY_ADMIN, owner_user_id=owner))await PrismaDreamPass.prisma().update(where={"id": pass_id},data={"cleanupPendingAt": marked, "leaseExpiresAt": lease_expires_at},@@ -358,7 +364,7 @@ async def test_retention_keeps_a_closed_row_whose_cleanup_is_pending(owner):pending = await _new_pass(owner, lease_expires_at=_LONG_AGO)row = await _row(pending)assert await update_dream_pass(- pending, reaped("lapsed", not_updated_since=row.updated_at)+ pending, reaped(_REAPED, not_updated_since=row.updated_at))await PrismaDreamPass.prisma().update(where={"id": pending}, data={"createdAt": _LONG_AGO}@@ -376,7 +382,7 @@ async def test_retention_deletes_closed_passes_past_their_time_a_batch_at_a_timeold_open = await _new_pass(owner)recent_closed = await _new_pass(owner)for pass_id in [*old_closed, recent_closed]:- await update_dream_pass(pass_id, cancelled("testing", owner_user_id=owner))+ await update_dream_pass(pass_id, cancelled(_BY_ADMIN, owner_user_id=owner))await update_dream_pass(pass_id, cleanup_finished())for pass_id in [*old_closed, old_open]:await PrismaDreamPass.prisma().update(@@ -395,7 +401,7 @@ async def test_retention_deletes_closed_passes_past_their_time_a_batch_at_a_timeasync def test_a_users_passes_list_the_open_ones_only_when_asked(owner):open_pass = await _new_pass(owner)closed = await _new_pass(owner)- await update_dream_pass(closed, cancelled("testing", owner_user_id=owner))+ await update_dream_pass(closed, cancelled(_BY_ADMIN, owner_user_id=owner))everything = await list_dream_passes(owner, limit=10)only_open = await list_dream_passes(owner, limit=10, open_only=True)diff --git a/autogpt_platform/backend/backend/data/dream_pass_models.py b/autogpt_platform/backend/backend/data/dream_pass_models.pyindex 9acde3572f..75eece89f0 100644--- a/autogpt_platform/backend/backend/data/dream_pass_models.py+++ b/autogpt_platform/backend/backend/data/dream_pass_models.py@@ -63,6 +63,24 @@ CLOSED_ROW_CLEARS: frozenset[ClearableColumn] = frozenset(# pass's lock with, and the expiry that says whether the pass may still run.MARKED_ROW_CLEARS: frozenset[ClearableColumn] = frozenset({"input_bundle"})+# The fields holding the dream's own words in a pass's phase outputs and+# operations, at any depth, and what a hard forget leaves in each and in a+# narrative: the placeholder a forgotten fact reads+# (``copilot/graphiti/recall.FORGOTTEN_FACT``).+DREAM_TEXT_FIELDS: tuple[str, ...] = (+ "content",+ "rationale",+ "reason",+ "summary_for_user",+)+ERASED_TEXT = "[forgotten]"+# What a pass's read set holds when its writer did not state it: a pass from+# before read sets were recorded, or one a release without them wrote during+# its rollout. It may have read anything, so any hard forget in its scope+# reaches it, and its narrative's session takes the mark with its read set+# (``DreamPass.readUuids``'s default; never a graph uuid).+UNKNOWN_READS = "*"+class DreamPhaseOutputs(BaseModel):"""Each phase's validated output, as the next phase and apply read it."""@@ -80,9 +98,24 @@ class DreamPassApplied(BaseModel):proposal_count: int = 0demotion_count: int = 0entity_invalidation_count: int = 0+ # Writes and proposals the worker dropped unwritten for resting on a+ # forget, before the pass was reported; a row written before it reads 0.dropped_forgotten: int = 0+ # Writes and proposals not written yet when the pass was reported; a+ # row written before it reads 0.pending_writes: int = 0+ # Writes and proposals the worker did not make (their citation marker or+ # the graph write failed), and writes made whose derivation record+ # failed and awaits reconcile; a row written before them reads 0.failed_writes: int = 0+ provenance_pending: int = 0+ # Writes and proposals dropped before they were queued for citing+ # nothing the pass read, or a fact of another scope; a row written+ # before it reads 0.+ uncited_writes_dropped: int = 0+ # Their citations of a fact of another scope; a row written before it+ # reads 0.+ cross_scope_citations_dropped: int = 0# Distinct facts an acknowledged write spared and the accounting read# found live (provisional when the accounting is incomplete); a row# written before it reads 0.@@ -164,6 +197,8 @@ class DreamPassUpdate(BaseModel):usage: DreamPassUsage | None = Nonewindow_start: datetime | None = Nonewindow_end: datetime | None = None+ # Every fact and episode the pass gathered, added to what the row holds.+ read_uuids: list[str] | None = Noneerror: str | None = Nonesubmitted_at: datetime | None = Noneapplied_at: datetime | None = None@@ -233,6 +268,11 @@ class DreamPassRecord(BaseModel):window_start: datetime | Nonewindow_end: datetime | Noneerror: str | None+ # Set by a hard forget that reached the pass, never cleared.+ erased_at: datetime | None = None+ # Every fact and episode the pass gathered, never cleared;+ # ``[UNKNOWN_READS]`` for a pass from before read sets were recorded.+ read_uuids: list[str] = Field(default_factory=list)created_at: datetimestarted_at: datetime | Nonesubmitted_at: datetime | None@@ -275,6 +315,8 @@ class DreamPassRecord(BaseModel):window_start=row.windowStart,window_end=row.windowEnd,error=row.error,+ erased_at=row.erasedAt,+ read_uuids=list(row.readUuids or []),created_at=row.createdAt,started_at=row.startedAt,submitted_at=row.submittedAt,diff --git a/autogpt_platform/backend/backend/data/dream_pass_stop_test.py b/autogpt_platform/backend/backend/data/dream_pass_stop_test.pyindex 5b356a7375..b25b26dbcf 100644--- a/autogpt_platform/backend/backend/data/dream_pass_stop_test.py+++ b/autogpt_platform/backend/backend/data/dream_pass_stop_test.py@@ -17,6 +17,7 @@ from prisma.enums import ()from prisma.models import User+from backend.copilot.dream.pass_errors import pass_errorfrom backend.copilot.dream.pass_record import cancelled, expired, failedfrom backend.copilot.dream.schemas import ConsolidationOutputfrom backend.data.db import execute_raw_with_schema@@ -37,6 +38,12 @@ from .dream_pass_models import (pytestmark = pytest.mark.asyncio(loop_scope="session")+# The stops' errors, as the cancel and the guard write them.+_BY_ADMIN = pass_error("cancelled", "by_admin")+_BY_WIPE = pass_error("cancelled", "by_wipe")+_STALLED = pass_error("expired", "stalled")+_FORCED = pass_error("expired", "forced")+@pytest.fixtureasync def owner() -> AsyncIterator[str]:@@ -78,12 +85,12 @@ async def _row(pass_id: str) -> DreamPassRecord:async def test_a_cancel_closes_an_open_row_and_bumps_its_generation(owner):pass_id = await _new_pass(owner)- assert await update_dream_pass(pass_id, cancelled("testing", owner_user_id=owner))+ assert await update_dream_pass(pass_id, cancelled(_BY_ADMIN, owner_user_id=owner))row = await _row(pass_id)assert (row.status, row.error, row.cancel_generation) == (DreamPassStatus.CANCELLED,- "testing",+ _BY_ADMIN,1,)assert row.completed_at is not None@@ -101,7 +108,7 @@ async def test_a_cancel_reaches_only_an_open_row(owner):)assert not await update_dream_pass(- pass_id, cancelled("testing", owner_user_id=owner)+ pass_id, cancelled(_BY_ADMIN, owner_user_id=owner))row = await _row(pass_id)@@ -116,7 +123,7 @@ async def test_a_cancel_reaches_only_the_owners_row(owner):pass_id = await _new_pass(owner)assert not await update_dream_pass(- pass_id, cancelled("testing", owner_user_id=str(uuid.uuid4()))+ pass_id, cancelled(_BY_ADMIN, owner_user_id=str(uuid.uuid4())))row = await _row(pass_id)@@ -125,13 +132,13 @@ async def test_a_cancel_reaches_only_the_owners_row(owner):async def test_nothing_moves_a_cancelled_row(owner):pass_id = await _new_pass(owner)- await update_dream_pass(pass_id, cancelled("testing", owner_user_id=owner))+ await update_dream_pass(pass_id, cancelled(_BY_ADMIN, owner_user_id=owner))later = datetime.now(timezone.utc)for update in (- failed("cancelled: testing", None, later),- cancelled("again", owner_user_id=owner),- expired("stale", not_updated_since=None),+ failed(pass_error("apply", "crashed"), None, later),+ cancelled(_BY_WIPE, owner_user_id=owner),+ expired(_STALLED, not_updated_since=None),DreamPassUpdate(phase_outputs=DreamPhaseOutputs(consolidate=ConsolidationOutput())),@@ -141,29 +148,34 @@ async def test_nothing_moves_a_cancelled_row(owner):row = await _row(pass_id)assert (row.status, row.error, row.cancel_generation) == (DreamPassStatus.CANCELLED,- "testing",+ _BY_ADMIN,1,)assert row.phase_outputs == DreamPhaseOutputs()async def test_racing_cancels_bump_the_generation_exactly_once(owner):+ """Eight cancels race; each names its own completion time, so the row+ shows which one landed: exactly one, the row holding its write."""+ base = datetime.now(timezone.utc).replace(microsecond=0)for _ in range(5):pass_id = await _new_pass(owner)+ cancels = [+ cancelled(_BY_WIPE if n % 2 else _BY_ADMIN, owner_user_id=owner).model_copy(+ update={"completed_at": base + timedelta(seconds=n)}+ )+ for n in range(8)+ ]landed = await asyncio.gather(- *(- update_dream_pass(- pass_id, cancelled(f"cancel {n}", owner_user_id=owner)- )- for n in range(8)- )+ *(update_dream_pass(pass_id, cancel) for cancel in cancels))assert landed.count(True) == 1+ winner = cancels[landed.index(True)]row = await _row(pass_id)assert (row.status, row.cancel_generation) == (DreamPassStatus.CANCELLED, 1)- assert row.error == f"cancel {landed.index(True)}"+ assert (row.error, row.completed_at) == (winner.error, winner.completed_at)async def test_a_cancel_and_an_expiry_racing_close_the_row_once(owner):@@ -171,8 +183,8 @@ async def test_a_cancel_and_an_expiry_racing_close_the_row_once(owner):pass_id = await _new_pass(owner)landed = await asyncio.gather(- update_dream_pass(pass_id, cancelled("testing", owner_user_id=owner)),- update_dream_pass(pass_id, expired("stale", not_updated_since=None)),+ update_dream_pass(pass_id, cancelled(_BY_ADMIN, owner_user_id=owner)),+ update_dream_pass(pass_id, expired(_STALLED, not_updated_since=None)),)assert landed.count(True) == 1@@ -189,18 +201,18 @@ async def test_an_expiry_lands_only_if_nothing_wrote_the_row_since(owner):await update_dream_pass(pass_id, DreamPassUpdate(phase=DreamPassPhase.RECOMBINE))assert not await update_dream_pass(- pass_id, expired("stale", not_updated_since=seen.updated_at)+ pass_id, expired(_STALLED, not_updated_since=seen.updated_at))assert (await _row(pass_id)).status is DreamPassStatus.RUNNINGlatest = await _row(pass_id)assert await update_dream_pass(- pass_id, expired("stale", not_updated_since=latest.updated_at)+ pass_id, expired(_STALLED, not_updated_since=latest.updated_at))row = await _row(pass_id)assert (row.status, row.error, row.cancel_generation) == (DreamPassStatus.EXPIRED,- "stale",+ _STALLED,1,)assert row.completed_at is not None@@ -212,7 +224,7 @@ async def test_an_expiry_with_a_stale_cutoff_lands_only_on_an_old_row(owner):cutoff = datetime.now(timezone.utc) - timedelta(minutes=30)assert not await update_dream_pass(- pass_id, expired("stale", not_updated_since=cutoff)+ pass_id, expired(_STALLED, not_updated_since=cutoff))await execute_raw_with_schema(@@ -220,7 +232,7 @@ async def test_an_expiry_with_a_stale_cutoff_lands_only_on_an_old_row(owner):"""SET "updatedAt" = "updatedAt" - interval '1 hour' WHERE "id" = $1""",pass_id,)- assert await update_dream_pass(pass_id, expired("stale", not_updated_since=cutoff))+ assert await update_dream_pass(pass_id, expired(_STALLED, not_updated_since=cutoff))assert (await _row(pass_id)).status is DreamPassStatus.EXPIRED@@ -228,10 +240,14 @@ async def test_a_forced_expiry_lands_on_a_fresh_row(owner):pass_id = await _new_pass(owner)await update_dream_pass(pass_id, DreamPassUpdate(phase=DreamPassPhase.CONSOLIDATE))- assert await update_dream_pass(pass_id, expired("forced", not_updated_since=None))+ assert await update_dream_pass(pass_id, expired(_FORCED, not_updated_since=None))row = await _row(pass_id)- assert (row.status, row.cancel_generation) == (DreamPassStatus.EXPIRED, 1)+ assert (row.status, row.error, row.cancel_generation) == (+ DreamPassStatus.EXPIRED,+ _FORCED,+ 1,+ )async def test_open_passes_list_newest_first_and_capped(owner):diff --git a/autogpt_platform/backend/backend/data/dream_pass_test.py b/autogpt_platform/backend/backend/data/dream_pass_test.pyindex 1fc64ac80d..21cf1761f4 100644--- a/autogpt_platform/backend/backend/data/dream_pass_test.py+++ b/autogpt_platform/backend/backend/data/dream_pass_test.py@@ -17,6 +17,7 @@ from prisma.models import Expert, Userfrom backend.copilot.dream.fetch import DreamInput, EpisodeRow, FactRow, SessionRowfrom backend.copilot.dream.input_bundle import input_bundle_to_dict+from backend.copilot.dream.pass_errors import UNRECORDED, pass_errorfrom backend.copilot.dream.schemas import (ConsolidatedFact,ConsolidationOutput,@@ -162,6 +163,9 @@ async def test_phase_outputs_and_operations_merge_one_field_at_a_time(make_user)planned = DreamOperations(summary_for_user="clamped")applied = DreamPassApplied(consolidated_count=1,+ dropped_forgotten=4,+ uncited_writes_dropped=3,+ cross_scope_citations_dropped=5,protected_demotions=2,indeterminate_demotion_writes=1,demotion_accounting_complete=False,@@ -277,7 +281,9 @@ async def test_a_finished_pass_is_final(make_user):assert not await update_dream_pass(draft.id,- DreamPassUpdate(status=DreamPassStatus.ERRORED, error="late crash"),+ DreamPassUpdate(+ status=DreamPassStatus.ERRORED, error=pass_error("pass", "crashed")+ ),)assert not await update_dream_pass(draft.id,@@ -371,14 +377,21 @@ async def test_deleting_the_expert_or_the_user_removes_their_passes(make_user):async def test_text_loses_the_control_characters_postgres_rejects(make_user):+ """Free text (a skip's reason) is kept without them. An error is not free+ text: one that is not a fixed code is recorded as ``UNRECORDED``."""owner = await make_user()draft = _draft(owner)await create_dream_pass(draft)assert await update_dream_pass(draft.id,- DreamPassUpdate(status=DreamPassStatus.ERRORED, error="bad\x00byte"),+ DreamPassUpdate(+ status=DreamPassStatus.SKIPPED,+ skip_reason="bad\x00byte",+ error="bad\x00byte",+ ),)row = await get_dream_pass(draft.id)- assert row is not None and row.error == "badbyte"+ assert row is not None+ assert (row.skip_reason, row.error) == ("badbyte", UNRECORDED)diff --git a/autogpt_platform/backend/backend/data/dream_pass_update.py b/autogpt_platform/backend/backend/data/dream_pass_update.pyindex 08772461d8..a2ebcb367b 100644--- a/autogpt_platform/backend/backend/data/dream_pass_update.py+++ b/autogpt_platform/backend/backend/data/dream_pass_update.py@@ -16,6 +16,7 @@ statement against the row as it stands when the statement runs:* ``cancelGeneration`` goes up by one when the update stops the pass fromoutside, in the same statement that closes the row, so of two stopsracing for one open row exactly one lands and bumps it;+ * ``readUuids`` gains the update's, never losing one;* ``leaseExpiresAt`` only moves later: a renewal written late (a syncrenewal the store gave up on, landing after the batch submit's longerlease) never shortens the lease the row holds. Every renewal is now@@ -27,6 +28,22 @@ statement against the row as it stands when the statement runs:only way a lease is dropped or cut short;* every other column takes the new value when the update gives one.+A row a hard forget marked (``erasedAt``, ``copilot/dream/forget_copies.py``)+has the dream's text blanked by the update that closes it, in the same+statement, whichever way it closes: the pass's own end, a cancel, an expiry,+the reaper's close, and again by the cleanup's end. Every string under one of+``DREAM_TEXT_FIELDS`` in its phase outputs and operations, the update's own+included, reads ``ERASED_TEXT`` (``dream_text_blanked``, a function of the+migration that added the mark), and its error is kept only when it is one of+the fixed codes (``dream_error_blanked``). The mark itself is the forget's to+write and nothing clears it, so a pass that was still running when it landed+is blanked however late it ends.++Every error the row takes is one of the fixed codes+(``copilot/dream/pass_errors.py``): any other is recorded as+``pass: unrecorded``, logged, so no exception's or model's text reaches the+row.+Nothing is written to a row that has reached a terminal status, nor to onethat fails the update's own conditions: another user's row when it names theowner, a row written since the instant it names. The one exception is the@@ -35,19 +52,30 @@ finished), which applies to a closed row only and only clears. The statementtext is constant; every value is a bound parameter."""+import loggingfrom typing import Anyfrom pydantic import BaseModelfrom backend.copilot.dream.input_bundle import input_bundle_to_dict+from backend.copilot.dream.pass_errors import (+ PASS_ERROR_CODES,+ PASS_ERROR_STAGES,+ UNRECORDED,+ is_pass_error,+)from backend.data.dream_pass_models import (CLOSED_STATUSES,+ DREAM_TEXT_FIELDS,+ ERASED_TEXT,OPEN_STATUSES,ClearableColumn,DreamPassUpdate,)from backend.util.json import dumps, sanitize_json, sanitize_string+logger = logging.getLogger(__name__)+# The column each clearable field names in ``TRANSITION_SQL``._CLEARABLE_COLUMNS: dict[ClearableColumn, str] = {"lease_token": "leaseToken",@@ -68,22 +96,43 @@ UPDATE {schema_prefix}"DreamPass" SETAND "phase" <= $3::{schema_prefix}"DreamPassPhase"THEN $4::text ELSE "providerBatchId" END,"skipReason" = COALESCE($5::text, "skipReason"),- "error" = COALESCE($6::text, "error"),+ "error" = CASE+ WHEN "erasedAt" IS NOT NULL AND $24::boolean+ THEN {schema_prefix}"dream_error_blanked"(+ COALESCE($6::text, "error"), $28::text[], $29::text[], $26::text)+ ELSE COALESCE($6::text, "error") END,"leaseToken" = CASE WHEN 'leaseToken' = ANY($22::text[]) THEN NULLELSE COALESCE($7::text, "leaseToken") END,"leaseExpiresAt" = CASE WHEN 'leaseExpiresAt' = ANY($22::text[]) THEN NULLELSE GREATEST($8::timestamptz AT TIME ZONE 'UTC', "leaseExpiresAt") END,"windowStart" = COALESCE($9::timestamptz AT TIME ZONE 'UTC', "windowStart"),"windowEnd" = COALESCE($10::timestamptz AT TIME ZONE 'UTC', "windowEnd"),+ "readUuids" = CASE WHEN $27::text[] IS NULL THEN "readUuids"+ ELSE ARRAY(SELECT DISTINCT uuid+ FROM unnest(COALESCE("readUuids", ARRAY[]::text[]) || $27::text[])+ AS read(uuid)+ ORDER BY uuid) END,"submittedAt" = COALESCE($11::timestamptz AT TIME ZONE 'UTC', "submittedAt"),"appliedAt" = COALESCE($12::timestamptz AT TIME ZONE 'UTC', "appliedAt"),"completedAt" = COALESCE($13::timestamptz AT TIME ZONE 'UTC', "completedAt"),"inputBundle" = CASE WHEN 'inputBundle' = ANY($22::text[]) THEN NULLELSE COALESCE($14::jsonb, "inputBundle") END,"usage" = COALESCE($15::jsonb, "usage"),- "phaseOutputs" = CASE WHEN $16::jsonb IS NULL THEN "phaseOutputs"+ "phaseOutputs" = CASE+ WHEN "erasedAt" IS NOT NULL AND $24::boolean+ THEN {schema_prefix}"dream_text_blanked"(+ CASE WHEN $16::jsonb IS NULL THEN "phaseOutputs"+ ELSE COALESCE("phaseOutputs", '{{}}'::jsonb) || $16::jsonb END,+ $25::text[], $26::text)+ WHEN $16::jsonb IS NULL THEN "phaseOutputs"ELSE COALESCE("phaseOutputs", '{{}}'::jsonb) || $16::jsonb END,- "operations" = CASE WHEN $17::jsonb IS NULL THEN "operations"+ "operations" = CASE+ WHEN "erasedAt" IS NOT NULL AND $24::boolean+ THEN {schema_prefix}"dream_text_blanked"(+ CASE WHEN $17::jsonb IS NULL THEN "operations"+ ELSE COALESCE("operations", '{{}}'::jsonb) || $17::jsonb END,+ $25::text[], $26::text)+ WHEN $17::jsonb IS NULL THEN "operations"ELSE COALESCE("operations", '{{}}'::jsonb) || $17::jsonb END,"cancelGeneration" = "cancelGeneration"+ CASE WHEN $19::boolean THEN 1 ELSE 0 END,@@ -107,7 +156,7 @@ def transition_args(pass_id: str, update: DreamPassUpdate) -> list[Any]:update.phase.value if update.phase is not None else None,update.provider_batch_id,_text(update.skip_reason),- _text(update.error),+ _recorded(update.error),update.lease_token,update.lease_expires_at,update.window_start,@@ -132,6 +181,12 @@ def transition_args(pass_id: str, update: DreamPassUpdate) -> list[Any]:update.not_updated_since,sorted(_CLEARABLE_COLUMNS[column] for column in update.clear),update.cleanup_pending_at,+ update.closed_row or update.status in CLOSED_STATUSES,+ list(DREAM_TEXT_FIELDS),+ ERASED_TEXT,+ update.read_uuids,+ list(PASS_ERROR_STAGES),+ list(PASS_ERROR_CODES),]@@ -148,5 +203,17 @@ def _text(value: str | None) -> str | None:return sanitize_string(value) if value is not None else None+def _recorded(error: str | None) -> str | None:+ """*error* when it is one of the fixed codes, else ``UNRECORDED``."""+ if error is None or is_pass_error(error):+ return error+ logger.warning(+ "A dream pass error that is not one of the fixed codes is recorded as %r",+ UNRECORDED,+ stack_info=True,+ )+ return UNRECORDED++def _json(value: Any) -> str:return dumps(sanitize_json(value))diff --git a/autogpt_platform/backend/migrations/20261013120000_add_dream_pass_erased_at/migration.sql b/autogpt_platform/backend/migrations/20261013120000_add_dream_pass_erased_at/migration.sqlnew file mode 100644index 0000000000..437f72eb2d--- /dev/null+++ b/autogpt_platform/backend/migrations/20261013120000_add_dream_pass_erased_at/migration.sql@@ -0,0 +1,75 @@+-- A hard forget marks with erasedAt every dream pass it reached+-- (backend/copilot/dream/forget_copies.py); the mark is never cleared. A pass+-- is reached when it read a fact or episode the forget reached (readUuids:+-- every one it gathered, recorded before its first model call and never+-- cleared), may still be writing (open, or closed with its cleanup pending),+-- or wrote what the forget erased. A pass already closed has the dream's text+-- in its row blanked by the forget itself, and one still open has it blanked+-- in the statement that closes it, whichever way it closes+-- (backend/data/dream_pass_update.py). A narrative written after the mark is+-- written blanked, and carries its pass's readUuids into its session's+-- metadata, where a forget finds it once the pass's row has aged out+-- (backend/copilot/db.py, add_dream_narrative).+-- Nothing recorded what a pass from before this migration read, nor what a+-- dream session from before it was written from. So every row that exists+-- when it runs reads "*" in readUuids, as does a row a release without the+-- column writes while it rolls out: it may have read anything, and any hard+-- forget in its scope marks it. Its narrative's session takes the "*" with+-- the rest of its read set, and a hard forget blanks every narrative of its+-- scope whose session holds "*" or no read set at all (blank_dream_narratives).+-- A pass this release creates states its read set (empty until it gathers),+-- which never holds "*".+-- A nullable column and an array with a constant default on an existing+-- table: no row is rewritten.+-- Prisma does not track functions, so the two below cause no drift.++-- AlterTable+ALTER TABLE "DreamPass" ADD COLUMN "erasedAt" TIMESTAMP(3),+ADD COLUMN "readUuids" TEXT[] DEFAULT ARRAY['*']::TEXT[];++-- CreateFunction+-- value, with every string held under one of fields, at any depth, replaced+-- by placeholder. Anything else (numbers, uuids, nulls, nested objects under+-- those keys) stays.+CREATE OR REPLACE FUNCTION dream_text_blanked(value jsonb, fields text[], placeholder text)+RETURNS jsonb LANGUAGE plpgsql IMMUTABLE SET search_path FROM CURRENT AS $$+BEGIN+ IF jsonb_typeof(value) = 'object' THEN+ RETURN COALESCE((+ SELECT jsonb_object_agg(+ member.key,+ CASE WHEN member.key = ANY(fields) AND jsonb_typeof(member.item) = 'string'+ THEN to_jsonb(placeholder)+ ELSE dream_text_blanked(member.item, fields, placeholder) END)+ FROM jsonb_each(value) AS member(key, item)+ ), '{}'::jsonb);+ END IF;+ IF jsonb_typeof(value) = 'array' THEN+ RETURN COALESCE((+ SELECT jsonb_agg(+ dream_text_blanked(element.item, fields, placeholder)+ ORDER BY element.position)+ FROM jsonb_array_elements(value) WITH ORDINALITY AS element(item, position)+ ), '[]'::jsonb);+ END IF;+ RETURN value;+END;+$$;++-- CreateFunction+-- A pass's error when it is "<stage>: <code>" with the stage one of stages+-- and the code one of codes (backend/copilot/dream/pass_errors.py, the only+-- errors a pass records). Any other is placeholder, after its "<stage>: "+-- when that stage is one of stages.+CREATE OR REPLACE FUNCTION dream_error_blanked(+ error text, stages text[], codes text[], placeholder text)+RETURNS text LANGUAGE sql IMMUTABLE AS $$+ SELECT CASE+ WHEN error IS NULL THEN NULL+ WHEN position(': ' IN error) = 0+ OR NOT (split_part(error, ': ', 1) = ANY(stages))+ THEN placeholder+ WHEN substr(error, position(': ' IN error) + 2) = ANY(codes) THEN error+ ELSE split_part(error, ': ', 1) || ': ' || placeholder+ END+$$;diff --git a/autogpt_platform/backend/schema.prisma b/autogpt_platform/backend/schema.prismaindex 3c761fccb5..04160a330a 100644--- a/autogpt_platform/backend/schema.prisma+++ b/autogpt_platform/backend/schema.prisma@@ -635,6 +635,21 @@ model DreamPass {windowEnd DateTime?error String?+ // Set by a hard forget that may have reached the pass, and never cleared:+ // the dream's text in the row is blanked, by the forget if the pass is+ // closed, else by the transition that closes it, and its narrative is+ // written blanked (backend/copilot/dream/forget_copies.py).+ erasedAt DateTime?+ // Every fact and episode the pass gathered, recorded before its first+ // model call and never cleared: a hard forget marks a pass that read what+ // it reached however long ago the pass closed, and its narrative's session+ // keeps a copy for once the row has aged out. A pass's create states it+ // (empty until the pass gathers); a row whose writer did not (one from+ // before this column, or written by a release without it during a rollout)+ // holds "*": it may have read anything, and any hard forget in its scope+ // reaches it (backend/data/dream_pass_models.UNKNOWN_READS).+ readUuids String[] @default(["*"])+createdAt DateTime @default(now())startedAt DateTime?submittedAt DateTime?diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.jsonindex 0766d3df1c..9f10e49652 100644--- a/autogpt_platform/frontend/src/app/api/openapi.json+++ b/autogpt_platform/frontend/src/app/api/openapi.json@@ -21866,12 +21866,18 @@"type": "array","title": "Source Episode Uuids","description": "UUIDs of the :Episodic nodes the fact was consolidated from."+ },+ "source_fact_uuids": {+ "items": { "type": "string" },+ "type": "array",+ "title": "Source Fact Uuids",+ "description": "UUIDs of the active facts (:RELATES_TO edges) the fact was consolidated from."}},"type": "object","required": ["content", "confidence"],"title": "ConsolidatedFact",- "description": "A cluster of related facts merged into a single canonical statement.\n\nPhase 1 output. Provenance always points back to the source episodes\nso apply.py can record where the consolidation came from."+ "description": "A cluster of related facts merged into a single canonical statement.\n\nPhase 1 output. It cites the episodes and active facts it was drawn from,\nso apply can record where the consolidation came from and a forget of one\nof them reaches it (``dream/citations.py``); one citing nothing the pass\nread is dropped unwritten."},"ConsolidationOutput": {"properties": {@@ -23520,7 +23526,7 @@},"type": "object","title": "DreamOperations",- "description": "Phase 3 output — what apply.py writes back to the world.\n\nGuardrails enforced by phase 3's prompt and double-checked in\napply.py:\n * ≤ ``max_demotions_per_pass`` demotions per pass (runaway-demotion\n mitigation per spec §3 / TODO P0.3b).\n * Scope match enforced — proposals cannot cross scopes.\n * Empty ``writes`` and ``proposals`` is fine; a pass can be no-op."+ "description": "Phase 3 output — what apply.py writes back to the world.\n\nGuardrails enforced by phase 3's prompt and double-checked in\napply.py:\n * ≤ ``max_demotions_per_pass`` demotions per pass (runaway-demotion\n mitigation per spec §3 / TODO P0.3b).\n * Scope match enforced — a write or proposal citing a fact of\n another scope (a fact's scope, else ``real:global``) is dropped\n whole, as is one citing nothing the pass read\n (``dream/citations.py``); episode citations are not scoped. A\n source in the same scope that the write does not truly rest on is\n not caught.\n * Empty ``writes`` and ``proposals`` is fine; a pass can be no-op."},"DreamOperationsSnapshot": {"properties": {@@ -23588,6 +23594,21 @@"title": "Failed Writes","default": 0},+ "provenance_pending": {+ "type": "integer",+ "title": "Provenance Pending",+ "default": 0+ },+ "uncited_writes_dropped": {+ "type": "integer",+ "title": "Uncited Writes Dropped",+ "default": 0+ },+ "cross_scope_citations_dropped": {+ "type": "integer",+ "title": "Cross Scope Citations Dropped",+ "default": 0+ },"protected_demotions": {"type": "integer","title": "Protected Demotions",@@ -23639,7 +23660,7 @@},"type": "object","title": "DreamPassCancelRequest",- "description": "Why an admin cancels a dream pass, kept as the pass's error."+ "description": "Why an admin cancels a dream pass, for the log; the pass records\n``cancelled: by_admin``."},"DreamPassListResponse": {"properties": {@@ -23767,6 +23788,18 @@"anyOf": [{ "type": "string" }, { "type": "null" }],"title": "Error"},+ "erased_at": {+ "anyOf": [+ { "type": "string", "format": "date-time" },+ { "type": "null" }+ ],+ "title": "Erased At"+ },+ "read_uuids": {+ "items": { "type": "string" },+ "type": "array",+ "title": "Read Uuids"+ },"created_at": {"type": "string","format": "date-time",@@ -23911,6 +23944,21 @@"title": "Failed Writes","default": 0},+ "provenance_pending": {+ "type": "integer",+ "title": "Provenance Pending",+ "default": 0+ },+ "uncited_writes_dropped": {+ "type": "integer",+ "title": "Uncited Writes Dropped",+ "default": 0+ },+ "cross_scope_citations_dropped": {+ "type": "integer",+ "title": "Cross Scope Citations Dropped",+ "default": 0+ },"protected_demotions": {"type": "integer","title": "Protected Demotions",@@ -26024,11 +26072,18 @@"ForgetFactResponse": {"properties": {"uuid": { "type": "string", "title": "Uuid" },- "forgotten": { "type": "boolean", "title": "Forgotten" }+ "forgotten": { "type": "boolean", "title": "Forgotten" },+ "derived_forgotten": {+ "type": "integer",+ "title": "Derived Forgotten",+ "default": 0+ },+ "resumed": { "type": "boolean", "title": "Resumed", "default": false }},"type": "object","required": ["uuid", "forgotten"],- "title": "ForgetFactResponse"+ "title": "ForgetFactResponse",+ "description": "A forgotten fact. ``derived_forgotten`` counts the facts the dream had\nderived from it, retracted with it (``graphiti/recall_cascade.py``).\n``resumed`` is True when the fact had already been erased by an earlier\nhard forget, and this one went on retracting and erasing what the dream\nderived from it (``graphiti/recall_forget.py``)."},"FrontendFlag": {"type": "string",@@ -29300,6 +29355,16 @@"type": "array","title": "Deleted Uuids"},+ "derived_uuids": {+ "items": { "type": "string" },+ "type": "array",+ "title": "Derived Uuids"+ },+ "resumed_uuids": {+ "items": { "type": "string" },+ "type": "array",+ "title": "Resumed Uuids"+ },"failed_uuids": {"items": { "type": "string" },"type": "array",