Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

fix(graph): abort the snapshot delta instead of zeroing it on a failed read (#1384)

cbsincool29ecfde4df59
diff --git a/src/functions/graph.ts b/src/functions/graph.ts
index fedf97aa..68e78117 100644
--- a/src/functions/graph.ts
+++ b/src/functions/graph.ts
@@ -93,6 +93,27 @@ async function readSnapshot(kv: StateKV): Promise<GraphSnapshot | null> {
}
}
+async function readSnapshotStrict(kv: StateKV): Promise<GraphSnapshot | null> {
+ let raw: unknown;
+ try {
+ raw = await kv.get<unknown>(KV.graphSnapshot, SNAPSHOT_KEY);
+ } catch (err) {
+ logger.warn("Graph snapshot read failed, retrying once", {
+ error: err instanceof Error ? err.message : String(err),
+ });
+ raw = await kv.get<unknown>(KV.graphSnapshot, SNAPSHOT_KEY);
+ }
+ if (raw === null || raw === undefined) return null;
+ if (typeof raw === "object" && (raw as { version?: unknown }).version === 1) {
+ return raw as GraphSnapshot;
+ }
+ throw new Error(
+ `Graph snapshot has unknown schema version: ${JSON.stringify(
+ (raw as { version?: unknown }).version,
+ )}`,
+ );
+}
+
function buildSnapshotFromArrays(
nodes: GraphNode[],
edges: GraphEdge[],
@@ -551,7 +572,6 @@ export function extractGraphHeuristics(
// maintenance — which also makes re-imports idempotent: an existing
// (type, name) resolves through the name index and merges instead of
// duplicating.
-//
// #814 v2: targeted name-index lookups replace the O(n) scan over
// `kv.list<GraphNode>(KV.graphNodes)`. At 75K nodes the list payload
// exceeds the iii heartbeat budget and the worker dies before merge can
@@ -563,7 +583,7 @@ export async function persistGraphDelta(
obsIds: string[],
): Promise<{ newNodeCount: number; newEdgeCount: number }> {
return withKeyedLock("graph:persist", async () => {
- const snap = (await readSnapshot(kv)) ?? emptySnapshot();
+ const snap = (await readSnapshotStrict(kv)) ?? emptySnapshot();
const capturedAt = new Date().toISOString();
let newNodeCount = 0;
let newEdgeCount = 0;
diff --git a/src/functions/remember.ts b/src/functions/remember.ts
index 89c47f4e..21fca5c6 100644
--- a/src/functions/remember.ts
+++ b/src/functions/remember.ts
@@ -289,6 +289,19 @@ export function registerRememberFunction(sdk: IIIClient, kv: StateKV): void {
failures.push({ id, error: "delete_failed" });
};
+ const cleanupFailures: Array<{ id: string; error: string }> = [];
+ const cleanup = async (id: string, steps: () => Promise<void>) => {
+ try {
+ await steps();
+ } catch (err) {
+ logger.warn("Forget cleanup failed", {
+ id,
+ error: err instanceof Error ? err.message : String(err),
+ });
+ cleanupFailures.push({ id, error: "cleanup_failed" });
+ }
+ };
+
const attempt = async (id: string, remove: () => Promise<boolean>) => {
try {
if (await remove()) deleted++;
@@ -308,18 +321,17 @@ export function registerRememberFunction(sdk: IIIClient, kv: StateKV): void {
(await kv.get<ObservationRef>(KV.observations(sessionId), obsId));
if (!obs) return false;
await kv.delete(KV.observations(sessionId), obsId);
+ deletedObservationIds.push(obsId);
+ getSearchIndex().remove(obsId);
+ vectorIndexRemove(obsId);
+ indexCleaned = true;
await unindexObservationSession(kv, obsId).catch(() => {});
- try {
+ await cleanup(obsId, async () => {
if (obs.imageData) await decrementImageRef(kv, sdk, obs.imageData);
if (obs.imageRef && obs.imageRef !== obs.imageData) {
await decrementImageRef(kv, sdk, obs.imageRef);
}
- } finally {
- getSearchIndex().remove(obsId);
- vectorIndexRemove(obsId);
- indexCleaned = true;
- }
- deletedObservationIds.push(obsId);
+ });
return true;
};
@@ -329,17 +341,14 @@ export function registerRememberFunction(sdk: IIIClient, kv: StateKV): void {
const mem = await kv.get<Memory>(KV.memories, memoryId);
if (!mem) return false;
await kv.delete(KV.memories, memoryId);
- try {
- if (mem.imageRef) {
- await decrementImageRef(kv, sdk, mem.imageRef);
- }
- } finally {
- getSearchIndex().remove(memoryId);
- vectorIndexRemove(memoryId);
- indexCleaned = true;
- }
- await deleteAccessLog(kv, memoryId);
deletedMemoryIds.push(memoryId);
+ getSearchIndex().remove(memoryId);
+ vectorIndexRemove(memoryId);
+ indexCleaned = true;
+ await cleanup(memoryId, async () => {
+ if (mem.imageRef) await decrementImageRef(kv, sdk, mem.imageRef);
+ await deleteAccessLog(kv, memoryId);
+ });
return true;
});
}
@@ -409,6 +418,8 @@ export function registerRememberFunction(sdk: IIIClient, kv: StateKV): void {
notFound: notFound.length,
failed: failures.length,
failures: failures.length > 0 ? failures : undefined,
+ cleanupFailed: cleanupFailures.length,
+ cleanupFailures: cleanupFailures.length > 0 ? cleanupFailures : undefined,
reason: "user-initiated forget",
},
);
@@ -425,6 +436,8 @@ export function registerRememberFunction(sdk: IIIClient, kv: StateKV): void {
notFound,
failed: failures.length,
failures: failures.length > 0 ? failures : undefined,
+ cleanupFailed: cleanupFailures.length,
+ cleanupFailures: cleanupFailures.length > 0 ? cleanupFailures : undefined,
};
},
);
diff --git a/test/graph.test.ts b/test/graph.test.ts
index 0149702c..176f63ba 100644
--- a/test/graph.test.ts
+++ b/test/graph.test.ts
@@ -597,6 +597,96 @@ describe("Graph Functions", () => {
// the oversized-corpus rebuild refusal. The hot path never enumerates
// any more, but the rebuild endpoint AND the BFS / query branches
// still call kv.list — both need explicit failure-mode tests.
+ describe("snapshot write must not fail open (#1381)", () => {
+ async function seedSnapshot(totalNodes: number) {
+ await kv.set("mem:graph:snapshot", "current", {
+ version: 1,
+ topNodes: [],
+ topEdges: [],
+ topDegrees: {},
+ stats: { totalNodes, totalEdges: 0, nodesByType: {}, edgesByType: {} },
+ updatedAt: "2026-01-01T00:00:00Z",
+ dirty: false,
+ });
+ }
+
+ async function extractWithFlakySnapshot(kvImpl: ReturnType<typeof mockKV>) {
+ registerGraphFunction(sdk as never, kvImpl as never, mockProvider as never);
+ return (await sdk.trigger("mem::graph-extract", {
+ observations: [testObs],
+ })) as { success: boolean; error?: string; nodesAdded?: number };
+ }
+
+ function flakyKV(failures: number) {
+ let reads = 0;
+ const realGet = kv.get.bind(kv);
+ return {
+ ...kv,
+ get: async <T>(scope: string, key: string): Promise<T | null> => {
+ if (scope === "mem:graph:snapshot" && reads++ < failures) {
+ throw new Error("Invocation timeout after 180000ms: state::get");
+ }
+ return realGet<T>(scope, key);
+ },
+ };
+ }
+
+ it("a persistent read failure aborts the delta instead of zeroing the snapshot", async () => {
+ await seedSnapshot(40000);
+
+ const result = await extractWithFlakySnapshot(flakyKV(Number.POSITIVE_INFINITY));
+
+ expect(result.success).toBe(false);
+ expect(result.error).toContain("Invocation timeout");
+
+ const snap = await kv.get<{ stats: { totalNodes: number } }>(
+ "mem:graph:snapshot",
+ "current",
+ );
+ expect(snap!.stats.totalNodes).toBe(40000);
+ });
+
+ it("retries a transient read failure once and then merges onto the real snapshot", async () => {
+ await seedSnapshot(40000);
+
+ const result = await extractWithFlakySnapshot(flakyKV(1));
+
+ expect(result.success).toBe(true);
+ const snap = await kv.get<{ stats: { totalNodes: number } }>(
+ "mem:graph:snapshot",
+ "current",
+ );
+ expect(snap!.stats.totalNodes).toBeGreaterThan(39999);
+ });
+
+ it("a snapshot under an unknown schema version aborts instead of reading as empty", async () => {
+ await kv.set("mem:graph:snapshot", "current", {
+ version: 2,
+ stats: { totalNodes: 40000, totalEdges: 0, nodesByType: {}, edgesByType: {} },
+ });
+
+ const result = await extractWithFlakySnapshot(kv);
+
+ expect(result.success).toBe(false);
+ expect(result.error).toContain("unknown schema version");
+
+ const snap = await kv.get<{ version: number }>("mem:graph:snapshot", "current");
+ expect(snap!.version).toBe(2);
+ });
+
+ it("an absent snapshot is still a clean first run", async () => {
+ const result = await extractWithFlakySnapshot(kv);
+
+ expect(result.success).toBe(true);
+ const snap = await kv.get<{ version: number; stats: { totalNodes: number } }>(
+ "mem:graph:snapshot",
+ "current",
+ );
+ expect(snap).not.toBeNull();
+ expect(snap!.stats.totalNodes).toBe(2);
+ });
+ });
+
describe("snapshot-reported total floor (#1382)", () => {
function seedSnapshot(
stats: { totalNodes: number; nodesByType: Record<string, number> },
diff --git a/test/remember-forget-audit.test.ts b/test/remember-forget-audit.test.ts
index e8fc5a4c..09d5e434 100644
--- a/test/remember-forget-audit.test.ts
+++ b/test/remember-forget-audit.test.ts
@@ -246,14 +246,43 @@ describe("mem::forget search-index cleanup", () => {
await kv.set("mem:obs:ses_1", "obs_img", { id: "obs_img", imageRef: "img_1" });
getSearchIndex().add(memoryToObservation(makeMemory("obs_img")));
- await sdk.trigger({
+ const result = (await sdk.trigger({
function_id: "mem::forget",
payload: { sessionId: "ses_1", observationIds: ["obs_img"] },
- });
+ })) as { deleted: number; failed: number; cleanupFailed: number; cleanupFailures?: Array<{ id: string }> };
expect(await kv.get("mem:obs:ses_1", "obs_img")).toBeNull();
expect(getSearchIndex().has("obs_img")).toBe(false);
expect(persistence.save).toHaveBeenCalled();
+ expect(result.deleted).toBe(1);
+ expect(result.failed).toBe(0);
+ expect(result.cleanupFailed).toBe(1);
+ expect(result.cleanupFailures?.[0].id).toBe("obs_img");
+ const audits = await kv.list<{ targetIds: string[]; details: { deleted: number } }>(currentAuditScope());
+ expect(audits).toHaveLength(1);
+ expect(audits[0].targetIds).toEqual(["obs_img"]);
+ expect(audits[0].details.deleted).toBe(1);
+ });
+
+ it("counts and audits a forgotten memory even when its cleanup fails", async () => {
+ const sdk = mockSdk();
+ const kv = mockKV();
+ registerRememberFunction(sdk as never, kv as never);
+ decrementImageRef.mockRejectedValueOnce(new Error("image store down"));
+
+ await kv.set("mem:memories", "mem_img", { ...makeMemory("mem_img"), imageRef: "img_2" });
+
+ const result = (await sdk.trigger({
+ function_id: "mem::forget",
+ payload: { memoryId: "mem_img" },
+ })) as { deleted: number; failed: number; cleanupFailed: number };
+
+ expect(await kv.get("mem:memories", "mem_img")).toBeNull();
+ expect(result.deleted).toBe(1);
+ expect(result.failed).toBe(0);
+ expect(result.cleanupFailed).toBe(1);
+ const audits = await kv.list<{ targetIds: string[] }>(currentAuditScope());
+ expect(audits[0].targetIds).toEqual(["mem_img"]);
});
it("flushes persistence immediately when a memory is forgotten", async () => {