Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

fix(state): lock session and graph read-modify-write paths (#1421)

Rohit Ghumare2493bc56850a
diff --git a/src/functions/cascade.ts b/src/functions/cascade.ts
index 2167c506..b2959676 100644
--- a/src/functions/cascade.ts
+++ b/src/functions/cascade.ts
@@ -1,6 +1,7 @@
import type { IIIClient } from "iii-sdk";
import type { StateKV } from "../state/kv.js";
import { KV } from "../state/schema.js";
+import { withKeyedLock } from "../state/keyed-mutex.js";
import type { Memory, GraphNode, GraphEdge } from "../types.js";
import { recordAudit } from "./audit.js";
@@ -28,10 +29,14 @@ export function registerCascadeFunction(sdk: IIIClient, kv: StateKV): void {
for (const node of nodes) {
if (node.stale) continue;
const overlap = (node.sourceObservationIds ?? []).some((id) => obsIds.has(id));
- if (overlap) {
- node.stale = true;
- node.updatedAt = now;
- await kv.set(KV.graphNodes, node.id, node);
+ if (!overlap) continue;
+ const flagged = await withKeyedLock("graph:persist", async () => {
+ const latest = (await kv.get<GraphNode>(KV.graphNodes, node.id)) ?? node;
+ if (latest.stale) return false;
+ await kv.set(KV.graphNodes, node.id, { ...latest, stale: true, updatedAt: now });
+ return true;
+ });
+ if (flagged) {
await recordAudit(kv, "consolidate", "mem::cascade-update", [node.id], {
resourceType: "GraphNode",
change: "marked stale from superseded memory",
@@ -45,9 +50,14 @@ export function registerCascadeFunction(sdk: IIIClient, kv: StateKV): void {
for (const edge of edges) {
if (edge.stale) continue;
const overlap = (edge.sourceObservationIds ?? []).some((id) => obsIds.has(id));
- if (overlap) {
- edge.stale = true;
- await kv.set(KV.graphEdges, edge.id, edge);
+ if (!overlap) continue;
+ const flagged = await withKeyedLock("graph:persist", async () => {
+ const latest = (await kv.get<GraphEdge>(KV.graphEdges, edge.id)) ?? edge;
+ if (latest.stale) return false;
+ await kv.set(KV.graphEdges, edge.id, { ...latest, stale: true });
+ return true;
+ });
+ if (flagged) {
await recordAudit(kv, "consolidate", "mem::cascade-update", [edge.id], {
resourceType: "GraphEdge",
change: "marked stale from superseded memory",
diff --git a/src/functions/export-import.ts b/src/functions/export-import.ts
index 6d0128f3..747cdd53 100644
--- a/src/functions/export-import.ts
+++ b/src/functions/export-import.ts
@@ -29,6 +29,7 @@ import { normalizeAccessLog } from "./access-tracker.js";
import { KV } from "../state/schema.js";
import { checkPayloadFrameSize } from "../state/frame-guard.js";
import { StateKV } from "../state/kv.js";
+import { withKeyedLock } from "../state/keyed-mutex.js";
import { VERSION } from "../version.js";
import { recordAudit } from "./audit.js";
import { indexRecords } from "./search.js";
@@ -372,13 +373,17 @@ export function registerExportImportFunction(sdk: IIIClient, kv: StateKV): void
await kv.list<Insight>(KV.insights).catch(() => []),
(i) => kv.delete(KV.insights, i.id),
);
- await runChunked(
- await kv.list<{ id: string }>(KV.graphNodes).catch(() => []),
- (n) => kv.delete(KV.graphNodes, n.id),
+ await withKeyedLock("graph:persist", async () =>
+ runChunked(
+ await kv.list<{ id: string }>(KV.graphNodes).catch(() => []),
+ (n) => kv.delete(KV.graphNodes, n.id),
+ ),
);
- await runChunked(
- await kv.list<{ id: string }>(KV.graphEdges).catch(() => []),
- (e) => kv.delete(KV.graphEdges, e.id),
+ await withKeyedLock("graph:persist", async () =>
+ runChunked(
+ await kv.list<{ id: string }>(KV.graphEdges).catch(() => []),
+ (e) => kv.delete(KV.graphEdges, e.id),
+ ),
);
await runChunked(
await kv.list<{ id: string }>(KV.semantic).catch(() => []),
@@ -407,17 +412,19 @@ export function registerExportImportFunction(sdk: IIIClient, kv: StateKV): void
const indexMems: Memory[] = [];
await runChunked(importData.sessions, async (session) => {
- if (strategy === "skip") {
- const existing = await kv
- .get<Session>(KV.sessions, session.id)
- .catch(() => null);
- if (existing) {
- stats.skipped++;
- return;
+ await withKeyedLock(`obs:${session.id}`, async () => {
+ if (strategy === "skip") {
+ const existing = await kv
+ .get<Session>(KV.sessions, session.id)
+ .catch(() => null);
+ if (existing) {
+ stats.skipped++;
+ return;
+ }
}
- }
- await kv.set(KV.sessions, session.id, session);
- stats.sessions++;
+ await kv.set(KV.sessions, session.id, session);
+ stats.sessions++;
+ });
});
for (const [sessionId, obs] of Object.entries(importData.observations)) {
@@ -473,22 +480,26 @@ export function registerExportImportFunction(sdk: IIIClient, kv: StateKV): void
});
if (importData.graphNodes) {
- await runChunked(importData.graphNodes, async (node) => {
- if (strategy === "skip") {
- const existing = await kv.get(KV.graphNodes, node.id).catch(() => null);
- if (existing) { stats.skipped++; return; }
- }
- await kv.set(KV.graphNodes, node.id, node);
- });
+ await withKeyedLock("graph:persist", () =>
+ runChunked(importData.graphNodes!, async (node) => {
+ if (strategy === "skip") {
+ const existing = await kv.get(KV.graphNodes, node.id).catch(() => null);
+ if (existing) { stats.skipped++; return; }
+ }
+ await kv.set(KV.graphNodes, node.id, node);
+ }),
+ );
}
if (importData.graphEdges) {
- await runChunked(importData.graphEdges, async (edge) => {
- if (strategy === "skip") {
- const existing = await kv.get(KV.graphEdges, edge.id).catch(() => null);
- if (existing) { stats.skipped++; return; }
- }
- await kv.set(KV.graphEdges, edge.id, edge);
- });
+ await withKeyedLock("graph:persist", () =>
+ runChunked(importData.graphEdges!, async (edge) => {
+ if (strategy === "skip") {
+ const existing = await kv.get(KV.graphEdges, edge.id).catch(() => null);
+ if (existing) { stats.skipped++; return; }
+ }
+ await kv.set(KV.graphEdges, edge.id, edge);
+ }),
+ );
}
if (importData.semanticMemories) {
await runChunked(importData.semanticMemories, async (sem) => {
diff --git a/src/functions/graph.ts b/src/functions/graph.ts
index 62e7343e..08188731 100644
--- a/src/functions/graph.ts
+++ b/src/functions/graph.ts
@@ -9,6 +9,7 @@ import type {
} from "../types.js";
import { KV, generateId } from "../state/schema.js";
import type { StateKV } from "../state/kv.js";
+import { withKeyedLock } from "../state/keyed-mutex.js";
import {
GRAPH_EXTRACTION_SYSTEM,
buildGraphExtractionPrompt,
@@ -554,129 +555,131 @@ export async function persistGraphDelta(
edges: GraphEdge[],
obsIds: string[],
): Promise<{ newNodeCount: number; newEdgeCount: number }> {
- const snap = (await readSnapshot(kv)) ?? emptySnapshot();
- const capturedAt = new Date().toISOString();
- let newNodeCount = 0;
- let newEdgeCount = 0;
- // Merge-only batches mutate cached topNodes/topEdges entries without
- // changing the counts; track that separately so the snapshot still persists.
- let snapMutated = false;
- const newEdgesForTopCheck: GraphEdge[] = [];
- // When a freshly-minted node merges into an existing row via the name
- // index, edges in the same batch still reference the fresh id. Remap edge
- // endpoints to the persisted ids so edges never dangle and re-runs hit the
- // same edge-index key instead of duplicating.
- const idRemap = new Map<string, string>();
-
- for (const node of nodes) {
- const indexKey = nameIndexKey(node.type, node.name);
- const existingId = await kv.get<string>(KV.graphNameIndex, indexKey);
-
- let existing: GraphNode | null = null;
- if (existingId) {
- existing = await kv.get<GraphNode>(KV.graphNodes, existingId);
- // #825 follow-up: name-index lookups can resolve into
- // pre-reset rows. Drop them so extract writes a fresh
- // node + index entry instead of silently reconnecting
- // to a legacy orphan (which would keep the snapshot at
- // 0 forever after a reset).
- if (
- existing &&
- snap.resetAt &&
- typeof existing.createdAt === "string" &&
- existing.createdAt < snap.resetAt
- ) {
- existing = null;
+ return withKeyedLock("graph:persist", async () => {
+ const snap = (await readSnapshot(kv)) ?? emptySnapshot();
+ const capturedAt = new Date().toISOString();
+ let newNodeCount = 0;
+ let newEdgeCount = 0;
+ // Merge-only batches mutate cached topNodes/topEdges entries without
+ // changing the counts; track that separately so the snapshot still persists.
+ let snapMutated = false;
+ const newEdgesForTopCheck: GraphEdge[] = [];
+ // When a freshly-minted node merges into an existing row via the name
+ // index, edges in the same batch still reference the fresh id. Remap edge
+ // endpoints to the persisted ids so edges never dangle and re-runs hit the
+ // same edge-index key instead of duplicating.
+ const idRemap = new Map<string, string>();
+
+ for (const node of nodes) {
+ const indexKey = nameIndexKey(node.type, node.name);
+ const existingId = await kv.get<string>(KV.graphNameIndex, indexKey);
+
+ let existing: GraphNode | null = null;
+ if (existingId) {
+ existing = await kv.get<GraphNode>(KV.graphNodes, existingId);
+ // #825 follow-up: name-index lookups can resolve into
+ // pre-reset rows. Drop them so extract writes a fresh
+ // node + index entry instead of silently reconnecting
+ // to a legacy orphan (which would keep the snapshot at
+ // 0 forever after a reset).
+ if (
+ existing &&
+ snap.resetAt &&
+ typeof existing.createdAt === "string" &&
+ existing.createdAt < snap.resetAt
+ ) {
+ existing = null;
+ }
}
- }
- if (existing) {
- idRemap.set(node.id, existing.id);
- const merged = mergeNode(existing, node, obsIds, capturedAt);
- await kv.set(KV.graphNodes, existing.id, merged);
- // Update topNodes entry if present so a stale clone isn't
- // returned from the snapshot fast path.
- const topIdx = snap.topNodes.findIndex((n) => n.id === existing!.id);
- if (topIdx !== -1) {
- snap.topNodes[topIdx] = merged;
- snapMutated = true;
- }
- } else {
- await kv.set(KV.graphNodes, node.id, node);
- await kv.set(KV.graphNameIndex, indexKey, node.id);
- await kv.set(KV.graphNodeDegree, node.id, 0);
- snap.stats.totalNodes += 1;
- snap.stats.nodesByType[node.type] =
- (snap.stats.nodesByType[node.type] ?? 0) + 1;
- newNodeCount += 1;
- if (snap.topNodes.length < SNAPSHOT_TOP_NODES) {
- // Degree 0 still beats an empty slot — sit at the tail
- // until edges arrive and promote.
- snap.topNodes.push(node);
- snap.topDegrees[node.id] = 0;
+ if (existing) {
+ idRemap.set(node.id, existing.id);
+ const merged = mergeNode(existing, node, obsIds, capturedAt);
+ await kv.set(KV.graphNodes, existing.id, merged);
+ // Update topNodes entry if present so a stale clone isn't
+ // returned from the snapshot fast path.
+ const topIdx = snap.topNodes.findIndex((n) => n.id === existing!.id);
+ if (topIdx !== -1) {
+ snap.topNodes[topIdx] = merged;
+ snapMutated = true;
+ }
+ } else {
+ await kv.set(KV.graphNodes, node.id, node);
+ await kv.set(KV.graphNameIndex, indexKey, node.id);
+ await kv.set(KV.graphNodeDegree, node.id, 0);
+ snap.stats.totalNodes += 1;
+ snap.stats.nodesByType[node.type] =
+ (snap.stats.nodesByType[node.type] ?? 0) + 1;
+ newNodeCount += 1;
+ if (snap.topNodes.length < SNAPSHOT_TOP_NODES) {
+ // Degree 0 still beats an empty slot — sit at the tail
+ // until edges arrive and promote.
+ snap.topNodes.push(node);
+ snap.topDegrees[node.id] = 0;
+ }
}
}
- }
- for (const rawEdge of edges) {
- const edge: GraphEdge = {
- ...rawEdge,
- sourceNodeId: idRemap.get(rawEdge.sourceNodeId) ?? rawEdge.sourceNodeId,
- targetNodeId: idRemap.get(rawEdge.targetNodeId) ?? rawEdge.targetNodeId,
- };
- const eKey = edgeIndexKey(edge.sourceNodeId, edge.targetNodeId, edge.type);
- const existingId = await kv.get<string>(KV.graphEdgeKey, eKey);
-
- let existing: GraphEdge | null = null;
- if (existingId) {
- existing = await kv.get<GraphEdge>(KV.graphEdges, existingId);
- // Same #825 orphan check as the node path above.
- if (
- existing &&
- snap.resetAt &&
- typeof existing.createdAt === "string" &&
- existing.createdAt < snap.resetAt
- ) {
- existing = null;
+ for (const rawEdge of edges) {
+ const edge: GraphEdge = {
+ ...rawEdge,
+ sourceNodeId: idRemap.get(rawEdge.sourceNodeId) ?? rawEdge.sourceNodeId,
+ targetNodeId: idRemap.get(rawEdge.targetNodeId) ?? rawEdge.targetNodeId,
+ };
+ const eKey = edgeIndexKey(edge.sourceNodeId, edge.targetNodeId, edge.type);
+ const existingId = await kv.get<string>(KV.graphEdgeKey, eKey);
+
+ let existing: GraphEdge | null = null;
+ if (existingId) {
+ existing = await kv.get<GraphEdge>(KV.graphEdges, existingId);
+ // Same #825 orphan check as the node path above.
+ if (
+ existing &&
+ snap.resetAt &&
+ typeof existing.createdAt === "string" &&
+ existing.createdAt < snap.resetAt
+ ) {
+ existing = null;
+ }
}
- }
- if (existing) {
- const merged = mergeEdge(existing, obsIds);
- await kv.set(KV.graphEdges, existing.id, merged);
- // Replace cached topEdges entry too if present.
- const topIdx = snap.topEdges.findIndex((e) => e.id === existing!.id);
- if (topIdx !== -1) {
- snap.topEdges[topIdx] = merged;
- snapMutated = true;
+ if (existing) {
+ const merged = mergeEdge(existing, obsIds);
+ await kv.set(KV.graphEdges, existing.id, merged);
+ // Replace cached topEdges entry too if present.
+ const topIdx = snap.topEdges.findIndex((e) => e.id === existing!.id);
+ if (topIdx !== -1) {
+ snap.topEdges[topIdx] = merged;
+ snapMutated = true;
+ }
+ } else {
+ await kv.set(KV.graphEdges, edge.id, edge);
+ await kv.set(KV.graphEdgeKey, eKey, edge.id);
+ snap.stats.totalEdges += 1;
+ snap.stats.edgesByType[edge.type] =
+ (snap.stats.edgesByType[edge.type] ?? 0) + 1;
+ newEdgeCount += 1;
+ await applyDegreeDelta(kv, snap, edge.sourceNodeId, +1);
+ await applyDegreeDelta(kv, snap, edge.targetNodeId, +1);
+ newEdgesForTopCheck.push(edge);
}
- } else {
- await kv.set(KV.graphEdges, edge.id, edge);
- await kv.set(KV.graphEdgeKey, eKey, edge.id);
- snap.stats.totalEdges += 1;
- snap.stats.edgesByType[edge.type] =
- (snap.stats.edgesByType[edge.type] ?? 0) + 1;
- newEdgeCount += 1;
- await applyDegreeDelta(kv, snap, edge.sourceNodeId, +1);
- await applyDegreeDelta(kv, snap, edge.targetNodeId, +1);
- newEdgesForTopCheck.push(edge);
}
- }
- // Push newly-added edges into snapshot.topEdges if both
- // endpoints are in the top-N (post-degree-delta). Done after
- // all degree updates so the topIds set is stable.
- for (const edge of newEdgesForTopCheck) {
- snapshotPushEdgeIfBothInTop(snap, edge);
- }
+ // Push newly-added edges into snapshot.topEdges if both
+ // endpoints are in the top-N (post-degree-delta). Done after
+ // all degree updates so the topIds set is stable.
+ for (const edge of newEdgesForTopCheck) {
+ snapshotPushEdgeIfBothInTop(snap, edge);
+ }
- if (newNodeCount > 0 || newEdgeCount > 0 || snapMutated) {
- snap.updatedAt = capturedAt;
- snap.dirty = false;
- await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, snap);
- }
+ if (newNodeCount > 0 || newEdgeCount > 0 || snapMutated) {
+ snap.updatedAt = capturedAt;
+ snap.dirty = false;
+ await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, snap);
+ }
- return { newNodeCount, newEdgeCount };
+ return { newNodeCount, newEdgeCount };
+ });
}
export function registerGraphFunction(
@@ -1014,70 +1017,74 @@ export function registerGraphFunction(
}
try {
- const [nodes, edges] = await withTimeout(
- Promise.all([
- kv.list<GraphNode>(KV.graphNodes),
- kv.list<GraphEdge>(KV.graphEdges),
- ]),
- LIVE_ENUMERATION_BUDGET_MS,
- "graph-snapshot-rebuild enumeration",
- );
+ const outcome = await withKeyedLock("graph:persist", async () => {
+ const [nodes, edges] = await withTimeout(
+ Promise.all([
+ kv.list<GraphNode>(KV.graphNodes),
+ kv.list<GraphEdge>(KV.graphEdges),
+ ]),
+ LIVE_ENUMERATION_BUDGET_MS,
+ "graph-snapshot-rebuild enumeration",
+ );
+
+ if (nodes.length > REBUILD_SAFE_NODE_CEILING) {
+ return { tooLargeNodeCount: nodes.length };
+ }
+
+ const liveNodes = nodes.filter((n) => !n.stale);
+ const liveEdges = edges.filter((e) => !e.stale);
+ const degree = new Map<string, number>();
+ for (const e of liveEdges) {
+ degree.set(e.sourceNodeId, (degree.get(e.sourceNodeId) ?? 0) + 1);
+ degree.set(e.targetNodeId, (degree.get(e.targetNodeId) ?? 0) + 1);
+ }
+ const BATCH_SIZE = 100;
+ for (let i = 0; i < liveNodes.length; i += BATCH_SIZE) {
+ const batch = liveNodes.slice(i, i + BATCH_SIZE);
+ await Promise.all(
+ batch.flatMap((n) => [
+ kv.set(KV.graphNameIndex, nameIndexKey(n.type, n.name), n.id),
+ kv.set(KV.graphNodeDegree, n.id, degree.get(n.id) ?? 0),
+ ]),
+ );
+ }
+ for (let i = 0; i < liveEdges.length; i += BATCH_SIZE) {
+ const batch = liveEdges.slice(i, i + BATCH_SIZE);
+ await Promise.all(
+ batch.map((e) =>
+ kv.set(
+ KV.graphEdgeKey,
+ edgeIndexKey(e.sourceNodeId, e.targetNodeId, e.type),
+ e.id,
+ ),
+ ),
+ );
+ }
- if (nodes.length > REBUILD_SAFE_NODE_CEILING) {
+ const rebuilt = buildSnapshotFromArrays(nodes, edges);
+ await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, rebuilt);
+ return { snap: rebuilt };
+ });
+
+ if ("tooLargeNodeCount" in outcome) {
+ const totalNodes = outcome.tooLargeNodeCount;
logger.warn("Graph snapshot rebuild aborted: corpus too large", {
- totalNodes: nodes.length,
+ totalNodes,
ceiling: REBUILD_SAFE_NODE_CEILING,
});
return {
success: false,
tooLarge: true,
- totalNodes: nodes.length,
+ totalNodes,
ceiling: REBUILD_SAFE_NODE_CEILING,
error:
- `Corpus has ${nodes.length} graph nodes; safe-rebuild ceiling ` +
+ `Corpus has ${totalNodes} graph nodes; safe-rebuild ceiling ` +
`is ${REBUILD_SAFE_NODE_CEILING}. Run POST /agentmemory/graph/reset ` +
`to wipe and let future extracts rebuild incrementally.`,
};
}
- // Backfill the targeted-lookup indexes so post-rebuild
- // graph-extract calls hit the O(1) path instead of falling
- // through to the (already-removed) full-scope scan. Batch
- // writes via Promise.all to avoid N sequential round-trips —
- // BATCH_SIZE bounds in-flight writes so we don't open thousands
- // of concurrent state channels on huge corpora.
- const liveNodes = nodes.filter((n) => !n.stale);
- const liveEdges = edges.filter((e) => !e.stale);
- const degree = new Map<string, number>();
- for (const e of liveEdges) {
- degree.set(e.sourceNodeId, (degree.get(e.sourceNodeId) ?? 0) + 1);
- degree.set(e.targetNodeId, (degree.get(e.targetNodeId) ?? 0) + 1);
- }
- const BATCH_SIZE = 100;
- for (let i = 0; i < liveNodes.length; i += BATCH_SIZE) {
- const batch = liveNodes.slice(i, i + BATCH_SIZE);
- await Promise.all(
- batch.flatMap((n) => [
- kv.set(KV.graphNameIndex, nameIndexKey(n.type, n.name), n.id),
- kv.set(KV.graphNodeDegree, n.id, degree.get(n.id) ?? 0),
- ]),
- );
- }
- for (let i = 0; i < liveEdges.length; i += BATCH_SIZE) {
- const batch = liveEdges.slice(i, i + BATCH_SIZE);
- await Promise.all(
- batch.map((e) =>
- kv.set(
- KV.graphEdgeKey,
- edgeIndexKey(e.sourceNodeId, e.targetNodeId, e.type),
- e.id,
- ),
- ),
- );
- }
-
- const snap = buildSnapshotFromArrays(nodes, edges);
- await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, snap);
+ const { snap } = outcome;
const tookMs = Date.now() - started;
logger.info("Graph snapshot rebuilt", {
totalNodes: snap.stats.totalNodes,
@@ -1134,7 +1141,9 @@ export function registerGraphFunction(
...emptySnapshot(),
resetAt: new Date().toISOString(),
};
- await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, resetSnapshot);
+ await withKeyedLock("graph:persist", () =>
+ kv.set(KV.graphSnapshot, SNAPSHOT_KEY, resetSnapshot),
+ );
const counts: Record<string, number> = {
[KV.graphSnapshot]: 1,
};
diff --git a/src/functions/mesh.ts b/src/functions/mesh.ts
index 139765ad..0dd7e957 100644
--- a/src/functions/mesh.ts
+++ b/src/functions/mesh.ts
@@ -119,7 +119,7 @@ async function lwwMergeGraphNodes(
if (!item.id || typeof item.id !== "string") continue;
const ts = graphNodeTs(item);
if (!ts || Number.isNaN(new Date(ts).getTime())) continue;
- const wrote = await withKeyedLock(`mem:gnode:${item.id}`, async () => {
+ const wrote = await withKeyedLock("graph:persist", async () => {
const existing = await kv.get<GraphNode>(KV.graphNodes, item.id);
if (!existing) {
await kv.set(KV.graphNodes, item.id, item);
@@ -136,6 +136,33 @@ async function lwwMergeGraphNodes(
return count;
}
+async function lwwMergeGraphEdges(
+ kv: StateKV,
+ items: GraphEdge[] | undefined,
+): Promise<number> {
+ if (!items || !Array.isArray(items)) return 0;
+ let count = 0;
+ for (const item of items) {
+ if (!item.id || typeof item.id !== "string") continue;
+ const ts = item.createdAt;
+ if (!ts || Number.isNaN(new Date(ts).getTime())) continue;
+ const wrote = await withKeyedLock("graph:persist", async () => {
+ const existing = await kv.get<GraphEdge>(KV.graphEdges, item.id);
+ if (!existing) {
+ await kv.set(KV.graphEdges, item.id, item);
+ return true;
+ }
+ if (new Date(ts) > new Date(existing.createdAt)) {
+ await kv.set(KV.graphEdges, item.id, item);
+ return true;
+ }
+ return false;
+ });
+ if (wrote) count++;
+ }
+ return count;
+}
+
export function registerMeshFunction(
sdk: IIIClient,
kv: StateKV,
@@ -361,7 +388,7 @@ export function registerMeshFunction(
}
}
accepted += await lwwMergeGraphNodes(kv, data.graphNodes);
- accepted += await lwwMergeList(kv, KV.graphEdges, data.graphEdges, "mem:gedge", "createdAt");
+ accepted += await lwwMergeGraphEdges(kv, data.graphEdges);
await recordAudit(kv, "mesh_sync", "mem::mesh-receive", [], {
action: "mesh.receive",
accepted,
@@ -488,7 +515,7 @@ async function applySyncData(
applied += await lwwMergeGraphNodes(kv, data.graphNodes);
}
if (scopes.includes("graph:edges")) {
- applied += await lwwMergeList(kv, KV.graphEdges, data.graphEdges, "mem:gedge", "createdAt");
+ applied += await lwwMergeGraphEdges(kv, data.graphEdges);
}
return applied;
diff --git a/src/functions/migrate.ts b/src/functions/migrate.ts
index 8cabef53..350a5332 100644
--- a/src/functions/migrate.ts
+++ b/src/functions/migrate.ts
@@ -3,6 +3,7 @@ import { resolve } from "node:path";
import { homedir } from "node:os";
import { KV, generateId } from "../state/schema.js";
import { StateKV } from "../state/kv.js";
+import { withKeyedLock } from "../state/keyed-mutex.js";
import type {
Memory,
Session,
@@ -150,7 +151,9 @@ export function registerMigrateFunction(sdk: IIIClient, kv: StateKV): void {
status: "completed",
observationCount: 0,
};
- await kv.set(KV.sessions, session.id, session);
+ await withKeyedLock(`obs:${session.id}`, () =>
+ kv.set(KV.sessions, session.id, session),
+ );
sessionCount++;
}
diff --git a/src/functions/replay.ts b/src/functions/replay.ts
index aea735ff..d158b913 100644
--- a/src/functions/replay.ts
+++ b/src/functions/replay.ts
@@ -12,6 +12,7 @@ import type {
import { importOrigin } from "../types.js";
import type { StateKV } from "../state/kv.js";
import { KV, generateId, fingerprintId } from "../state/schema.js";
+import { withKeyedLock } from "../state/keyed-mutex.js";
import { parseJsonlText } from "../replay/jsonl-parser.js";
import { resetLessonIndex } from "./lessons.js";
import { projectTimeline, type Timeline } from "../replay/timeline.js";
@@ -394,46 +395,48 @@ export function registerReplayFunctions(sdk: IIIClient, kv: StateKV): void {
? firstPromptObs.userPrompt.replace(/\s+/g, " ").trim().slice(0, 200)
: undefined;
- const existing = await kv.get<Session>(KV.sessions, parsed.sessionId);
- if (existing) {
- existing.observationCount =
- (existing.observationCount || 0) + parsed.observations.length;
- if (parsed.endedAt > (existing.endedAt || "")) {
- existing.endedAt = parsed.endedAt;
+ await withKeyedLock(`obs:${parsed.sessionId}`, async () => {
+ const existing = await kv.get<Session>(KV.sessions, parsed.sessionId);
+ if (existing) {
+ existing.observationCount =
+ (existing.observationCount || 0) + parsed.observations.length;
+ if (parsed.endedAt > (existing.endedAt || "")) {
+ existing.endedAt = parsed.endedAt;
+ }
+ if (existing.status === "active") existing.status = "completed";
+ const existingTags = existing.tags || [];
+ if (!existingTags.includes("jsonl-import")) {
+ existing.tags = [...existingTags, "jsonl-import"];
+ }
+ if (!existing.firstPrompt && firstPrompt) {
+ existing.firstPrompt = firstPrompt;
+ }
+ // #775: re-key on parsed.sessionId, not existing.id. Older
+ // session rows may be missing the `id` field; existing.id
+ // would then be undefined, JSON.stringify would drop the
+ // `key` from the state::set payload, and the engine would
+ // reject the call with `missing field \`key\``. Because the
+ // rejection aborts the whole import handler, a single
+ // legacy row killed the entire batch. parsed.sessionId is
+ // always populated (parseJsonlText has a three-level
+ // fallback) and is what we just used to read the row.
+ if (!existing.id) existing.id = parsed.sessionId;
+ await kv.set(KV.sessions, parsed.sessionId, existing);
+ } else {
+ const session: Session = {
+ id: parsed.sessionId,
+ project: parsed.project,
+ cwd: parsed.cwd,
+ startedAt: parsed.startedAt,
+ endedAt: parsed.endedAt,
+ status: "completed",
+ observationCount: parsed.observations.length,
+ tags: ["jsonl-import"],
+ firstPrompt,
+ };
+ await kv.set(KV.sessions, session.id, session);
}
- if (existing.status === "active") existing.status = "completed";
- const existingTags = existing.tags || [];
- if (!existingTags.includes("jsonl-import")) {
- existing.tags = [...existingTags, "jsonl-import"];
- }
- if (!existing.firstPrompt && firstPrompt) {
- existing.firstPrompt = firstPrompt;
- }
- // #775: re-key on parsed.sessionId, not existing.id. Older
- // session rows may be missing the `id` field; existing.id
- // would then be undefined, JSON.stringify would drop the
- // `key` from the state::set payload, and the engine would
- // reject the call with `missing field \`key\``. Because the
- // rejection aborts the whole import handler, a single
- // legacy row killed the entire batch. parsed.sessionId is
- // always populated (parseJsonlText has a three-level
- // fallback) and is what we just used to read the row.
- if (!existing.id) existing.id = parsed.sessionId;
- await kv.set(KV.sessions, parsed.sessionId, existing);
- } else {
- const session: Session = {
- id: parsed.sessionId,
- project: parsed.project,
- cwd: parsed.cwd,
- startedAt: parsed.startedAt,
- endedAt: parsed.endedAt,
- status: "completed",
- observationCount: parsed.observations.length,
- tags: ["jsonl-import"],
- firstPrompt,
- };
- await kv.set(KV.sessions, session.id, session);
- }
+ });
const compressed: CompressedObservation[] = [];
await Promise.all(
diff --git a/src/functions/snapshot.ts b/src/functions/snapshot.ts
index 483240e1..06673845 100644
--- a/src/functions/snapshot.ts
+++ b/src/functions/snapshot.ts
@@ -12,6 +12,7 @@ import type {
} from "../types.js";
import { KV, generateId } from "../state/schema.js";
import type { StateKV } from "../state/kv.js";
+import { withKeyedLock } from "../state/keyed-mutex.js";
import { recordAudit } from "./audit.js";
import { VERSION } from "../version.js";
import { logger } from "../logger.js";
@@ -195,7 +196,9 @@ export function registerSnapshotFunction(
if (state.sessions) {
for (const session of state.sessions) {
- await kv.set(KV.sessions, session.id, session);
+ await withKeyedLock(`obs:${session.id}`, () =>
+ kv.set(KV.sessions, session.id, session),
+ );
}
}
if (state.memories) {
@@ -205,7 +208,9 @@ export function registerSnapshotFunction(
}
if (state.graphNodes) {
for (const node of state.graphNodes) {
- await kv.set(KV.graphNodes, node.id, node);
+ await withKeyedLock("graph:persist", () =>
+ kv.set(KV.graphNodes, node.id, node),
+ );
}
}
if (state.observations) {
diff --git a/src/functions/temporal-graph.ts b/src/functions/temporal-graph.ts
index aed6730b..18d13c1f 100644
--- a/src/functions/temporal-graph.ts
+++ b/src/functions/temporal-graph.ts
@@ -9,6 +9,7 @@ import type {
} from "../types.js";
import { KV, generateId } from "../state/schema.js";
import type { StateKV } from "../state/kv.js";
+import { withKeyedLock } from "../state/keyed-mutex.js";
import { logger } from "../logger.js";
const TEMPORAL_EXTRACTION_SYSTEM = `You are a temporal knowledge extraction engine. Given observations, extract entities AND their temporal relationships with full context metadata.
@@ -186,76 +187,78 @@ export function registerTemporalGraphFunctions(
const obsIds = data.observations.map((o) => o.id);
const { nodes, edges } = parseTemporalGraphXml(response, obsIds);
- const existingNodes = await kv.list<GraphNode>(KV.graphNodes);
- const existingEdges = await kv.list<GraphEdge>(KV.graphEdges);
-
- const idRemap = new Map<string, string>();
- for (const node of nodes) {
- const existing = existingNodes.find(
- (n) =>
- n.name === node.name && n.type === node.type,
- );
- if (existing) {
- const oldId = node.id;
- const merged = {
- ...existing,
- sourceObservationIds: [
- ...new Set([
- ...existing.sourceObservationIds,
- ...obsIds,
- ]),
- ],
- properties: { ...existing.properties, ...node.properties },
- updatedAt: new Date().toISOString(),
- aliases: [
- ...new Set([
- ...(existing.aliases || []),
- ...(node.aliases || []),
- ]),
- ],
- };
- if (merged.aliases.length === 0) delete (merged as any).aliases;
- await kv.set(KV.graphNodes, existing.id, merged);
- node.id = existing.id;
- idRemap.set(oldId, existing.id);
- } else {
- await kv.set(KV.graphNodes, node.id, node);
- existingNodes.push(node);
+ await withKeyedLock("graph:persist", async () => {
+ const existingNodes = await kv.list<GraphNode>(KV.graphNodes);
+ const existingEdges = await kv.list<GraphEdge>(KV.graphEdges);
+
+ const idRemap = new Map<string, string>();
+ for (const node of nodes) {
+ const existing = existingNodes.find(
+ (n) =>
+ n.name === node.name && n.type === node.type,
+ );
+ if (existing) {
+ const oldId = node.id;
+ const merged = {
+ ...existing,
+ sourceObservationIds: [
+ ...new Set([
+ ...existing.sourceObservationIds,
+ ...obsIds,
+ ]),
+ ],
+ properties: { ...existing.properties, ...node.properties },
+ updatedAt: new Date().toISOString(),
+ aliases: [
+ ...new Set([
+ ...(existing.aliases || []),
+ ...(node.aliases || []),
+ ]),
+ ],
+ };
+ if (merged.aliases.length === 0) delete (merged as any).aliases;
+ await kv.set(KV.graphNodes, existing.id, merged);
+ node.id = existing.id;
+ idRemap.set(oldId, existing.id);
+ } else {
+ await kv.set(KV.graphNodes, node.id, node);
+ existingNodes.push(node);
+ }
}
- }
- for (const edge of edges) {
- if (idRemap.has(edge.sourceNodeId)) {
- edge.sourceNodeId = idRemap.get(edge.sourceNodeId)!;
+ for (const edge of edges) {
+ if (idRemap.has(edge.sourceNodeId)) {
+ edge.sourceNodeId = idRemap.get(edge.sourceNodeId)!;
+ }
+ if (idRemap.has(edge.targetNodeId)) {
+ edge.targetNodeId = idRemap.get(edge.targetNodeId)!;
+ }
+ const existingKey = `${edge.sourceNodeId}|${edge.targetNodeId}|${edge.type}`;
+ const existingEdge = existingEdges.find(
+ (e) =>
+ `${e.sourceNodeId}|${e.targetNodeId}|${e.type}` ===
+ existingKey,
+ );
+
+ if (existingEdge) {
+ const updatedOld = {
+ ...existingEdge,
+ isLatest: false,
+ tvalidEnd:
+ existingEdge.tvalidEnd || new Date().toISOString(),
+ supersededBy: edge.id,
+ };
+ await kv.set(KV.graphEdges, existingEdge.id, updatedOld);
+
+ await kv.set(KV.graphEdgeHistory, existingEdge.id, updatedOld);
+
+ edge.version = (existingEdge.version || 1) + 1;
+ }
+
+ await kv.set(KV.graphEdges, edge.id, edge);
+ existingEdges.push(edge);
}
- if (idRemap.has(edge.targetNodeId)) {
- edge.targetNodeId = idRemap.get(edge.targetNodeId)!;
- }
- const existingKey = `${edge.sourceNodeId}|${edge.targetNodeId}|${edge.type}`;
- const existingEdge = existingEdges.find(
- (e) =>
- `${e.sourceNodeId}|${e.targetNodeId}|${e.type}` ===
- existingKey,
- );
-
- if (existingEdge) {
- const updatedOld = {
- ...existingEdge,
- isLatest: false,
- tvalidEnd:
- existingEdge.tvalidEnd || new Date().toISOString(),
- supersededBy: edge.id,
- };
- await kv.set(KV.graphEdges, existingEdge.id, updatedOld);
-
- await kv.set(KV.graphEdgeHistory, existingEdge.id, updatedOld);
-
- edge.version = (existingEdge.version || 1) + 1;
- }
-
- await kv.set(KV.graphEdges, edge.id, edge);
- existingEdges.push(edge);
- }
+ });
logger.info("Temporal graph extraction complete", {
nodes: nodes.length,
diff --git a/src/triggers/api.ts b/src/triggers/api.ts
index 4cf5431b..752ec03c 100644
--- a/src/triggers/api.ts
+++ b/src/triggers/api.ts
@@ -731,7 +731,7 @@ export function registerApiTriggers(
? body.agentId.trim().slice(0, 128)
: undefined;
const agentId = requestAgentId ?? getAgentId();
- const session: Session = {
+ const freshSession: Session = {
id: sessionId,
project,
cwd,
@@ -742,7 +742,18 @@ export function registerApiTriggers(
...(title ? { firstPrompt: title.slice(0, 200) } : {}),
...(agentId ? { agentId } : {}),
};
- await kv.set(KV.sessions, sessionId, session);
+ const session = await withKeyedLock(`obs:${sessionId}`, async () => {
+ const existing = await kv.get<Session>(KV.sessions, sessionId);
+ const merged: Session = {
+ ...freshSession,
+ observationCount: existing?.observationCount ?? freshSession.observationCount,
+ firstPrompt: freshSession.firstPrompt ?? existing?.firstPrompt,
+ summary: freshSession.summary ?? existing?.summary,
+ commitShas: existing?.commitShas ?? freshSession.commitShas,
+ };
+ await kv.set(KV.sessions, sessionId, merged);
+ return merged;
+ });
const contextResult = await sdk.trigger<
{ sessionId: string; project: string; agentId?: string },
{ context: string }
@@ -775,10 +786,12 @@ export function registerApiTriggers(
body: { error: "sessionId is required and must be a non-empty string" },
};
}
- await kv.update(KV.sessions, sessionId, [
- { type: "set", path: "endedAt", value: new Date().toISOString() },
- { type: "set", path: "status", value: "completed" },
- ]);
+ await withKeyedLock(`obs:${sessionId}`, () =>
+ kv.update(KV.sessions, sessionId, [
+ { type: "set", path: "endedAt", value: new Date().toISOString() },
+ { type: "set", path: "status", value: "completed" },
+ ]),
+ );
// Fan out session-stopped lifecycle (non-blocking).
try {
sdk.trigger({
@@ -871,7 +884,7 @@ export function registerApiTriggers(
});
if (sessionId) {
- await withKeyedLock(`session:${sessionId}`, async () => {
+ await withKeyedLock(`obs:${sessionId}`, async () => {
const session = await kv.get<Session>(KV.sessions, sessionId);
if (!session) return;
const shaSet = new Set<string>(session.commitShas ?? []);
diff --git a/src/triggers/events.ts b/src/triggers/events.ts
index 1e248236..c3187ce5 100644
--- a/src/triggers/events.ts
+++ b/src/triggers/events.ts
@@ -2,6 +2,7 @@ import { TriggerAction, type IIIClient } from "iii-sdk";
import type { CompressedObservation, HookPayload, Memory, Session } from "../types.js";
import { KV, STREAM } from "../state/schema.js";
import { StateKV } from "../state/kv.js";
+import { withKeyedLock } from "../state/keyed-mutex.js";
import { isReflectEnabled } from "../functions/slots.js";
import {
detectLlmProviderKind,
@@ -56,7 +57,7 @@ export function registerEventTriggers(sdk: IIIClient, kv: StateKV): void {
? data.agentId.trim().slice(0, 128)
: undefined;
const agentId = requestAgentId ?? getAgentId();
- const session: Session = {
+ const freshSession: Session = {
id: data.sessionId,
project: data.project,
cwd: data.cwd,
@@ -65,7 +66,18 @@ export function registerEventTriggers(sdk: IIIClient, kv: StateKV): void {
observationCount: 0,
...(agentId ? { agentId } : {}),
};
- await kv.set(KV.sessions, data.sessionId, session);
+ const session = await withKeyedLock(`obs:${data.sessionId}`, async () => {
+ const existing = await kv.get<Session>(KV.sessions, data.sessionId);
+ const merged: Session = {
+ ...freshSession,
+ observationCount: existing?.observationCount ?? freshSession.observationCount,
+ firstPrompt: existing?.firstPrompt,
+ summary: existing?.summary,
+ commitShas: existing?.commitShas,
+ };
+ await kv.set(KV.sessions, data.sessionId, merged);
+ return merged;
+ });
const contextResult = await sdk.trigger<
{ sessionId: string; project: string; agentId?: string },
{ context: string }
@@ -160,10 +172,12 @@ export function registerEventTriggers(sdk: IIIClient, kv: StateKV): void {
sdk.registerFunction(
"event::session::ended",
async (data: { sessionId: string }) => {
- await kv.update(KV.sessions, data.sessionId, [
- { type: "set", path: "endedAt", value: new Date().toISOString() },
- { type: "set", path: "status", value: "completed" },
- ]);
+ await withKeyedLock(`obs:${data.sessionId}`, () =>
+ kv.update(KV.sessions, data.sessionId, [
+ { type: "set", path: "endedAt", value: new Date().toISOString() },
+ { type: "set", path: "status", value: "completed" },
+ ]),
+ );
return { success: true };
},
);
diff --git a/test/graph-cross-path-locks.test.ts b/test/graph-cross-path-locks.test.ts
new file mode 100644
index 00000000..cbab748d
--- /dev/null
+++ b/test/graph-cross-path-locks.test.ts
@@ -0,0 +1,161 @@
+import { describe, it, expect, vi } from "vitest";
+
+vi.mock("../src/logger.js", () => ({
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+}));
+
+import { persistGraphDelta } from "../src/functions/graph.js";
+import { registerCascadeFunction } from "../src/functions/cascade.js";
+import { KV } from "../src/state/schema.js";
+import type { GraphNode, GraphSnapshot, Memory } from "../src/types.js";
+
+function clone<T>(value: T): T {
+ return value === null || value === undefined
+ ? value
+ : (JSON.parse(JSON.stringify(value)) as T);
+}
+
+function mockKV() {
+ const store = new Map<string, Map<string, unknown>>();
+ let gate: {
+ scope: string;
+ key: string;
+ release: () => void;
+ promise: Promise<void>;
+ } | null = null;
+
+ return {
+ store,
+ armGetGate: (scope: string, key: string) => {
+ let release!: () => void;
+ const promise = new Promise<void>((res) => {
+ release = res;
+ });
+ gate = { scope, key, release, promise };
+ return release;
+ },
+ get: async <T>(scope: string, key: string): Promise<T | null> => {
+ const value = clone((store.get(scope)?.get(key) as T) ?? null);
+ if (gate && gate.scope === scope && gate.key === key) {
+ const g = gate;
+ gate = null;
+ await g.promise;
+ }
+ return value;
+ },
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ const stored = clone(data);
+ if (!store.has(scope)) store.set(scope, new Map());
+ store.get(scope)!.set(key, stored);
+ return clone(stored);
+ },
+ delete: async (scope: string, key: string) => {
+ store.get(scope)?.delete(key);
+ },
+ list: async <T>(scope: string): Promise<T[]> => {
+ const m = store.get(scope);
+ return m ? clone(Array.from(m.values()) as T[]) : [];
+ },
+ };
+}
+
+function mockSdk() {
+ const fns = new Map<string, (data: unknown) => Promise<unknown>>();
+ return {
+ registerFunction: (id: string, handler: (data: unknown) => Promise<unknown>) => {
+ fns.set(id, handler);
+ },
+ trigger: (id: string, data: unknown) => fns.get(id)!(data),
+ };
+}
+
+async function flush() {
+ await new Promise((r) => setTimeout(r, 0));
+}
+
+describe("graph:persist serializes cross-path graph mutations", () => {
+ it("does not lose cascade's stale flag or persistGraphDelta's merge when they race on the same node", async () => {
+ const kv = mockKV();
+
+ const node: GraphNode = {
+ id: "node_1",
+ type: "concept",
+ name: "react",
+ properties: {},
+ sourceObservationIds: ["obs_old"],
+ createdAt: "2026-03-01T00:00:00Z",
+ };
+ await kv.set(KV.graphNodes, "node_1", node);
+ await kv.set(KV.graphNameIndex, "concept|react", "node_1");
+ await kv.set(KV.graphNodeDegree, "node_1", 0);
+ await kv.set(KV.graphSnapshot, "current", {
+ version: 1,
+ topNodes: [node],
+ topEdges: [],
+ topDegrees: {},
+ stats: {
+ totalNodes: 1,
+ totalEdges: 0,
+ nodesByType: { concept: 1 },
+ edgesByType: {},
+ },
+ updatedAt: new Date().toISOString(),
+ dirty: false,
+ } satisfies GraphSnapshot);
+
+ const supersededMemory: Memory = {
+ id: "mem_old",
+ createdAt: "2026-03-01T00:00:00Z",
+ updatedAt: "2026-03-01T00:00:00Z",
+ type: "fact",
+ title: "Old fact",
+ content: "Old content",
+ concepts: [],
+ files: [],
+ sessionIds: [],
+ strength: 5,
+ version: 1,
+ isLatest: false,
+ sourceObservationIds: ["obs_old"],
+ };
+ await kv.set(KV.memories, "mem_old", supersededMemory);
+
+ const sdk = mockSdk();
+ registerCascadeFunction(sdk as never, kv as never);
+
+ const releaseNodeRowRead = kv.armGetGate(KV.graphNodes, "node_1");
+
+ const incomingNode: GraphNode = {
+ id: "node_new",
+ type: "concept",
+ name: "react",
+ properties: {},
+ sourceObservationIds: ["obs_new"],
+ createdAt: new Date().toISOString(),
+ };
+ const persistPromise = persistGraphDelta(
+ kv as never,
+ [incomingNode],
+ [],
+ ["obs_new"],
+ );
+
+ await flush();
+
+ const cascadePromise = sdk.trigger("mem::cascade-update", {
+ supersededMemoryId: "mem_old",
+ }) as Promise<{ success: boolean; flagged: { nodes: number } }>;
+
+ await flush();
+ releaseNodeRowRead();
+
+ const [, cascadeResult] = await Promise.all([persistPromise, cascadePromise]);
+
+ expect(cascadeResult.success).toBe(true);
+ expect(cascadeResult.flagged.nodes).toBe(1);
+
+ const finalNode = await kv.get<GraphNode>(KV.graphNodes, "node_1");
+ expect(finalNode!.stale).toBe(true);
+ expect(finalNode!.sourceObservationIds.sort()).toEqual(["obs_new", "obs_old"]);
+ });
+});
diff --git a/test/graph-extract-concurrency.test.ts b/test/graph-extract-concurrency.test.ts
new file mode 100644
index 00000000..ca022a66
--- /dev/null
+++ b/test/graph-extract-concurrency.test.ts
@@ -0,0 +1,225 @@
+import { describe, it, expect, vi } from "vitest";
+
+vi.mock("../src/logger.js", () => ({
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+}));
+
+import { persistGraphDelta, registerGraphFunction } from "../src/functions/graph.js";
+import { KV } from "../src/state/schema.js";
+import type { GraphNode, GraphEdge, GraphSnapshot } from "../src/types.js";
+
+function clone<T>(value: T): T {
+ return value === null || value === undefined
+ ? value
+ : (JSON.parse(JSON.stringify(value)) as T);
+}
+
+function mockKV() {
+ const store = new Map<string, Map<string, unknown>>();
+ let gate: {
+ scope: string;
+ key: string;
+ release: () => void;
+ promise: Promise<void>;
+ } | null = null;
+ let listGate: { scope: string; release: () => void; promise: Promise<void> } | null = null;
+
+ return {
+ store,
+ armListGate: (scope: string) => {
+ let release!: () => void;
+ const promise = new Promise<void>((res) => {
+ release = res;
+ });
+ listGate = { scope, release, promise };
+ return release;
+ },
+ armGetGate: (scope: string, key: string) => {
+ let release!: () => void;
+ const promise = new Promise<void>((res) => {
+ release = res;
+ });
+ gate = { scope, key, release, promise };
+ return release;
+ },
+ get: async <T>(scope: string, key: string): Promise<T | null> => {
+ const value = clone((store.get(scope)?.get(key) as T) ?? null);
+ if (gate && gate.scope === scope && gate.key === key) {
+ const g = gate;
+ gate = null;
+ await g.promise;
+ }
+ return value;
+ },
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ const stored = clone(data);
+ if (!store.has(scope)) store.set(scope, new Map());
+ store.get(scope)!.set(key, stored);
+ return clone(stored);
+ },
+ delete: async (scope: string, key: string) => {
+ store.get(scope)?.delete(key);
+ },
+ list: async <T>(scope: string): Promise<T[]> => {
+ const m = store.get(scope);
+ const values = m ? clone(Array.from(m.values()) as T[]) : [];
+ if (listGate && listGate.scope === scope) {
+ const g = listGate;
+ listGate = null;
+ await g.promise;
+ }
+ return values;
+ },
+ };
+}
+
+function node(overrides: Partial<GraphNode> & { id: string; name: string }): GraphNode {
+ return {
+ type: "file",
+ properties: {},
+ sourceObservationIds: [],
+ createdAt: new Date().toISOString(),
+ ...overrides,
+ };
+}
+
+async function flush() {
+ await new Promise((r) => setTimeout(r, 0));
+}
+
+describe("persistGraphDelta concurrent merges (finding 7)", () => {
+ it("does not create duplicate nodes when two concurrent extractions resolve the same (type, name) key", async () => {
+ const kv = mockKV();
+
+ const nameIndexKey = "file|src/index.ts";
+ const releaseNameIndexRead = kv.armGetGate(KV.graphNameIndex, nameIndexKey);
+
+ const nodeA = node({
+ id: "node_a",
+ name: "src/index.ts",
+ sourceObservationIds: ["obs_a"],
+ });
+ const promiseA = persistGraphDelta(kv as never, [nodeA], [], ["obs_a"]);
+
+ await flush();
+
+ const nodeB = node({
+ id: "node_b",
+ name: "src/index.ts",
+ sourceObservationIds: ["obs_b"],
+ });
+ const promiseB = persistGraphDelta(kv as never, [nodeB], [], ["obs_b"]);
+
+ await flush();
+ releaseNameIndexRead();
+
+ await Promise.all([promiseA, promiseB]);
+
+ const allNodes = await kv.list<GraphNode>(KV.graphNodes);
+ expect(allNodes.length).toBe(1);
+
+ const merged = allNodes[0];
+ expect(merged.sourceObservationIds.sort()).toEqual(["obs_a", "obs_b"]);
+
+ const snapshot = (await kv.get<GraphSnapshot>(
+ KV.graphSnapshot,
+ "current",
+ )) as GraphSnapshot;
+ expect(snapshot.stats.totalNodes).toBe(1);
+ });
+
+ it("does not lose either batch's obsIds when merging into an existing node concurrently", async () => {
+ const kv = mockKV();
+
+ const existing: GraphNode = node({
+ id: "node_existing",
+ name: "src/utils.ts",
+ sourceObservationIds: ["obs_seed"],
+ });
+ await kv.set(KV.graphNodes, "node_existing", existing);
+ await kv.set(KV.graphNameIndex, "file|src/utils.ts", "node_existing");
+ await kv.set(KV.graphNodeDegree, "node_existing", 0);
+ await kv.set(KV.graphSnapshot, "current", {
+ version: 1,
+ topNodes: [existing],
+ topEdges: [],
+ topDegrees: {},
+ stats: {
+ totalNodes: 1,
+ totalEdges: 0,
+ nodesByType: { file: 1 },
+ edgesByType: {},
+ },
+ updatedAt: new Date().toISOString(),
+ dirty: false,
+ } satisfies GraphSnapshot);
+
+ const releaseExistingNodeRead = kv.armGetGate(KV.graphNodes, "node_existing");
+
+ const nodeA = node({
+ id: "node_a2",
+ name: "src/utils.ts",
+ sourceObservationIds: ["obs_x"],
+ });
+ const promiseA = persistGraphDelta(kv as never, [nodeA], [], ["obs_x"]);
+
+ await flush();
+
+ const nodeB = node({
+ id: "node_b2",
+ name: "src/utils.ts",
+ sourceObservationIds: ["obs_y"],
+ });
+ const promiseB = persistGraphDelta(kv as never, [nodeB], [], ["obs_y"]);
+
+ await flush();
+ releaseExistingNodeRead();
+
+ await Promise.all([promiseA, promiseB]);
+
+ const allNodes = await kv.list<GraphNode>(KV.graphNodes);
+ expect(allNodes.length).toBe(1);
+ expect(allNodes[0].sourceObservationIds.sort()).toEqual([
+ "obs_seed",
+ "obs_x",
+ "obs_y",
+ ]);
+ });
+
+ it("snapshot rebuild does not overwrite a delta that commits while it enumerates", async () => {
+ const kv = mockKV();
+ const handlers = new Map<string, (data?: unknown) => Promise<unknown>>();
+ const sdk = {
+ registerFunction: (id: string, fn: (data?: unknown) => Promise<unknown>) => {
+ handlers.set(id, fn);
+ },
+ };
+ registerGraphFunction(sdk as never, kv as never, { name: "noop" } as never);
+
+ const seeded = node({ id: "node_seed", name: "src/seed.ts" });
+ await persistGraphDelta(kv as never, [seeded], [], ["obs_seed"]);
+
+ const releaseNodeList = kv.armListGate(KV.graphNodes);
+ const rebuild = handlers.get("mem::graph-snapshot-rebuild")!();
+ await flush();
+
+ const fresh = node({ id: "node_fresh", name: "src/fresh.ts" });
+ const delta = persistGraphDelta(kv as never, [fresh], [], ["obs_fresh"]);
+ await flush();
+ releaseNodeList();
+
+ const rebuilt = (await rebuild) as { success: boolean };
+ await delta;
+
+ expect(rebuilt.success).toBe(true);
+ const snapshot = (await kv.get<GraphSnapshot>(
+ KV.graphSnapshot,
+ "current",
+ )) as GraphSnapshot;
+ expect(snapshot.stats.totalNodes).toBe(2);
+ expect(snapshot.topNodes.map((n) => n.id).sort()).toEqual([
+ "node_fresh",
+ "node_seed",
+ ]);
+ });
+});
diff --git a/test/session-kv-races.test.ts b/test/session-kv-races.test.ts
new file mode 100644
index 00000000..40c2b0f1
--- /dev/null
+++ b/test/session-kv-races.test.ts
@@ -0,0 +1,295 @@
+import { describe, it, expect, vi } from "vitest";
+
+vi.mock("../src/logger.js", () => ({
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+}));
+
+import { registerObserveFunction } from "../src/functions/observe.js";
+import { registerApiTriggers } from "../src/triggers/api.js";
+import { registerEventTriggers } from "../src/triggers/events.js";
+import { KV } from "../src/state/schema.js";
+import type { Session } from "../src/types.js";
+
+type Gate = { scope: string; key: string; phase: "get" | "set" };
+
+function clone<T>(value: T): T {
+ return value === null || value === undefined
+ ? value
+ : (JSON.parse(JSON.stringify(value)) as T);
+}
+
+function mockKV() {
+ const store = new Map<string, Map<string, unknown>>();
+ let gate: (Gate & { release: () => void; promise: Promise<void> }) | null = null;
+
+ function arm(scope: string, key: string, phase: "get" | "set"): () => void {
+ let release!: () => void;
+ const promise = new Promise<void>((res) => {
+ release = res;
+ });
+ gate = { scope, key, phase, release, promise };
+ return release;
+ }
+
+ async function maybeWait(scope: string, key: string, phase: "get" | "set") {
+ if (gate && gate.scope === scope && gate.key === key && gate.phase === phase) {
+ const g = gate;
+ gate = null;
+ await g.promise;
+ }
+ }
+
+ return {
+ store,
+ armGetGate: (scope: string, key: string) => arm(scope, key, "get"),
+ armSetGate: (scope: string, key: string) => arm(scope, key, "set"),
+ get: async <T>(scope: string, key: string): Promise<T | null> => {
+ const value = clone((store.get(scope)?.get(key) as T) ?? null);
+ await maybeWait(scope, key, "get");
+ return value;
+ },
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ const stored = clone(data);
+ await maybeWait(scope, key, "set");
+ if (!store.has(scope)) store.set(scope, new Map());
+ store.get(scope)!.set(key, stored);
+ return clone(stored);
+ },
+ update: async (
+ scope: string,
+ key: string,
+ ops: Array<{ type: string; path: string; value?: unknown }>,
+ ) => {
+ if (!store.has(scope)) store.set(scope, new Map());
+ const m = store.get(scope)!;
+ const current = clone((m.get(key) as Record<string, unknown>) ?? {});
+ for (const op of ops) {
+ if (op.type === "set") current[op.path] = op.value;
+ }
+ m.set(key, current);
+ return clone(current);
+ },
+ delete: async (scope: string, key: string) => {
+ store.get(scope)?.delete(key);
+ },
+ list: async <T>(scope: string): Promise<T[]> => {
+ const m = store.get(scope);
+ return m ? clone(Array.from(m.values()) as T[]) : [];
+ },
+ };
+}
+
+function mockSdk() {
+ const fns = new Map<string, Function>();
+ return {
+ fns,
+ registerFunction: (idOrOpts: string | { id: string }, fn: Function) => {
+ const id = typeof idOrOpts === "string" ? idOrOpts : idOrOpts.id;
+ fns.set(id, fn);
+ },
+ registerTrigger: () => {},
+ trigger: async (
+ idOrInput: string | { function_id: string; payload: unknown; action?: unknown },
+ data?: unknown,
+ ) => {
+ const id = typeof idOrInput === "string" ? idOrInput : idOrInput.function_id;
+ const payload = typeof idOrInput === "string" ? data : idOrInput.payload;
+ const fn = fns.get(id);
+ if (fn) return fn(payload);
+ return null;
+ },
+ };
+}
+
+async function flush() {
+ await new Promise((r) => setTimeout(r, 0));
+}
+
+describe("KV.sessions concurrent mutation races (finding 3)", () => {
+ it("api::session::commit does not revert a concurrent mem::observe's observationCount increment", async () => {
+ const kv = mockKV();
+ const sdk = mockSdk();
+ registerObserveFunction(sdk as never, kv as never);
+ registerApiTriggers(sdk as never, kv as never);
+
+ const session: Session = {
+ id: "ses_race1",
+ project: "/repo",
+ cwd: "/repo",
+ startedAt: new Date().toISOString(),
+ status: "active",
+ observationCount: 0,
+ };
+ await kv.set(KV.sessions, "ses_race1", session);
+
+ const releaseCommitRead = kv.armGetGate(KV.sessions, "ses_race1");
+
+ const commitPromise = sdk.trigger("api::session::commit", {
+ body: { sha: "abc123def", sessionId: "ses_race1" },
+ });
+
+ await flush();
+
+ const observePromise = sdk.trigger("mem::observe", {
+ sessionId: "ses_race1",
+ project: "/repo",
+ cwd: "/repo",
+ hookType: "prompt_submit",
+ timestamp: new Date().toISOString(),
+ data: { prompt: "do the thing" },
+ });
+
+ await flush();
+ releaseCommitRead();
+
+ await Promise.all([commitPromise, observePromise]);
+
+ const finalSession = kv.store.get(KV.sessions)?.get("ses_race1") as Session;
+ expect(finalSession.observationCount).toBe(1);
+ expect(finalSession.commitShas).toEqual(["abc123def"]);
+ });
+
+ it("mem::observe's implicit session create does not clobber a concurrent api::session::start's title", async () => {
+ const kv = mockKV();
+ const sdk = mockSdk();
+ registerObserveFunction(sdk as never, kv as never);
+ registerApiTriggers(sdk as never, kv as never);
+ sdk.fns.set("mem::context", async () => ({ context: "" }));
+
+ const releaseStartWrite = kv.armSetGate(KV.sessions, "ses_race2");
+
+ const startPromise = sdk.trigger("api::session::start", {
+ body: {
+ sessionId: "ses_race2",
+ project: "/repo",
+ cwd: "/repo",
+ title: "fix the parser",
+ },
+ });
+
+ await flush();
+
+ const observePromise = sdk.trigger("mem::observe", {
+ sessionId: "ses_race2",
+ project: "/repo",
+ cwd: "/repo",
+ hookType: "prompt_submit",
+ timestamp: new Date().toISOString(),
+ data: { prompt: "fix the parser" },
+ });
+
+ await flush();
+ releaseStartWrite();
+
+ await Promise.all([startPromise, observePromise]);
+
+ const finalSession = kv.store.get(KV.sessions)?.get("ses_race2") as Session;
+ expect(finalSession.summary).toBe("fix the parser");
+ expect(finalSession.observationCount).toBe(1);
+ });
+
+ it("event::session::started racing mem::observe's implicit create does not revert the observation count to zero", async () => {
+ const kv = mockKV();
+ const sdk = mockSdk();
+ registerEventTriggers(sdk as never, kv as never);
+ registerObserveFunction(sdk as never, kv as never);
+ sdk.fns.set("mem::context", async () => ({ context: "" }));
+
+ const releaseEventWrite = kv.armSetGate(KV.sessions, "ses_race3");
+
+ const eventPromise = sdk.trigger("event::session::started", {
+ sessionId: "ses_race3",
+ project: "/repo",
+ cwd: "/repo",
+ });
+
+ await flush();
+
+ const observePromise = sdk.trigger("mem::observe", {
+ sessionId: "ses_race3",
+ project: "/repo",
+ cwd: "/repo",
+ hookType: "prompt_submit",
+ timestamp: new Date().toISOString(),
+ data: { prompt: "start work" },
+ });
+
+ await flush();
+ releaseEventWrite();
+
+ await Promise.all([eventPromise, observePromise]);
+
+ const finalSession = kv.store.get(KV.sessions)?.get("ses_race3") as Session;
+ expect(finalSession).toBeTruthy();
+ expect(finalSession.observationCount).toBe(1);
+ });
+
+ it("api::session::commit does not revert a concurrent api::session::end status update", async () => {
+ const kv = mockKV();
+ const sdk = mockSdk();
+ registerApiTriggers(sdk as never, kv as never);
+
+ const session: Session = {
+ id: "ses_race5",
+ project: "/repo",
+ cwd: "/repo",
+ startedAt: new Date().toISOString(),
+ status: "active",
+ observationCount: 3,
+ };
+ await kv.set(KV.sessions, "ses_race5", session);
+
+ const releaseCommitRead = kv.armGetGate(KV.sessions, "ses_race5");
+
+ const commitPromise = sdk.trigger("api::session::commit", {
+ body: { sha: "deadbeef01", sessionId: "ses_race5" },
+ });
+
+ await flush();
+
+ const endPromise = sdk.trigger("api::session::end", {
+ body: { sessionId: "ses_race5" },
+ });
+
+ await flush();
+ releaseCommitRead();
+
+ await Promise.all([commitPromise, endPromise]);
+
+ const finalSession = kv.store.get(KV.sessions)?.get("ses_race5") as Session;
+ expect(finalSession.commitShas).toEqual(["deadbeef01"]);
+ expect(finalSession.status).toBe("completed");
+ expect(finalSession.endedAt).toBeTruthy();
+ });
+
+ it("api::session::start does not reset an observation count or firstPrompt already created by mem::observe", async () => {
+ const kv = mockKV();
+ const sdk = mockSdk();
+ registerObserveFunction(sdk as never, kv as never);
+ registerApiTriggers(sdk as never, kv as never);
+ sdk.fns.set("mem::context", async () => ({ context: "" }));
+
+ await sdk.trigger("mem::observe", {
+ sessionId: "ses_race4",
+ project: "/repo",
+ cwd: "/repo",
+ hookType: "prompt_submit",
+ timestamp: new Date().toISOString(),
+ data: { prompt: "do the thing" },
+ });
+
+ await sdk.trigger("api::session::start", {
+ body: {
+ sessionId: "ses_race4",
+ project: "/repo",
+ cwd: "/repo",
+ title: "fix the parser",
+ },
+ });
+
+ const finalSession = kv.store.get(KV.sessions)?.get("ses_race4") as Session;
+ expect(finalSession.observationCount).toBe(1);
+ expect(finalSession.firstPrompt).toBe("fix the parser");
+ expect(finalSession.status).toBe("active");
+ });
+});