#1 Persistent memory for AI coding agents based on real-world benchmarks
TypeScript agentmemoryagentsaiclaudeclaudecodecodexcopilotcursorgenaiharnesshermesmemoryompopenclawopencodepi
fix(state): lock session and graph read-modify-write paths (#1421)
Rohit Ghumare2493bc56850a
diff --git a/src/functions/cascade.ts b/src/functions/cascade.tsindex 2167c506..b2959676 100644--- a/src/functions/cascade.ts+++ b/src/functions/cascade.ts@@ -1,6 +1,7 @@import type { IIIClient } from "iii-sdk";import type { StateKV } from "../state/kv.js";import { KV } from "../state/schema.js";+import { withKeyedLock } from "../state/keyed-mutex.js";import type { Memory, GraphNode, GraphEdge } from "../types.js";import { recordAudit } from "./audit.js";@@ -28,10 +29,14 @@ export function registerCascadeFunction(sdk: IIIClient, kv: StateKV): void {for (const node of nodes) {if (node.stale) continue;const overlap = (node.sourceObservationIds ?? []).some((id) => obsIds.has(id));- if (overlap) {- node.stale = true;- node.updatedAt = now;- await kv.set(KV.graphNodes, node.id, node);+ if (!overlap) continue;+ const flagged = await withKeyedLock("graph:persist", async () => {+ const latest = (await kv.get<GraphNode>(KV.graphNodes, node.id)) ?? node;+ if (latest.stale) return false;+ await kv.set(KV.graphNodes, node.id, { ...latest, stale: true, updatedAt: now });+ return true;+ });+ if (flagged) {await recordAudit(kv, "consolidate", "mem::cascade-update", [node.id], {resourceType: "GraphNode",change: "marked stale from superseded memory",@@ -45,9 +50,14 @@ export function registerCascadeFunction(sdk: IIIClient, kv: StateKV): void {for (const edge of edges) {if (edge.stale) continue;const overlap = (edge.sourceObservationIds ?? []).some((id) => obsIds.has(id));- if (overlap) {- edge.stale = true;- await kv.set(KV.graphEdges, edge.id, edge);+ if (!overlap) continue;+ const flagged = await withKeyedLock("graph:persist", async () => {+ const latest = (await kv.get<GraphEdge>(KV.graphEdges, edge.id)) ?? edge;+ if (latest.stale) return false;+ await kv.set(KV.graphEdges, edge.id, { ...latest, stale: true });+ return true;+ });+ if (flagged) {await recordAudit(kv, "consolidate", "mem::cascade-update", [edge.id], {resourceType: "GraphEdge",change: "marked stale from superseded memory",diff --git a/src/functions/export-import.ts b/src/functions/export-import.tsindex 6d0128f3..747cdd53 100644--- a/src/functions/export-import.ts+++ b/src/functions/export-import.ts@@ -29,6 +29,7 @@ import { normalizeAccessLog } from "./access-tracker.js";import { KV } from "../state/schema.js";import { checkPayloadFrameSize } from "../state/frame-guard.js";import { StateKV } from "../state/kv.js";+import { withKeyedLock } from "../state/keyed-mutex.js";import { VERSION } from "../version.js";import { recordAudit } from "./audit.js";import { indexRecords } from "./search.js";@@ -372,13 +373,17 @@ export function registerExportImportFunction(sdk: IIIClient, kv: StateKV): voidawait kv.list<Insight>(KV.insights).catch(() => []),(i) => kv.delete(KV.insights, i.id),);- await runChunked(- await kv.list<{ id: string }>(KV.graphNodes).catch(() => []),- (n) => kv.delete(KV.graphNodes, n.id),+ await withKeyedLock("graph:persist", async () =>+ runChunked(+ await kv.list<{ id: string }>(KV.graphNodes).catch(() => []),+ (n) => kv.delete(KV.graphNodes, n.id),+ ),);- await runChunked(- await kv.list<{ id: string }>(KV.graphEdges).catch(() => []),- (e) => kv.delete(KV.graphEdges, e.id),+ await withKeyedLock("graph:persist", async () =>+ runChunked(+ await kv.list<{ id: string }>(KV.graphEdges).catch(() => []),+ (e) => kv.delete(KV.graphEdges, e.id),+ ),);await runChunked(await kv.list<{ id: string }>(KV.semantic).catch(() => []),@@ -407,17 +412,19 @@ export function registerExportImportFunction(sdk: IIIClient, kv: StateKV): voidconst indexMems: Memory[] = [];await runChunked(importData.sessions, async (session) => {- if (strategy === "skip") {- const existing = await kv- .get<Session>(KV.sessions, session.id)- .catch(() => null);- if (existing) {- stats.skipped++;- return;+ await withKeyedLock(`obs:${session.id}`, async () => {+ if (strategy === "skip") {+ const existing = await kv+ .get<Session>(KV.sessions, session.id)+ .catch(() => null);+ if (existing) {+ stats.skipped++;+ return;+ }}- }- await kv.set(KV.sessions, session.id, session);- stats.sessions++;+ await kv.set(KV.sessions, session.id, session);+ stats.sessions++;+ });});for (const [sessionId, obs] of Object.entries(importData.observations)) {@@ -473,22 +480,26 @@ export function registerExportImportFunction(sdk: IIIClient, kv: StateKV): void});if (importData.graphNodes) {- await runChunked(importData.graphNodes, async (node) => {- if (strategy === "skip") {- const existing = await kv.get(KV.graphNodes, node.id).catch(() => null);- if (existing) { stats.skipped++; return; }- }- await kv.set(KV.graphNodes, node.id, node);- });+ await withKeyedLock("graph:persist", () =>+ runChunked(importData.graphNodes!, async (node) => {+ if (strategy === "skip") {+ const existing = await kv.get(KV.graphNodes, node.id).catch(() => null);+ if (existing) { stats.skipped++; return; }+ }+ await kv.set(KV.graphNodes, node.id, node);+ }),+ );}if (importData.graphEdges) {- await runChunked(importData.graphEdges, async (edge) => {- if (strategy === "skip") {- const existing = await kv.get(KV.graphEdges, edge.id).catch(() => null);- if (existing) { stats.skipped++; return; }- }- await kv.set(KV.graphEdges, edge.id, edge);- });+ await withKeyedLock("graph:persist", () =>+ runChunked(importData.graphEdges!, async (edge) => {+ if (strategy === "skip") {+ const existing = await kv.get(KV.graphEdges, edge.id).catch(() => null);+ if (existing) { stats.skipped++; return; }+ }+ await kv.set(KV.graphEdges, edge.id, edge);+ }),+ );}if (importData.semanticMemories) {await runChunked(importData.semanticMemories, async (sem) => {diff --git a/src/functions/graph.ts b/src/functions/graph.tsindex 62e7343e..08188731 100644--- a/src/functions/graph.ts+++ b/src/functions/graph.ts@@ -9,6 +9,7 @@ import type {} from "../types.js";import { KV, generateId } from "../state/schema.js";import type { StateKV } from "../state/kv.js";+import { withKeyedLock } from "../state/keyed-mutex.js";import {GRAPH_EXTRACTION_SYSTEM,buildGraphExtractionPrompt,@@ -554,129 +555,131 @@ export async function persistGraphDelta(edges: GraphEdge[],obsIds: string[],): Promise<{ newNodeCount: number; newEdgeCount: number }> {- const snap = (await readSnapshot(kv)) ?? emptySnapshot();- const capturedAt = new Date().toISOString();- let newNodeCount = 0;- let newEdgeCount = 0;- // Merge-only batches mutate cached topNodes/topEdges entries without- // changing the counts; track that separately so the snapshot still persists.- let snapMutated = false;- const newEdgesForTopCheck: GraphEdge[] = [];- // When a freshly-minted node merges into an existing row via the name- // index, edges in the same batch still reference the fresh id. Remap edge- // endpoints to the persisted ids so edges never dangle and re-runs hit the- // same edge-index key instead of duplicating.- const idRemap = new Map<string, string>();-- for (const node of nodes) {- const indexKey = nameIndexKey(node.type, node.name);- const existingId = await kv.get<string>(KV.graphNameIndex, indexKey);-- let existing: GraphNode | null = null;- if (existingId) {- existing = await kv.get<GraphNode>(KV.graphNodes, existingId);- // #825 follow-up: name-index lookups can resolve into- // pre-reset rows. Drop them so extract writes a fresh- // node + index entry instead of silently reconnecting- // to a legacy orphan (which would keep the snapshot at- // 0 forever after a reset).- if (- existing &&- snap.resetAt &&- typeof existing.createdAt === "string" &&- existing.createdAt < snap.resetAt- ) {- existing = null;+ return withKeyedLock("graph:persist", async () => {+ const snap = (await readSnapshot(kv)) ?? emptySnapshot();+ const capturedAt = new Date().toISOString();+ let newNodeCount = 0;+ let newEdgeCount = 0;+ // Merge-only batches mutate cached topNodes/topEdges entries without+ // changing the counts; track that separately so the snapshot still persists.+ let snapMutated = false;+ const newEdgesForTopCheck: GraphEdge[] = [];+ // When a freshly-minted node merges into an existing row via the name+ // index, edges in the same batch still reference the fresh id. Remap edge+ // endpoints to the persisted ids so edges never dangle and re-runs hit the+ // same edge-index key instead of duplicating.+ const idRemap = new Map<string, string>();++ for (const node of nodes) {+ const indexKey = nameIndexKey(node.type, node.name);+ const existingId = await kv.get<string>(KV.graphNameIndex, indexKey);++ let existing: GraphNode | null = null;+ if (existingId) {+ existing = await kv.get<GraphNode>(KV.graphNodes, existingId);+ // #825 follow-up: name-index lookups can resolve into+ // pre-reset rows. Drop them so extract writes a fresh+ // node + index entry instead of silently reconnecting+ // to a legacy orphan (which would keep the snapshot at+ // 0 forever after a reset).+ if (+ existing &&+ snap.resetAt &&+ typeof existing.createdAt === "string" &&+ existing.createdAt < snap.resetAt+ ) {+ existing = null;+ }}- }- if (existing) {- idRemap.set(node.id, existing.id);- const merged = mergeNode(existing, node, obsIds, capturedAt);- await kv.set(KV.graphNodes, existing.id, merged);- // Update topNodes entry if present so a stale clone isn't- // returned from the snapshot fast path.- const topIdx = snap.topNodes.findIndex((n) => n.id === existing!.id);- if (topIdx !== -1) {- snap.topNodes[topIdx] = merged;- snapMutated = true;- }- } else {- await kv.set(KV.graphNodes, node.id, node);- await kv.set(KV.graphNameIndex, indexKey, node.id);- await kv.set(KV.graphNodeDegree, node.id, 0);- snap.stats.totalNodes += 1;- snap.stats.nodesByType[node.type] =- (snap.stats.nodesByType[node.type] ?? 0) + 1;- newNodeCount += 1;- if (snap.topNodes.length < SNAPSHOT_TOP_NODES) {- // Degree 0 still beats an empty slot — sit at the tail- // until edges arrive and promote.- snap.topNodes.push(node);- snap.topDegrees[node.id] = 0;+ if (existing) {+ idRemap.set(node.id, existing.id);+ const merged = mergeNode(existing, node, obsIds, capturedAt);+ await kv.set(KV.graphNodes, existing.id, merged);+ // Update topNodes entry if present so a stale clone isn't+ // returned from the snapshot fast path.+ const topIdx = snap.topNodes.findIndex((n) => n.id === existing!.id);+ if (topIdx !== -1) {+ snap.topNodes[topIdx] = merged;+ snapMutated = true;+ }+ } else {+ await kv.set(KV.graphNodes, node.id, node);+ await kv.set(KV.graphNameIndex, indexKey, node.id);+ await kv.set(KV.graphNodeDegree, node.id, 0);+ snap.stats.totalNodes += 1;+ snap.stats.nodesByType[node.type] =+ (snap.stats.nodesByType[node.type] ?? 0) + 1;+ newNodeCount += 1;+ if (snap.topNodes.length < SNAPSHOT_TOP_NODES) {+ // Degree 0 still beats an empty slot — sit at the tail+ // until edges arrive and promote.+ snap.topNodes.push(node);+ snap.topDegrees[node.id] = 0;+ }}}- }- for (const rawEdge of edges) {- const edge: GraphEdge = {- ...rawEdge,- sourceNodeId: idRemap.get(rawEdge.sourceNodeId) ?? rawEdge.sourceNodeId,- targetNodeId: idRemap.get(rawEdge.targetNodeId) ?? rawEdge.targetNodeId,- };- const eKey = edgeIndexKey(edge.sourceNodeId, edge.targetNodeId, edge.type);- const existingId = await kv.get<string>(KV.graphEdgeKey, eKey);-- let existing: GraphEdge | null = null;- if (existingId) {- existing = await kv.get<GraphEdge>(KV.graphEdges, existingId);- // Same #825 orphan check as the node path above.- if (- existing &&- snap.resetAt &&- typeof existing.createdAt === "string" &&- existing.createdAt < snap.resetAt- ) {- existing = null;+ for (const rawEdge of edges) {+ const edge: GraphEdge = {+ ...rawEdge,+ sourceNodeId: idRemap.get(rawEdge.sourceNodeId) ?? rawEdge.sourceNodeId,+ targetNodeId: idRemap.get(rawEdge.targetNodeId) ?? rawEdge.targetNodeId,+ };+ const eKey = edgeIndexKey(edge.sourceNodeId, edge.targetNodeId, edge.type);+ const existingId = await kv.get<string>(KV.graphEdgeKey, eKey);++ let existing: GraphEdge | null = null;+ if (existingId) {+ existing = await kv.get<GraphEdge>(KV.graphEdges, existingId);+ // Same #825 orphan check as the node path above.+ if (+ existing &&+ snap.resetAt &&+ typeof existing.createdAt === "string" &&+ existing.createdAt < snap.resetAt+ ) {+ existing = null;+ }}- }- if (existing) {- const merged = mergeEdge(existing, obsIds);- await kv.set(KV.graphEdges, existing.id, merged);- // Replace cached topEdges entry too if present.- const topIdx = snap.topEdges.findIndex((e) => e.id === existing!.id);- if (topIdx !== -1) {- snap.topEdges[topIdx] = merged;- snapMutated = true;+ if (existing) {+ const merged = mergeEdge(existing, obsIds);+ await kv.set(KV.graphEdges, existing.id, merged);+ // Replace cached topEdges entry too if present.+ const topIdx = snap.topEdges.findIndex((e) => e.id === existing!.id);+ if (topIdx !== -1) {+ snap.topEdges[topIdx] = merged;+ snapMutated = true;+ }+ } else {+ await kv.set(KV.graphEdges, edge.id, edge);+ await kv.set(KV.graphEdgeKey, eKey, edge.id);+ snap.stats.totalEdges += 1;+ snap.stats.edgesByType[edge.type] =+ (snap.stats.edgesByType[edge.type] ?? 0) + 1;+ newEdgeCount += 1;+ await applyDegreeDelta(kv, snap, edge.sourceNodeId, +1);+ await applyDegreeDelta(kv, snap, edge.targetNodeId, +1);+ newEdgesForTopCheck.push(edge);}- } else {- await kv.set(KV.graphEdges, edge.id, edge);- await kv.set(KV.graphEdgeKey, eKey, edge.id);- snap.stats.totalEdges += 1;- snap.stats.edgesByType[edge.type] =- (snap.stats.edgesByType[edge.type] ?? 0) + 1;- newEdgeCount += 1;- await applyDegreeDelta(kv, snap, edge.sourceNodeId, +1);- await applyDegreeDelta(kv, snap, edge.targetNodeId, +1);- newEdgesForTopCheck.push(edge);}- }- // Push newly-added edges into snapshot.topEdges if both- // endpoints are in the top-N (post-degree-delta). Done after- // all degree updates so the topIds set is stable.- for (const edge of newEdgesForTopCheck) {- snapshotPushEdgeIfBothInTop(snap, edge);- }+ // Push newly-added edges into snapshot.topEdges if both+ // endpoints are in the top-N (post-degree-delta). Done after+ // all degree updates so the topIds set is stable.+ for (const edge of newEdgesForTopCheck) {+ snapshotPushEdgeIfBothInTop(snap, edge);+ }- if (newNodeCount > 0 || newEdgeCount > 0 || snapMutated) {- snap.updatedAt = capturedAt;- snap.dirty = false;- await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, snap);- }+ if (newNodeCount > 0 || newEdgeCount > 0 || snapMutated) {+ snap.updatedAt = capturedAt;+ snap.dirty = false;+ await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, snap);+ }- return { newNodeCount, newEdgeCount };+ return { newNodeCount, newEdgeCount };+ });}export function registerGraphFunction(@@ -1014,70 +1017,74 @@ export function registerGraphFunction(}try {- const [nodes, edges] = await withTimeout(- Promise.all([- kv.list<GraphNode>(KV.graphNodes),- kv.list<GraphEdge>(KV.graphEdges),- ]),- LIVE_ENUMERATION_BUDGET_MS,- "graph-snapshot-rebuild enumeration",- );+ const outcome = await withKeyedLock("graph:persist", async () => {+ const [nodes, edges] = await withTimeout(+ Promise.all([+ kv.list<GraphNode>(KV.graphNodes),+ kv.list<GraphEdge>(KV.graphEdges),+ ]),+ LIVE_ENUMERATION_BUDGET_MS,+ "graph-snapshot-rebuild enumeration",+ );++ if (nodes.length > REBUILD_SAFE_NODE_CEILING) {+ return { tooLargeNodeCount: nodes.length };+ }++ const liveNodes = nodes.filter((n) => !n.stale);+ const liveEdges = edges.filter((e) => !e.stale);+ const degree = new Map<string, number>();+ for (const e of liveEdges) {+ degree.set(e.sourceNodeId, (degree.get(e.sourceNodeId) ?? 0) + 1);+ degree.set(e.targetNodeId, (degree.get(e.targetNodeId) ?? 0) + 1);+ }+ const BATCH_SIZE = 100;+ for (let i = 0; i < liveNodes.length; i += BATCH_SIZE) {+ const batch = liveNodes.slice(i, i + BATCH_SIZE);+ await Promise.all(+ batch.flatMap((n) => [+ kv.set(KV.graphNameIndex, nameIndexKey(n.type, n.name), n.id),+ kv.set(KV.graphNodeDegree, n.id, degree.get(n.id) ?? 0),+ ]),+ );+ }+ for (let i = 0; i < liveEdges.length; i += BATCH_SIZE) {+ const batch = liveEdges.slice(i, i + BATCH_SIZE);+ await Promise.all(+ batch.map((e) =>+ kv.set(+ KV.graphEdgeKey,+ edgeIndexKey(e.sourceNodeId, e.targetNodeId, e.type),+ e.id,+ ),+ ),+ );+ }- if (nodes.length > REBUILD_SAFE_NODE_CEILING) {+ const rebuilt = buildSnapshotFromArrays(nodes, edges);+ await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, rebuilt);+ return { snap: rebuilt };+ });++ if ("tooLargeNodeCount" in outcome) {+ const totalNodes = outcome.tooLargeNodeCount;logger.warn("Graph snapshot rebuild aborted: corpus too large", {- totalNodes: nodes.length,+ totalNodes,ceiling: REBUILD_SAFE_NODE_CEILING,});return {success: false,tooLarge: true,- totalNodes: nodes.length,+ totalNodes,ceiling: REBUILD_SAFE_NODE_CEILING,error:- `Corpus has ${nodes.length} graph nodes; safe-rebuild ceiling ` ++ `Corpus has ${totalNodes} graph nodes; safe-rebuild ceiling ` +`is ${REBUILD_SAFE_NODE_CEILING}. Run POST /agentmemory/graph/reset ` +`to wipe and let future extracts rebuild incrementally.`,};}- // Backfill the targeted-lookup indexes so post-rebuild- // graph-extract calls hit the O(1) path instead of falling- // through to the (already-removed) full-scope scan. Batch- // writes via Promise.all to avoid N sequential round-trips —- // BATCH_SIZE bounds in-flight writes so we don't open thousands- // of concurrent state channels on huge corpora.- const liveNodes = nodes.filter((n) => !n.stale);- const liveEdges = edges.filter((e) => !e.stale);- const degree = new Map<string, number>();- for (const e of liveEdges) {- degree.set(e.sourceNodeId, (degree.get(e.sourceNodeId) ?? 0) + 1);- degree.set(e.targetNodeId, (degree.get(e.targetNodeId) ?? 0) + 1);- }- const BATCH_SIZE = 100;- for (let i = 0; i < liveNodes.length; i += BATCH_SIZE) {- const batch = liveNodes.slice(i, i + BATCH_SIZE);- await Promise.all(- batch.flatMap((n) => [- kv.set(KV.graphNameIndex, nameIndexKey(n.type, n.name), n.id),- kv.set(KV.graphNodeDegree, n.id, degree.get(n.id) ?? 0),- ]),- );- }- for (let i = 0; i < liveEdges.length; i += BATCH_SIZE) {- const batch = liveEdges.slice(i, i + BATCH_SIZE);- await Promise.all(- batch.map((e) =>- kv.set(- KV.graphEdgeKey,- edgeIndexKey(e.sourceNodeId, e.targetNodeId, e.type),- e.id,- ),- ),- );- }-- const snap = buildSnapshotFromArrays(nodes, edges);- await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, snap);+ const { snap } = outcome;const tookMs = Date.now() - started;logger.info("Graph snapshot rebuilt", {totalNodes: snap.stats.totalNodes,@@ -1134,7 +1141,9 @@ export function registerGraphFunction(...emptySnapshot(),resetAt: new Date().toISOString(),};- await kv.set(KV.graphSnapshot, SNAPSHOT_KEY, resetSnapshot);+ await withKeyedLock("graph:persist", () =>+ kv.set(KV.graphSnapshot, SNAPSHOT_KEY, resetSnapshot),+ );const counts: Record<string, number> = {[KV.graphSnapshot]: 1,};diff --git a/src/functions/mesh.ts b/src/functions/mesh.tsindex 139765ad..0dd7e957 100644--- a/src/functions/mesh.ts+++ b/src/functions/mesh.ts@@ -119,7 +119,7 @@ async function lwwMergeGraphNodes(if (!item.id || typeof item.id !== "string") continue;const ts = graphNodeTs(item);if (!ts || Number.isNaN(new Date(ts).getTime())) continue;- const wrote = await withKeyedLock(`mem:gnode:${item.id}`, async () => {+ const wrote = await withKeyedLock("graph:persist", async () => {const existing = await kv.get<GraphNode>(KV.graphNodes, item.id);if (!existing) {await kv.set(KV.graphNodes, item.id, item);@@ -136,6 +136,33 @@ async function lwwMergeGraphNodes(return count;}+async function lwwMergeGraphEdges(+ kv: StateKV,+ items: GraphEdge[] | undefined,+): Promise<number> {+ if (!items || !Array.isArray(items)) return 0;+ let count = 0;+ for (const item of items) {+ if (!item.id || typeof item.id !== "string") continue;+ const ts = item.createdAt;+ if (!ts || Number.isNaN(new Date(ts).getTime())) continue;+ const wrote = await withKeyedLock("graph:persist", async () => {+ const existing = await kv.get<GraphEdge>(KV.graphEdges, item.id);+ if (!existing) {+ await kv.set(KV.graphEdges, item.id, item);+ return true;+ }+ if (new Date(ts) > new Date(existing.createdAt)) {+ await kv.set(KV.graphEdges, item.id, item);+ return true;+ }+ return false;+ });+ if (wrote) count++;+ }+ return count;+}+export function registerMeshFunction(sdk: IIIClient,kv: StateKV,@@ -361,7 +388,7 @@ export function registerMeshFunction(}}accepted += await lwwMergeGraphNodes(kv, data.graphNodes);- accepted += await lwwMergeList(kv, KV.graphEdges, data.graphEdges, "mem:gedge", "createdAt");+ accepted += await lwwMergeGraphEdges(kv, data.graphEdges);await recordAudit(kv, "mesh_sync", "mem::mesh-receive", [], {action: "mesh.receive",accepted,@@ -488,7 +515,7 @@ async function applySyncData(applied += await lwwMergeGraphNodes(kv, data.graphNodes);}if (scopes.includes("graph:edges")) {- applied += await lwwMergeList(kv, KV.graphEdges, data.graphEdges, "mem:gedge", "createdAt");+ applied += await lwwMergeGraphEdges(kv, data.graphEdges);}return applied;diff --git a/src/functions/migrate.ts b/src/functions/migrate.tsindex 8cabef53..350a5332 100644--- a/src/functions/migrate.ts+++ b/src/functions/migrate.ts@@ -3,6 +3,7 @@ import { resolve } from "node:path";import { homedir } from "node:os";import { KV, generateId } from "../state/schema.js";import { StateKV } from "../state/kv.js";+import { withKeyedLock } from "../state/keyed-mutex.js";import type {Memory,Session,@@ -150,7 +151,9 @@ export function registerMigrateFunction(sdk: IIIClient, kv: StateKV): void {status: "completed",observationCount: 0,};- await kv.set(KV.sessions, session.id, session);+ await withKeyedLock(`obs:${session.id}`, () =>+ kv.set(KV.sessions, session.id, session),+ );sessionCount++;}diff --git a/src/functions/replay.ts b/src/functions/replay.tsindex aea735ff..d158b913 100644--- a/src/functions/replay.ts+++ b/src/functions/replay.ts@@ -12,6 +12,7 @@ import type {import { importOrigin } from "../types.js";import type { StateKV } from "../state/kv.js";import { KV, generateId, fingerprintId } from "../state/schema.js";+import { withKeyedLock } from "../state/keyed-mutex.js";import { parseJsonlText } from "../replay/jsonl-parser.js";import { resetLessonIndex } from "./lessons.js";import { projectTimeline, type Timeline } from "../replay/timeline.js";@@ -394,46 +395,48 @@ export function registerReplayFunctions(sdk: IIIClient, kv: StateKV): void {? firstPromptObs.userPrompt.replace(/\s+/g, " ").trim().slice(0, 200): undefined;- const existing = await kv.get<Session>(KV.sessions, parsed.sessionId);- if (existing) {- existing.observationCount =- (existing.observationCount || 0) + parsed.observations.length;- if (parsed.endedAt > (existing.endedAt || "")) {- existing.endedAt = parsed.endedAt;+ await withKeyedLock(`obs:${parsed.sessionId}`, async () => {+ const existing = await kv.get<Session>(KV.sessions, parsed.sessionId);+ if (existing) {+ existing.observationCount =+ (existing.observationCount || 0) + parsed.observations.length;+ if (parsed.endedAt > (existing.endedAt || "")) {+ existing.endedAt = parsed.endedAt;+ }+ if (existing.status === "active") existing.status = "completed";+ const existingTags = existing.tags || [];+ if (!existingTags.includes("jsonl-import")) {+ existing.tags = [...existingTags, "jsonl-import"];+ }+ if (!existing.firstPrompt && firstPrompt) {+ existing.firstPrompt = firstPrompt;+ }+ // #775: re-key on parsed.sessionId, not existing.id. Older+ // session rows may be missing the `id` field; existing.id+ // would then be undefined, JSON.stringify would drop the+ // `key` from the state::set payload, and the engine would+ // reject the call with `missing field \`key\``. Because the+ // rejection aborts the whole import handler, a single+ // legacy row killed the entire batch. parsed.sessionId is+ // always populated (parseJsonlText has a three-level+ // fallback) and is what we just used to read the row.+ if (!existing.id) existing.id = parsed.sessionId;+ await kv.set(KV.sessions, parsed.sessionId, existing);+ } else {+ const session: Session = {+ id: parsed.sessionId,+ project: parsed.project,+ cwd: parsed.cwd,+ startedAt: parsed.startedAt,+ endedAt: parsed.endedAt,+ status: "completed",+ observationCount: parsed.observations.length,+ tags: ["jsonl-import"],+ firstPrompt,+ };+ await kv.set(KV.sessions, session.id, session);}- if (existing.status === "active") existing.status = "completed";- const existingTags = existing.tags || [];- if (!existingTags.includes("jsonl-import")) {- existing.tags = [...existingTags, "jsonl-import"];- }- if (!existing.firstPrompt && firstPrompt) {- existing.firstPrompt = firstPrompt;- }- // #775: re-key on parsed.sessionId, not existing.id. Older- // session rows may be missing the `id` field; existing.id- // would then be undefined, JSON.stringify would drop the- // `key` from the state::set payload, and the engine would- // reject the call with `missing field \`key\``. Because the- // rejection aborts the whole import handler, a single- // legacy row killed the entire batch. parsed.sessionId is- // always populated (parseJsonlText has a three-level- // fallback) and is what we just used to read the row.- if (!existing.id) existing.id = parsed.sessionId;- await kv.set(KV.sessions, parsed.sessionId, existing);- } else {- const session: Session = {- id: parsed.sessionId,- project: parsed.project,- cwd: parsed.cwd,- startedAt: parsed.startedAt,- endedAt: parsed.endedAt,- status: "completed",- observationCount: parsed.observations.length,- tags: ["jsonl-import"],- firstPrompt,- };- await kv.set(KV.sessions, session.id, session);- }+ });const compressed: CompressedObservation[] = [];await Promise.all(diff --git a/src/functions/snapshot.ts b/src/functions/snapshot.tsindex 483240e1..06673845 100644--- a/src/functions/snapshot.ts+++ b/src/functions/snapshot.ts@@ -12,6 +12,7 @@ import type {} from "../types.js";import { KV, generateId } from "../state/schema.js";import type { StateKV } from "../state/kv.js";+import { withKeyedLock } from "../state/keyed-mutex.js";import { recordAudit } from "./audit.js";import { VERSION } from "../version.js";import { logger } from "../logger.js";@@ -195,7 +196,9 @@ export function registerSnapshotFunction(if (state.sessions) {for (const session of state.sessions) {- await kv.set(KV.sessions, session.id, session);+ await withKeyedLock(`obs:${session.id}`, () =>+ kv.set(KV.sessions, session.id, session),+ );}}if (state.memories) {@@ -205,7 +208,9 @@ export function registerSnapshotFunction(}if (state.graphNodes) {for (const node of state.graphNodes) {- await kv.set(KV.graphNodes, node.id, node);+ await withKeyedLock("graph:persist", () =>+ kv.set(KV.graphNodes, node.id, node),+ );}}if (state.observations) {diff --git a/src/functions/temporal-graph.ts b/src/functions/temporal-graph.tsindex aed6730b..18d13c1f 100644--- a/src/functions/temporal-graph.ts+++ b/src/functions/temporal-graph.ts@@ -9,6 +9,7 @@ import type {} from "../types.js";import { KV, generateId } from "../state/schema.js";import type { StateKV } from "../state/kv.js";+import { withKeyedLock } from "../state/keyed-mutex.js";import { logger } from "../logger.js";const TEMPORAL_EXTRACTION_SYSTEM = `You are a temporal knowledge extraction engine. Given observations, extract entities AND their temporal relationships with full context metadata.@@ -186,76 +187,78 @@ export function registerTemporalGraphFunctions(const obsIds = data.observations.map((o) => o.id);const { nodes, edges } = parseTemporalGraphXml(response, obsIds);- const existingNodes = await kv.list<GraphNode>(KV.graphNodes);- const existingEdges = await kv.list<GraphEdge>(KV.graphEdges);-- const idRemap = new Map<string, string>();- for (const node of nodes) {- const existing = existingNodes.find(- (n) =>- n.name === node.name && n.type === node.type,- );- if (existing) {- const oldId = node.id;- const merged = {- ...existing,- sourceObservationIds: [- ...new Set([- ...existing.sourceObservationIds,- ...obsIds,- ]),- ],- properties: { ...existing.properties, ...node.properties },- updatedAt: new Date().toISOString(),- aliases: [- ...new Set([- ...(existing.aliases || []),- ...(node.aliases || []),- ]),- ],- };- if (merged.aliases.length === 0) delete (merged as any).aliases;- await kv.set(KV.graphNodes, existing.id, merged);- node.id = existing.id;- idRemap.set(oldId, existing.id);- } else {- await kv.set(KV.graphNodes, node.id, node);- existingNodes.push(node);+ await withKeyedLock("graph:persist", async () => {+ const existingNodes = await kv.list<GraphNode>(KV.graphNodes);+ const existingEdges = await kv.list<GraphEdge>(KV.graphEdges);++ const idRemap = new Map<string, string>();+ for (const node of nodes) {+ const existing = existingNodes.find(+ (n) =>+ n.name === node.name && n.type === node.type,+ );+ if (existing) {+ const oldId = node.id;+ const merged = {+ ...existing,+ sourceObservationIds: [+ ...new Set([+ ...existing.sourceObservationIds,+ ...obsIds,+ ]),+ ],+ properties: { ...existing.properties, ...node.properties },+ updatedAt: new Date().toISOString(),+ aliases: [+ ...new Set([+ ...(existing.aliases || []),+ ...(node.aliases || []),+ ]),+ ],+ };+ if (merged.aliases.length === 0) delete (merged as any).aliases;+ await kv.set(KV.graphNodes, existing.id, merged);+ node.id = existing.id;+ idRemap.set(oldId, existing.id);+ } else {+ await kv.set(KV.graphNodes, node.id, node);+ existingNodes.push(node);+ }}- }- for (const edge of edges) {- if (idRemap.has(edge.sourceNodeId)) {- edge.sourceNodeId = idRemap.get(edge.sourceNodeId)!;+ for (const edge of edges) {+ if (idRemap.has(edge.sourceNodeId)) {+ edge.sourceNodeId = idRemap.get(edge.sourceNodeId)!;+ }+ if (idRemap.has(edge.targetNodeId)) {+ edge.targetNodeId = idRemap.get(edge.targetNodeId)!;+ }+ const existingKey = `${edge.sourceNodeId}|${edge.targetNodeId}|${edge.type}`;+ const existingEdge = existingEdges.find(+ (e) =>+ `${e.sourceNodeId}|${e.targetNodeId}|${e.type}` ===+ existingKey,+ );++ if (existingEdge) {+ const updatedOld = {+ ...existingEdge,+ isLatest: false,+ tvalidEnd:+ existingEdge.tvalidEnd || new Date().toISOString(),+ supersededBy: edge.id,+ };+ await kv.set(KV.graphEdges, existingEdge.id, updatedOld);++ await kv.set(KV.graphEdgeHistory, existingEdge.id, updatedOld);++ edge.version = (existingEdge.version || 1) + 1;+ }++ await kv.set(KV.graphEdges, edge.id, edge);+ existingEdges.push(edge);}- if (idRemap.has(edge.targetNodeId)) {- edge.targetNodeId = idRemap.get(edge.targetNodeId)!;- }- const existingKey = `${edge.sourceNodeId}|${edge.targetNodeId}|${edge.type}`;- const existingEdge = existingEdges.find(- (e) =>- `${e.sourceNodeId}|${e.targetNodeId}|${e.type}` ===- existingKey,- );-- if (existingEdge) {- const updatedOld = {- ...existingEdge,- isLatest: false,- tvalidEnd:- existingEdge.tvalidEnd || new Date().toISOString(),- supersededBy: edge.id,- };- await kv.set(KV.graphEdges, existingEdge.id, updatedOld);-- await kv.set(KV.graphEdgeHistory, existingEdge.id, updatedOld);-- edge.version = (existingEdge.version || 1) + 1;- }-- await kv.set(KV.graphEdges, edge.id, edge);- existingEdges.push(edge);- }+ });logger.info("Temporal graph extraction complete", {nodes: nodes.length,diff --git a/src/triggers/api.ts b/src/triggers/api.tsindex 4cf5431b..752ec03c 100644--- a/src/triggers/api.ts+++ b/src/triggers/api.ts@@ -731,7 +731,7 @@ export function registerApiTriggers(? body.agentId.trim().slice(0, 128): undefined;const agentId = requestAgentId ?? getAgentId();- const session: Session = {+ const freshSession: Session = {id: sessionId,project,cwd,@@ -742,7 +742,18 @@ export function registerApiTriggers(...(title ? { firstPrompt: title.slice(0, 200) } : {}),...(agentId ? { agentId } : {}),};- await kv.set(KV.sessions, sessionId, session);+ const session = await withKeyedLock(`obs:${sessionId}`, async () => {+ const existing = await kv.get<Session>(KV.sessions, sessionId);+ const merged: Session = {+ ...freshSession,+ observationCount: existing?.observationCount ?? freshSession.observationCount,+ firstPrompt: freshSession.firstPrompt ?? existing?.firstPrompt,+ summary: freshSession.summary ?? existing?.summary,+ commitShas: existing?.commitShas ?? freshSession.commitShas,+ };+ await kv.set(KV.sessions, sessionId, merged);+ return merged;+ });const contextResult = await sdk.trigger<{ sessionId: string; project: string; agentId?: string },{ context: string }@@ -775,10 +786,12 @@ export function registerApiTriggers(body: { error: "sessionId is required and must be a non-empty string" },};}- await kv.update(KV.sessions, sessionId, [- { type: "set", path: "endedAt", value: new Date().toISOString() },- { type: "set", path: "status", value: "completed" },- ]);+ await withKeyedLock(`obs:${sessionId}`, () =>+ kv.update(KV.sessions, sessionId, [+ { type: "set", path: "endedAt", value: new Date().toISOString() },+ { type: "set", path: "status", value: "completed" },+ ]),+ );// Fan out session-stopped lifecycle (non-blocking).try {sdk.trigger({@@ -871,7 +884,7 @@ export function registerApiTriggers(});if (sessionId) {- await withKeyedLock(`session:${sessionId}`, async () => {+ await withKeyedLock(`obs:${sessionId}`, async () => {const session = await kv.get<Session>(KV.sessions, sessionId);if (!session) return;const shaSet = new Set<string>(session.commitShas ?? []);diff --git a/src/triggers/events.ts b/src/triggers/events.tsindex 1e248236..c3187ce5 100644--- a/src/triggers/events.ts+++ b/src/triggers/events.ts@@ -2,6 +2,7 @@ import { TriggerAction, type IIIClient } from "iii-sdk";import type { CompressedObservation, HookPayload, Memory, Session } from "../types.js";import { KV, STREAM } from "../state/schema.js";import { StateKV } from "../state/kv.js";+import { withKeyedLock } from "../state/keyed-mutex.js";import { isReflectEnabled } from "../functions/slots.js";import {detectLlmProviderKind,@@ -56,7 +57,7 @@ export function registerEventTriggers(sdk: IIIClient, kv: StateKV): void {? data.agentId.trim().slice(0, 128): undefined;const agentId = requestAgentId ?? getAgentId();- const session: Session = {+ const freshSession: Session = {id: data.sessionId,project: data.project,cwd: data.cwd,@@ -65,7 +66,18 @@ export function registerEventTriggers(sdk: IIIClient, kv: StateKV): void {observationCount: 0,...(agentId ? { agentId } : {}),};- await kv.set(KV.sessions, data.sessionId, session);+ const session = await withKeyedLock(`obs:${data.sessionId}`, async () => {+ const existing = await kv.get<Session>(KV.sessions, data.sessionId);+ const merged: Session = {+ ...freshSession,+ observationCount: existing?.observationCount ?? freshSession.observationCount,+ firstPrompt: existing?.firstPrompt,+ summary: existing?.summary,+ commitShas: existing?.commitShas,+ };+ await kv.set(KV.sessions, data.sessionId, merged);+ return merged;+ });const contextResult = await sdk.trigger<{ sessionId: string; project: string; agentId?: string },{ context: string }@@ -160,10 +172,12 @@ export function registerEventTriggers(sdk: IIIClient, kv: StateKV): void {sdk.registerFunction("event::session::ended",async (data: { sessionId: string }) => {- await kv.update(KV.sessions, data.sessionId, [- { type: "set", path: "endedAt", value: new Date().toISOString() },- { type: "set", path: "status", value: "completed" },- ]);+ await withKeyedLock(`obs:${data.sessionId}`, () =>+ kv.update(KV.sessions, data.sessionId, [+ { type: "set", path: "endedAt", value: new Date().toISOString() },+ { type: "set", path: "status", value: "completed" },+ ]),+ );return { success: true };},);diff --git a/test/graph-cross-path-locks.test.ts b/test/graph-cross-path-locks.test.tsnew file mode 100644index 00000000..cbab748d--- /dev/null+++ b/test/graph-cross-path-locks.test.ts@@ -0,0 +1,161 @@+import { describe, it, expect, vi } from "vitest";++vi.mock("../src/logger.js", () => ({+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },+}));++import { persistGraphDelta } from "../src/functions/graph.js";+import { registerCascadeFunction } from "../src/functions/cascade.js";+import { KV } from "../src/state/schema.js";+import type { GraphNode, GraphSnapshot, Memory } from "../src/types.js";++function clone<T>(value: T): T {+ return value === null || value === undefined+ ? value+ : (JSON.parse(JSON.stringify(value)) as T);+}++function mockKV() {+ const store = new Map<string, Map<string, unknown>>();+ let gate: {+ scope: string;+ key: string;+ release: () => void;+ promise: Promise<void>;+ } | null = null;++ return {+ store,+ armGetGate: (scope: string, key: string) => {+ let release!: () => void;+ const promise = new Promise<void>((res) => {+ release = res;+ });+ gate = { scope, key, release, promise };+ return release;+ },+ get: async <T>(scope: string, key: string): Promise<T | null> => {+ const value = clone((store.get(scope)?.get(key) as T) ?? null);+ if (gate && gate.scope === scope && gate.key === key) {+ const g = gate;+ gate = null;+ await g.promise;+ }+ return value;+ },+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {+ const stored = clone(data);+ if (!store.has(scope)) store.set(scope, new Map());+ store.get(scope)!.set(key, stored);+ return clone(stored);+ },+ delete: async (scope: string, key: string) => {+ store.get(scope)?.delete(key);+ },+ list: async <T>(scope: string): Promise<T[]> => {+ const m = store.get(scope);+ return m ? clone(Array.from(m.values()) as T[]) : [];+ },+ };+}++function mockSdk() {+ const fns = new Map<string, (data: unknown) => Promise<unknown>>();+ return {+ registerFunction: (id: string, handler: (data: unknown) => Promise<unknown>) => {+ fns.set(id, handler);+ },+ trigger: (id: string, data: unknown) => fns.get(id)!(data),+ };+}++async function flush() {+ await new Promise((r) => setTimeout(r, 0));+}++describe("graph:persist serializes cross-path graph mutations", () => {+ it("does not lose cascade's stale flag or persistGraphDelta's merge when they race on the same node", async () => {+ const kv = mockKV();++ const node: GraphNode = {+ id: "node_1",+ type: "concept",+ name: "react",+ properties: {},+ sourceObservationIds: ["obs_old"],+ createdAt: "2026-03-01T00:00:00Z",+ };+ await kv.set(KV.graphNodes, "node_1", node);+ await kv.set(KV.graphNameIndex, "concept|react", "node_1");+ await kv.set(KV.graphNodeDegree, "node_1", 0);+ await kv.set(KV.graphSnapshot, "current", {+ version: 1,+ topNodes: [node],+ topEdges: [],+ topDegrees: {},+ stats: {+ totalNodes: 1,+ totalEdges: 0,+ nodesByType: { concept: 1 },+ edgesByType: {},+ },+ updatedAt: new Date().toISOString(),+ dirty: false,+ } satisfies GraphSnapshot);++ const supersededMemory: Memory = {+ id: "mem_old",+ createdAt: "2026-03-01T00:00:00Z",+ updatedAt: "2026-03-01T00:00:00Z",+ type: "fact",+ title: "Old fact",+ content: "Old content",+ concepts: [],+ files: [],+ sessionIds: [],+ strength: 5,+ version: 1,+ isLatest: false,+ sourceObservationIds: ["obs_old"],+ };+ await kv.set(KV.memories, "mem_old", supersededMemory);++ const sdk = mockSdk();+ registerCascadeFunction(sdk as never, kv as never);++ const releaseNodeRowRead = kv.armGetGate(KV.graphNodes, "node_1");++ const incomingNode: GraphNode = {+ id: "node_new",+ type: "concept",+ name: "react",+ properties: {},+ sourceObservationIds: ["obs_new"],+ createdAt: new Date().toISOString(),+ };+ const persistPromise = persistGraphDelta(+ kv as never,+ [incomingNode],+ [],+ ["obs_new"],+ );++ await flush();++ const cascadePromise = sdk.trigger("mem::cascade-update", {+ supersededMemoryId: "mem_old",+ }) as Promise<{ success: boolean; flagged: { nodes: number } }>;++ await flush();+ releaseNodeRowRead();++ const [, cascadeResult] = await Promise.all([persistPromise, cascadePromise]);++ expect(cascadeResult.success).toBe(true);+ expect(cascadeResult.flagged.nodes).toBe(1);++ const finalNode = await kv.get<GraphNode>(KV.graphNodes, "node_1");+ expect(finalNode!.stale).toBe(true);+ expect(finalNode!.sourceObservationIds.sort()).toEqual(["obs_new", "obs_old"]);+ });+});diff --git a/test/graph-extract-concurrency.test.ts b/test/graph-extract-concurrency.test.tsnew file mode 100644index 00000000..ca022a66--- /dev/null+++ b/test/graph-extract-concurrency.test.ts@@ -0,0 +1,225 @@+import { describe, it, expect, vi } from "vitest";++vi.mock("../src/logger.js", () => ({+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },+}));++import { persistGraphDelta, registerGraphFunction } from "../src/functions/graph.js";+import { KV } from "../src/state/schema.js";+import type { GraphNode, GraphEdge, GraphSnapshot } from "../src/types.js";++function clone<T>(value: T): T {+ return value === null || value === undefined+ ? value+ : (JSON.parse(JSON.stringify(value)) as T);+}++function mockKV() {+ const store = new Map<string, Map<string, unknown>>();+ let gate: {+ scope: string;+ key: string;+ release: () => void;+ promise: Promise<void>;+ } | null = null;+ let listGate: { scope: string; release: () => void; promise: Promise<void> } | null = null;++ return {+ store,+ armListGate: (scope: string) => {+ let release!: () => void;+ const promise = new Promise<void>((res) => {+ release = res;+ });+ listGate = { scope, release, promise };+ return release;+ },+ armGetGate: (scope: string, key: string) => {+ let release!: () => void;+ const promise = new Promise<void>((res) => {+ release = res;+ });+ gate = { scope, key, release, promise };+ return release;+ },+ get: async <T>(scope: string, key: string): Promise<T | null> => {+ const value = clone((store.get(scope)?.get(key) as T) ?? null);+ if (gate && gate.scope === scope && gate.key === key) {+ const g = gate;+ gate = null;+ await g.promise;+ }+ return value;+ },+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {+ const stored = clone(data);+ if (!store.has(scope)) store.set(scope, new Map());+ store.get(scope)!.set(key, stored);+ return clone(stored);+ },+ delete: async (scope: string, key: string) => {+ store.get(scope)?.delete(key);+ },+ list: async <T>(scope: string): Promise<T[]> => {+ const m = store.get(scope);+ const values = m ? clone(Array.from(m.values()) as T[]) : [];+ if (listGate && listGate.scope === scope) {+ const g = listGate;+ listGate = null;+ await g.promise;+ }+ return values;+ },+ };+}++function node(overrides: Partial<GraphNode> & { id: string; name: string }): GraphNode {+ return {+ type: "file",+ properties: {},+ sourceObservationIds: [],+ createdAt: new Date().toISOString(),+ ...overrides,+ };+}++async function flush() {+ await new Promise((r) => setTimeout(r, 0));+}++describe("persistGraphDelta concurrent merges (finding 7)", () => {+ it("does not create duplicate nodes when two concurrent extractions resolve the same (type, name) key", async () => {+ const kv = mockKV();++ const nameIndexKey = "file|src/index.ts";+ const releaseNameIndexRead = kv.armGetGate(KV.graphNameIndex, nameIndexKey);++ const nodeA = node({+ id: "node_a",+ name: "src/index.ts",+ sourceObservationIds: ["obs_a"],+ });+ const promiseA = persistGraphDelta(kv as never, [nodeA], [], ["obs_a"]);++ await flush();++ const nodeB = node({+ id: "node_b",+ name: "src/index.ts",+ sourceObservationIds: ["obs_b"],+ });+ const promiseB = persistGraphDelta(kv as never, [nodeB], [], ["obs_b"]);++ await flush();+ releaseNameIndexRead();++ await Promise.all([promiseA, promiseB]);++ const allNodes = await kv.list<GraphNode>(KV.graphNodes);+ expect(allNodes.length).toBe(1);++ const merged = allNodes[0];+ expect(merged.sourceObservationIds.sort()).toEqual(["obs_a", "obs_b"]);++ const snapshot = (await kv.get<GraphSnapshot>(+ KV.graphSnapshot,+ "current",+ )) as GraphSnapshot;+ expect(snapshot.stats.totalNodes).toBe(1);+ });++ it("does not lose either batch's obsIds when merging into an existing node concurrently", async () => {+ const kv = mockKV();++ const existing: GraphNode = node({+ id: "node_existing",+ name: "src/utils.ts",+ sourceObservationIds: ["obs_seed"],+ });+ await kv.set(KV.graphNodes, "node_existing", existing);+ await kv.set(KV.graphNameIndex, "file|src/utils.ts", "node_existing");+ await kv.set(KV.graphNodeDegree, "node_existing", 0);+ await kv.set(KV.graphSnapshot, "current", {+ version: 1,+ topNodes: [existing],+ topEdges: [],+ topDegrees: {},+ stats: {+ totalNodes: 1,+ totalEdges: 0,+ nodesByType: { file: 1 },+ edgesByType: {},+ },+ updatedAt: new Date().toISOString(),+ dirty: false,+ } satisfies GraphSnapshot);++ const releaseExistingNodeRead = kv.armGetGate(KV.graphNodes, "node_existing");++ const nodeA = node({+ id: "node_a2",+ name: "src/utils.ts",+ sourceObservationIds: ["obs_x"],+ });+ const promiseA = persistGraphDelta(kv as never, [nodeA], [], ["obs_x"]);++ await flush();++ const nodeB = node({+ id: "node_b2",+ name: "src/utils.ts",+ sourceObservationIds: ["obs_y"],+ });+ const promiseB = persistGraphDelta(kv as never, [nodeB], [], ["obs_y"]);++ await flush();+ releaseExistingNodeRead();++ await Promise.all([promiseA, promiseB]);++ const allNodes = await kv.list<GraphNode>(KV.graphNodes);+ expect(allNodes.length).toBe(1);+ expect(allNodes[0].sourceObservationIds.sort()).toEqual([+ "obs_seed",+ "obs_x",+ "obs_y",+ ]);+ });++ it("snapshot rebuild does not overwrite a delta that commits while it enumerates", async () => {+ const kv = mockKV();+ const handlers = new Map<string, (data?: unknown) => Promise<unknown>>();+ const sdk = {+ registerFunction: (id: string, fn: (data?: unknown) => Promise<unknown>) => {+ handlers.set(id, fn);+ },+ };+ registerGraphFunction(sdk as never, kv as never, { name: "noop" } as never);++ const seeded = node({ id: "node_seed", name: "src/seed.ts" });+ await persistGraphDelta(kv as never, [seeded], [], ["obs_seed"]);++ const releaseNodeList = kv.armListGate(KV.graphNodes);+ const rebuild = handlers.get("mem::graph-snapshot-rebuild")!();+ await flush();++ const fresh = node({ id: "node_fresh", name: "src/fresh.ts" });+ const delta = persistGraphDelta(kv as never, [fresh], [], ["obs_fresh"]);+ await flush();+ releaseNodeList();++ const rebuilt = (await rebuild) as { success: boolean };+ await delta;++ expect(rebuilt.success).toBe(true);+ const snapshot = (await kv.get<GraphSnapshot>(+ KV.graphSnapshot,+ "current",+ )) as GraphSnapshot;+ expect(snapshot.stats.totalNodes).toBe(2);+ expect(snapshot.topNodes.map((n) => n.id).sort()).toEqual([+ "node_fresh",+ "node_seed",+ ]);+ });+});diff --git a/test/session-kv-races.test.ts b/test/session-kv-races.test.tsnew file mode 100644index 00000000..40c2b0f1--- /dev/null+++ b/test/session-kv-races.test.ts@@ -0,0 +1,295 @@+import { describe, it, expect, vi } from "vitest";++vi.mock("../src/logger.js", () => ({+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },+}));++import { registerObserveFunction } from "../src/functions/observe.js";+import { registerApiTriggers } from "../src/triggers/api.js";+import { registerEventTriggers } from "../src/triggers/events.js";+import { KV } from "../src/state/schema.js";+import type { Session } from "../src/types.js";++type Gate = { scope: string; key: string; phase: "get" | "set" };++function clone<T>(value: T): T {+ return value === null || value === undefined+ ? value+ : (JSON.parse(JSON.stringify(value)) as T);+}++function mockKV() {+ const store = new Map<string, Map<string, unknown>>();+ let gate: (Gate & { release: () => void; promise: Promise<void> }) | null = null;++ function arm(scope: string, key: string, phase: "get" | "set"): () => void {+ let release!: () => void;+ const promise = new Promise<void>((res) => {+ release = res;+ });+ gate = { scope, key, phase, release, promise };+ return release;+ }++ async function maybeWait(scope: string, key: string, phase: "get" | "set") {+ if (gate && gate.scope === scope && gate.key === key && gate.phase === phase) {+ const g = gate;+ gate = null;+ await g.promise;+ }+ }++ return {+ store,+ armGetGate: (scope: string, key: string) => arm(scope, key, "get"),+ armSetGate: (scope: string, key: string) => arm(scope, key, "set"),+ get: async <T>(scope: string, key: string): Promise<T | null> => {+ const value = clone((store.get(scope)?.get(key) as T) ?? null);+ await maybeWait(scope, key, "get");+ return value;+ },+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {+ const stored = clone(data);+ await maybeWait(scope, key, "set");+ if (!store.has(scope)) store.set(scope, new Map());+ store.get(scope)!.set(key, stored);+ return clone(stored);+ },+ update: async (+ scope: string,+ key: string,+ ops: Array<{ type: string; path: string; value?: unknown }>,+ ) => {+ if (!store.has(scope)) store.set(scope, new Map());+ const m = store.get(scope)!;+ const current = clone((m.get(key) as Record<string, unknown>) ?? {});+ for (const op of ops) {+ if (op.type === "set") current[op.path] = op.value;+ }+ m.set(key, current);+ return clone(current);+ },+ delete: async (scope: string, key: string) => {+ store.get(scope)?.delete(key);+ },+ list: async <T>(scope: string): Promise<T[]> => {+ const m = store.get(scope);+ return m ? clone(Array.from(m.values()) as T[]) : [];+ },+ };+}++function mockSdk() {+ const fns = new Map<string, Function>();+ return {+ fns,+ registerFunction: (idOrOpts: string | { id: string }, fn: Function) => {+ const id = typeof idOrOpts === "string" ? idOrOpts : idOrOpts.id;+ fns.set(id, fn);+ },+ registerTrigger: () => {},+ trigger: async (+ idOrInput: string | { function_id: string; payload: unknown; action?: unknown },+ data?: unknown,+ ) => {+ const id = typeof idOrInput === "string" ? idOrInput : idOrInput.function_id;+ const payload = typeof idOrInput === "string" ? data : idOrInput.payload;+ const fn = fns.get(id);+ if (fn) return fn(payload);+ return null;+ },+ };+}++async function flush() {+ await new Promise((r) => setTimeout(r, 0));+}++describe("KV.sessions concurrent mutation races (finding 3)", () => {+ it("api::session::commit does not revert a concurrent mem::observe's observationCount increment", async () => {+ const kv = mockKV();+ const sdk = mockSdk();+ registerObserveFunction(sdk as never, kv as never);+ registerApiTriggers(sdk as never, kv as never);++ const session: Session = {+ id: "ses_race1",+ project: "/repo",+ cwd: "/repo",+ startedAt: new Date().toISOString(),+ status: "active",+ observationCount: 0,+ };+ await kv.set(KV.sessions, "ses_race1", session);++ const releaseCommitRead = kv.armGetGate(KV.sessions, "ses_race1");++ const commitPromise = sdk.trigger("api::session::commit", {+ body: { sha: "abc123def", sessionId: "ses_race1" },+ });++ await flush();++ const observePromise = sdk.trigger("mem::observe", {+ sessionId: "ses_race1",+ project: "/repo",+ cwd: "/repo",+ hookType: "prompt_submit",+ timestamp: new Date().toISOString(),+ data: { prompt: "do the thing" },+ });++ await flush();+ releaseCommitRead();++ await Promise.all([commitPromise, observePromise]);++ const finalSession = kv.store.get(KV.sessions)?.get("ses_race1") as Session;+ expect(finalSession.observationCount).toBe(1);+ expect(finalSession.commitShas).toEqual(["abc123def"]);+ });++ it("mem::observe's implicit session create does not clobber a concurrent api::session::start's title", async () => {+ const kv = mockKV();+ const sdk = mockSdk();+ registerObserveFunction(sdk as never, kv as never);+ registerApiTriggers(sdk as never, kv as never);+ sdk.fns.set("mem::context", async () => ({ context: "" }));++ const releaseStartWrite = kv.armSetGate(KV.sessions, "ses_race2");++ const startPromise = sdk.trigger("api::session::start", {+ body: {+ sessionId: "ses_race2",+ project: "/repo",+ cwd: "/repo",+ title: "fix the parser",+ },+ });++ await flush();++ const observePromise = sdk.trigger("mem::observe", {+ sessionId: "ses_race2",+ project: "/repo",+ cwd: "/repo",+ hookType: "prompt_submit",+ timestamp: new Date().toISOString(),+ data: { prompt: "fix the parser" },+ });++ await flush();+ releaseStartWrite();++ await Promise.all([startPromise, observePromise]);++ const finalSession = kv.store.get(KV.sessions)?.get("ses_race2") as Session;+ expect(finalSession.summary).toBe("fix the parser");+ expect(finalSession.observationCount).toBe(1);+ });++ it("event::session::started racing mem::observe's implicit create does not revert the observation count to zero", async () => {+ const kv = mockKV();+ const sdk = mockSdk();+ registerEventTriggers(sdk as never, kv as never);+ registerObserveFunction(sdk as never, kv as never);+ sdk.fns.set("mem::context", async () => ({ context: "" }));++ const releaseEventWrite = kv.armSetGate(KV.sessions, "ses_race3");++ const eventPromise = sdk.trigger("event::session::started", {+ sessionId: "ses_race3",+ project: "/repo",+ cwd: "/repo",+ });++ await flush();++ const observePromise = sdk.trigger("mem::observe", {+ sessionId: "ses_race3",+ project: "/repo",+ cwd: "/repo",+ hookType: "prompt_submit",+ timestamp: new Date().toISOString(),+ data: { prompt: "start work" },+ });++ await flush();+ releaseEventWrite();++ await Promise.all([eventPromise, observePromise]);++ const finalSession = kv.store.get(KV.sessions)?.get("ses_race3") as Session;+ expect(finalSession).toBeTruthy();+ expect(finalSession.observationCount).toBe(1);+ });++ it("api::session::commit does not revert a concurrent api::session::end status update", async () => {+ const kv = mockKV();+ const sdk = mockSdk();+ registerApiTriggers(sdk as never, kv as never);++ const session: Session = {+ id: "ses_race5",+ project: "/repo",+ cwd: "/repo",+ startedAt: new Date().toISOString(),+ status: "active",+ observationCount: 3,+ };+ await kv.set(KV.sessions, "ses_race5", session);++ const releaseCommitRead = kv.armGetGate(KV.sessions, "ses_race5");++ const commitPromise = sdk.trigger("api::session::commit", {+ body: { sha: "deadbeef01", sessionId: "ses_race5" },+ });++ await flush();++ const endPromise = sdk.trigger("api::session::end", {+ body: { sessionId: "ses_race5" },+ });++ await flush();+ releaseCommitRead();++ await Promise.all([commitPromise, endPromise]);++ const finalSession = kv.store.get(KV.sessions)?.get("ses_race5") as Session;+ expect(finalSession.commitShas).toEqual(["deadbeef01"]);+ expect(finalSession.status).toBe("completed");+ expect(finalSession.endedAt).toBeTruthy();+ });++ it("api::session::start does not reset an observation count or firstPrompt already created by mem::observe", async () => {+ const kv = mockKV();+ const sdk = mockSdk();+ registerObserveFunction(sdk as never, kv as never);+ registerApiTriggers(sdk as never, kv as never);+ sdk.fns.set("mem::context", async () => ({ context: "" }));++ await sdk.trigger("mem::observe", {+ sessionId: "ses_race4",+ project: "/repo",+ cwd: "/repo",+ hookType: "prompt_submit",+ timestamp: new Date().toISOString(),+ data: { prompt: "do the thing" },+ });++ await sdk.trigger("api::session::start", {+ body: {+ sessionId: "ses_race4",+ project: "/repo",+ cwd: "/repo",+ title: "fix the parser",+ },+ });++ const finalSession = kv.store.get(KV.sessions)?.get("ses_race4") as Session;+ expect(finalSession.observationCount).toBe(1);+ expect(finalSession.firstPrompt).toBe("fix the parser");+ expect(finalSession.status).toBe("active");+ });+});