Multiplayer agent harness for work.
Guard memory consolidation and promotion against concurrent notebook writes (#454)
Regan Bell8d39f020ec58
diff --git a/src/memory/memory-service.ts b/src/memory/memory-service.tsindex 4ff86a01f..d52e3afd0 100644--- a/src/memory/memory-service.ts+++ b/src/memory/memory-service.ts@@ -1,6 +1,7 @@import { createHash } from "node:crypto";import { type ScopeId, parseScopeId, scopeId as makeScopeId } from "../types.ts";import type { WorkspaceStore } from "../workspace/workspace-store.ts";+import { createKeyedQueue } from "../util/async.ts";import { RECALL_MAX_CHARS, bullets, capTail, dateStr, isBullet, normalize } from "./notebook.ts";export const MEMORY_FILE = "memory/MEMORY.md";@@ -105,17 +106,20 @@ export function normalizeReplace(content: string): string {}export function createMemoryService(workspace: WorkspaceStore): MemoryService {+ const perScope = createKeyedQueue<ScopeId>();return {async recall(scopeId) {return recallBody((await workspace.read(scopeId, MEMORY_FILE)) ?? "");},async capture(scopeId, facts, at, author) {- const existing = (await workspace.read(scopeId, MEMORY_FILE)) ?? "";- const { body, added } = foldCapture(existing, facts, at, author?.startsWith("cc:") === true);- if (!added) return 0;- await workspace.write(scopeId, MEMORY_FILE, `${body}\n`);- return added;+ return perScope(scopeId, async () => {+ const existing = (await workspace.read(scopeId, MEMORY_FILE)) ?? "";+ const { body, added } = foldCapture(existing, facts, at, author?.startsWith("cc:") === true);+ if (!added) return 0;+ await workspace.write(scopeId, MEMORY_FILE, `${body}\n`);+ return added;+ });},async query(scopeId, q, limit = 20) {@@ -127,26 +131,32 @@ export function createMemoryService(workspace: WorkspaceStore): MemoryService {},async replace(scopeId, content) {- const next = normalizeReplace(content);- if (!next) {- await workspace.remove(scopeId, MEMORY_FILE);- return;- }- await workspace.write(scopeId, MEMORY_FILE, next);+ await perScope(scopeId, async () => {+ const next = normalizeReplace(content);+ if (!next) {+ await workspace.remove(scopeId, MEMORY_FILE);+ return;+ }+ await workspace.write(scopeId, MEMORY_FILE, next);+ });},async readHead(scopeId) {- const content = (await workspace.read(scopeId, MEMORY_FILE)) ?? "";- return { content, revision: revisionToken(content) };+ return perScope(scopeId, async () => {+ const content = (await workspace.read(scopeId, MEMORY_FILE)) ?? "";+ return { content, revision: revisionToken(content) };+ });},async replaceIfRevision(scopeId, content, revision) {- const current = (await workspace.read(scopeId, MEMORY_FILE)) ?? "";- if (revisionToken(current) !== revision) return false;- const next = normalizeReplace(content);- if (!next) await workspace.remove(scopeId, MEMORY_FILE);- else await workspace.write(scopeId, MEMORY_FILE, next);- return true;+ return perScope(scopeId, async () => {+ const current = (await workspace.read(scopeId, MEMORY_FILE)) ?? "";+ if (revisionToken(current) !== revision) return false;+ const next = normalizeReplace(content);+ if (!next) await workspace.remove(scopeId, MEMORY_FILE);+ else await workspace.write(scopeId, MEMORY_FILE, next);+ return true;+ });},};}diff --git a/src/memory/strategies/consolidation.ts b/src/memory/strategies/consolidation.tsindex 34e324ad4..10d8d512b 100644--- a/src/memory/strategies/consolidation.ts+++ b/src/memory/strategies/consolidation.ts@@ -133,7 +133,9 @@ export function createConsolidator(deps: {const degraded = new Set<ScopeId>();async function maintain(scopeId: ScopeId): Promise<void> {if (degraded.has(scopeId) || !deps.harness.oneShot) return;- const body = await deps.memory.read(scopeId);+ const guarded = deps.memory.readHead && deps.memory.replaceIfRevision;+ const head = guarded ? await deps.memory.readHead!(scopeId) : undefined;+ const body = head?.content ?? (await deps.memory.read(scopeId));const bullets = body.split("\n").filter(isBullet);if (!bullets.length) return;@@ -146,6 +148,10 @@ export function createConsolidator(deps: {}const at = now();const next = applyConsolidationActions(body, parseConsolidationActions(out ?? ""), at);+ if (head) {+ await deps.memory.replaceIfRevision!(scopeId, next, head.revision, "system");+ return;+ }await deps.memory.replace(scopeId, next, "system");const after = await deps.memory.read(scopeId);diff --git a/src/memory/strategies/scratch-promote.ts b/src/memory/strategies/scratch-promote.tsindex e5113b28d..b719c86ea 100644--- a/src/memory/strategies/scratch-promote.ts+++ b/src/memory/strategies/scratch-promote.ts@@ -72,21 +72,41 @@ export interface ScratchPromoteDeps {export function createScratchPromote(deps: ScratchPromoteDeps): { strategy: MemoryStrategy; memory: MemoryService } {const base = deps.memory;const workspace = deps.workspace;+ const perScope = createKeyedQueue<ScopeId>();++ async function rewriteMarker(scopeId: ScopeId, edit: (body: string) => string | null): Promise<string | null> {+ for (let attempt = 0; attempt < 3; attempt++) {+ const head = base.readHead && base.replaceIfRevision ? await base.readHead(scopeId) : null;+ const body = head ? head.content : await base.read(scopeId);+ const next = edit(body);+ if (next === null) return null;+ if (!head) {+ await base.replace(scopeId, next);+ return next;+ }+ if (await base.replaceIfRevision!(scopeId, next, head.revision)) return next;+ }+ return null;+ }async function bumpMarker(scopeId: ScopeId, by: number): Promise<number> {+ let count = 0;+ const committed = await rewriteMarker(scopeId, (body) => {+ const m = body.match(MARKER_RE);+ count = (m ? Number(m[1]) : 0) + by;+ const marker = `<!-- captures-since-promote: ${count} -->`;+ return m ? body.replace(MARKER_RE, marker) : `${body.trim() || "# Memory"}\n\n${marker}`;+ });+ if (committed !== null) return count;const body = await base.read(scopeId);const m = body.match(MARKER_RE);- const count = (m ? Number(m[1]) : 0) + by;- const marker = `<!-- captures-since-promote: ${count} -->`;- const next = m ? body.replace(MARKER_RE, marker) : `${body.trim() || "# Memory"}\n\n${marker}`;- await base.replace(scopeId, next);- return count;+ return m ? Number(m[1]) : 0;}async function resetMarker(scopeId: ScopeId): Promise<void> {- const body = await base.read(scopeId);- if (MARKER_RE.test(body))- await base.replace(scopeId, body.replace(MARKER_RE, "<!-- captures-since-promote: 0 -->"));+ await rewriteMarker(scopeId, (body) =>+ MARKER_RE.test(body) ? body.replace(MARKER_RE, "<!-- captures-since-promote: 0 -->") : null,+ );}async function readLogWindow(@@ -114,31 +134,33 @@ export function createScratchPromote(deps: ScratchPromoteDeps): { strategy: Memo},async capture(scopeId, facts, at) {- const clean = facts.map((f) => f.replace(/\s+/g, " ").trim()).filter(Boolean);- if (!clean.length) return 0;- const path = logPath(at);- const existing = (await workspace.read(scopeId, path)) ?? "";- const seen = new Set(bullets(existing).map(normalize));- const date = dateStr(at);- const added: string[] = [];- for (const f of clean) {- const key = normalize(f);- if (!key || seen.has(key)) continue;- seen.add(key);- added.push(`- (${date}) ${f}`);- }- if (!added.length) return 0;- const body = existing.trim()- ? `${existing.replace(/\s+$/, "")}\n${added.join("\n")}`- : `# Scratch log ${date}\n\n${added.join("\n")}`;- await workspace.write(scopeId, path, `${body}\n`);-- const count = await bumpMarker(scopeId, added.length);- if (deps.consolidateAfter > 0 && count >= deps.consolidateAfter) {- await resetMarker(scopeId);- await strategy.maintain!(scopeId).catch(() => {});- }- return added.length;+ return perScope(scopeId, async () => {+ const clean = facts.map((f) => f.replace(/\s+/g, " ").trim()).filter(Boolean);+ if (!clean.length) return 0;+ const path = logPath(at);+ const existing = (await workspace.read(scopeId, path)) ?? "";+ const seen = new Set(bullets(existing).map(normalize));+ const date = dateStr(at);+ const added: string[] = [];+ for (const f of clean) {+ const key = normalize(f);+ if (!key || seen.has(key)) continue;+ seen.add(key);+ added.push(`- (${date}) ${f}`);+ }+ if (!added.length) return 0;+ const body = existing.trim()+ ? `${existing.replace(/\s+$/, "")}\n${added.join("\n")}`+ : `# Scratch log ${date}\n\n${added.join("\n")}`;+ await workspace.write(scopeId, path, `${body}\n`);++ const count = await bumpMarker(scopeId, added.length);+ if (deps.consolidateAfter > 0 && count >= deps.consolidateAfter) {+ await resetMarker(scopeId);+ await strategy.maintain!(scopeId).catch(() => {});+ }+ return added.length;+ });},async query(scopeId, q, limit = 20) {@@ -156,19 +178,15 @@ export function createScratchPromote(deps: ScratchPromoteDeps): { strategy: Memo},};- const perScope = createKeyedQueue<ScopeId>();-async function flushBurst(burst: Burst): Promise<void> {const autonomous = isAutonomousBurst(burst);const facts = await extractFacts(deps.harness, burst.turns, { autonomous });if (!facts.length) return;const at = Date.now();- await perScope(burst.scopeId, () => memory.capture(burst.scopeId, facts, at));+ await memory.capture(burst.scopeId, facts, at);const ccTarget = autonomous ? null : ccTargetFor(burst.conversationScopeId, burst.actorId);if (!ccTarget) return;- await perScope(ccTarget, () =>- ccCaptureToPersonal(memory, burst.conversationScopeId, burst.actorId, facts, at, burst.conversationLabel),- );+ await ccCaptureToPersonal(memory, burst.conversationScopeId, burst.actorId, facts, at, burst.conversationLabel);}const strategy: MemoryStrategy = {@@ -183,7 +201,9 @@ export function createScratchPromote(deps: ScratchPromoteDeps): { strategy: Memoconst now = Date.now();const window = await readLogWindow(scopeId, now, LOG_RETENTION_DAYS);if (window.length && deps.harness.oneShot) {- const longTerm = stripMarker(await base.read(scopeId));+ const head = base.readHead && base.replaceIfRevision ? await base.readHead(scopeId) : null;+ const raw = head ? head.content : await base.read(scopeId);+ const longTerm = stripMarker(raw);const scratch = window.map(({ date, body }) => `## ${date}\n${body}`).join("\n\n");const out = ((await deps.harness.oneShot(@@ -191,7 +211,13 @@ export function createScratchPromote(deps: ScratchPromoteDeps): { strategy: Memo`Current notebook:\n${longTerm || "(empty)"}\n\nScratch log:\n${scratch}`,)) ?? "").trim();- if (out && !/^none$/i.test(out)) await base.replace(scopeId, out);+ if (out && !/^none$/i.test(out)) {+ if (head) {+ await base.replaceIfRevision!(scopeId, out, head.revision);+ } else {+ await base.replace(scopeId, out);+ }+ }}const cutoff = dateStr(now - LOG_RETENTION_DAYS * 86_400_000);for (const abs of await workspace.list(scopeId)) {diff --git a/test/memory-strategy-consolidation.test.ts b/test/memory-strategy-consolidation.test.tsindex 9b8345cab..f8a28a0e9 100644--- a/test/memory-strategy-consolidation.test.ts+++ b/test/memory-strategy-consolidation.test.ts@@ -155,6 +155,46 @@ test("a one-shot failure leaves the notebook untouched — consolidation is bestassert.equal(await workspace.read(SCOPE, MEMORY_FILE), before);});+test("an edit landing during consolidation survives without disabling later consolidation", async () => {+ const { memory } = freshMemory();+ await memory.capture(SCOPE, ["original fact"], AT);+ let calls = 0;+ let release!: () => void;+ const blocked = new Promise<void>((resolve) => {+ release = resolve;+ });+ let modelStarted!: () => void;+ const waiting = new Promise<void>((resolve) => {+ modelStarted = resolve;+ });+ const logs: string[] = [];+ const consolidator = createConsolidator({+ harness: {+ async oneShot() {+ calls++;+ modelStarted();+ await blocked;+ return "UPDATE 1: consolidated fact";+ },+ },+ memory,+ log: (message) => logs.push(message),+ })!;++ const first = consolidator.maintain(SCOPE);+ await waiting;+ await memory.replace(SCOPE, "# Memory\n\n- user edit");+ release();+ await first;++ assert.match(await memory.read(SCOPE), /user edit/);+ assert.doesNotMatch(await memory.read(SCOPE), /consolidated fact/);+ assert.deepEqual(logs, []);++ await consolidator.maintain(SCOPE);+ assert.equal(calls, 2);+});+test("degrades to capture-only when the store can't round-trip a rewrite: logs once, stops trying, never crashes", async () => {const body = "# Memory\n\n- (2026-06-01) a fact\n";const memory: MemoryService = {diff --git a/test/memory-strategy-scratch-promote.test.ts b/test/memory-strategy-scratch-promote.test.tsindex c886ff373..59b5fce80 100644--- a/test/memory-strategy-scratch-promote.test.ts+++ b/test/memory-strategy-scratch-promote.test.ts@@ -53,6 +53,69 @@ test("capture lands in the dated scratch log, not MEMORY.md", async () => {assert.equal(await memory.capture(SCOPE, ["prefers terse replies"], TODAY), 0);});+test("a notebook edit landing during a marker bump survives", async () => {+ const workspace = createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "msp-")));+ const base = createMemoryService(workspace);+ let racedOnce = false;+ const racy: typeof base = {+ ...base,+ async readHead(scopeId) {+ const head = await base.readHead!(scopeId);+ if (!racedOnce) {+ racedOnce = true;+ await base.replace(scopeId, "# Memory\n\n- user edit mid-bump");+ }+ return head;+ },+ };+ const { memory } = createScratchPromote({+ harness: harnessOf(),+ memory: racy,+ workspace,+ consolidateAfter: 0,+ });++ await memory.capture(SCOPE, ["a fresh fact"], TODAY);++ const notebook = (await workspace.read(SCOPE, MEMORY_FILE)) ?? "";+ assert.match(notebook, /user edit mid-bump/, "the concurrent edit is not reverted by the marker write");+ assert.match(notebook, /captures-since-promote: 1/, "the marker still lands");+});++test("marker CAS exhaustion reports the persisted count, not the phantom bump", async () => {+ const workspace = createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "msp-")));+ const base = createMemoryService(workspace);+ let edits = 0;+ const contended: typeof base = {+ ...base,+ async readHead(scopeId) {+ const head = await base.readHead!(scopeId);+ edits += 1;+ await base.replace(scopeId, `# Memory\n\n- concurrent edit ${edits}`);+ return head;+ },+ };+ let consolidations = 0;+ const { memory, strategy } = createScratchPromote({+ harness: harnessOf(),+ memory: contended,+ workspace,+ consolidateAfter: 1,+ });+ strategy.maintain = async () => {+ consolidations += 1;+ };++ const added = await memory.capture(SCOPE, ["a fact under contention"], TODAY);++ assert.equal(added, 1, "the capture itself still lands in the scratch log");+ assert.ok(edits >= 3, "every CAS attempt lost to a concurrent edit");+ assert.equal(consolidations, 0, "an unpersisted counter must not trigger consolidation");+ const notebook = (await workspace.read(SCOPE, MEMORY_FILE)) ?? "";+ assert.match(notebook, /concurrent edit/, "the user's edit wins over the abandoned marker write");+ assert.doesNotMatch(notebook, /captures-since-promote: 1/, "no phantom marker landed");+});+test("recall window = MEMORY.md + today + yesterday, across the day boundary", async () => {const { memory } = fresh();await memory.replace(SCOPE, "# Memory\n\n- (2026-01-01) Long-term fact");@@ -80,6 +143,33 @@ test("query greps the scratch log window in addition to the notebook", async ()assert.deepEqual(hits, ["(2026-01-01) zebra notebook fact", "(2026-06-10) zebra scratch fact"]);});+test("maintain with readHead but no replaceIfRevision falls back to plain read and replace", async () => {+ const workspace = createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "msp-")));+ const base = createMemoryService(workspace);+ let headReads = 0;+ const partial: typeof base = {+ ...base,+ async readHead(scopeId) {+ headReads += 1;+ return base.readHead!(scopeId);+ },+ };+ delete (partial as { replaceIfRevision?: unknown }).replaceIfRevision;+ const promoted = "# Memory\n\n- (2026-06-10) Promoted without CAS";+ const { memory, strategy } = createScratchPromote({+ harness: harnessOf(() => Promise.resolve(promoted)),+ memory: partial,+ workspace,+ consolidateAfter: 0,+ });+ await memory.capture(SCOPE, ["Promoted without CAS"], TODAY);++ await withNow(TODAY, () => strategy.maintain!(SCOPE));++ assert.equal(headReads, 0, "a snapshot revision is never taken when it cannot be enforced");+ assert.equal(await workspace.read(SCOPE, MEMORY_FILE), `${promoted}\n`);+});+test("maintain promotes: one-shot judges the window, rewrites MEMORY.md, leaves the log untouched", async () => {const calls: Array<{ system: string; prompt: string }> = [];const promoted = "# Memory\n\n- (2026-06-10) Durable graduated fact";@@ -146,6 +236,15 @@ test("after-N marker trigger: the Nth capture fires promotion and resets the dur);});+test("concurrent captures retain both marker increments", async () => {+ const { memory } = fresh();+ await Promise.all([+ memory.capture(SCOPE, ["first concurrent fact"], TODAY),+ memory.capture(SCOPE, ["second concurrent fact"], TODAY),+ ]);+ assert.match(await memory.read(SCOPE), /captures-since-promote: 2/);+});+test("onTurnEnd extracts facts and captures them into today's log", async () => {const { workspace, strategy } = fresh({ oneShot: () => Promise.resolve("- Works at Acme") });await withNow(TODAY, () => strategy.onTurnEnd!({ scopeId: SCOPE, input: "hi", reply: "hello" }));@@ -173,3 +272,17 @@ test("strategy wiring: scratch-promote parses, wraps the store, and ships prompt"per-turn gets the consolidating store — captures by any path trigger the after-N check",);});++test("a save landing during promotion is not reverted by the promote write", async () => {+ const { base, strategy, memory } = fresh({+ oneShot: async () => {+ await base.replace(SCOPE, "# Memory\n\n- (2026-06-10) the newer edit");+ return "# Memory\n\n- (2026-06-10) promoted fact";+ },+ });+ await withNow(TODAY, () => memory.capture(SCOPE, ["something recent"], TODAY));+ await withNow(TODAY, () => strategy.maintain!(SCOPE));+ const after = await base.read(SCOPE);+ assert.match(after, /the newer edit/, "the mid-flight edit survives");+ assert.doesNotMatch(after, /promoted fact/, "the stale promotion is dropped, not applied");+});diff --git a/test/memory.test.ts b/test/memory.test.tsindex a0a0bbdc6..542f9560f 100644--- a/test/memory.test.ts+++ b/test/memory.test.ts@@ -24,6 +24,21 @@ function freshApp(overrides: Partial<Config> = {}) {const actor = { externalId: "U1" };+test("file memory compare-and-set permits only one writer for a revision", async () => {+ const workspace = createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "memory-cas-")));+ const memory = createMemoryService(workspace);+ const target = scopeId("personal", "cas-user");+ await memory.replace(target, "# Memory\n\n- original");+ const head = await memory.readHead!(target);++ const results = await Promise.all([+ memory.replaceIfRevision!(target, "# Memory\n\n- first", head.revision),+ memory.replaceIfRevision!(target, "# Memory\n\n- second", head.revision),+ ]);++ assert.deepEqual(results.sort(), [false, true]);+});+function dm(text: string, thread: string): TurnRequest {return { surface: "test", actor, conversation: { kind: "dm", threadRef: thread }, text };}