A self-organizing Obsidian vault that gives AI coding agents persistent memory.
TypeScript ai-agentsclaude-codecodex-cligemini-cliknowledge-managementobsidianobsidian-vaultpersistent-memorysecond-braintemplate
fix: guard markSuperseded against concurrent writers (#342)
akushonkamenfae7be467fc5
diff --git a/.claude/scripts/lib/atomic-write.ts b/.claude/scripts/lib/atomic-write.tsindex fa4a04f..74c5bb7 100644--- a/.claude/scripts/lib/atomic-write.ts+++ b/.claude/scripts/lib/atomic-write.ts@@ -18,7 +18,17 @@* half-written note for the indexer to pick up.*/-import { writeFileSync, copyFileSync, renameSync, unlinkSync, constants } from "node:fs";+import {+ writeFileSync,+ readFileSync,+ copyFileSync,+ renameSync,+ unlinkSync,+ openSync,+ closeSync,+ statSync,+ constants,+} from "node:fs";import { join } from "node:path";/** Per-process counter, so two in-flight writes cannot share a temp name. */@@ -100,3 +110,162 @@ export function writeFileAtomic(path: string, text: string): void {throw err;}}++/** What a guarded rewrite ended up doing. */+export type RewriteResult = "written" | "unchanged" | "contended";++/**+ * Rewrite an existing file only if it is unchanged since `transform` read it.+ *+ * WHY NOT just read, edit, writeFileAtomic+ *+ * That is a read-modify-write with nothing between the read and the write that+ * notices the file changed. Two writers each read the original, each applies+ * its own edit, and the second rename erases the first — both reported as+ * written. markSuperseded lost 90 of 200 titles exactly this way (#328).+ * `writeFileAtomic` makes each replacement invisible to readers; it does+ * nothing about the lost update between them.+ *+ * WHY A LOCK FILE, and not only a re-read before the rename+ *+ * Re-reading just before replacing shrinks the window from the whole edit to+ * the gap between that read and the rename, but does not close it: four+ * processes marking one memory together still lost a title in most runs of 50+ * rounds, each reported as written. So writers going through this function+ * first take `<file>.lock` by exclusive create, the same primitive `claimFile`+ * relies on, and only one of them is ever between its read and its rename.+ *+ * The re-read stays, for writers that take no lock — a person saving the note+ * in Obsidian. If the file changed since `transform` read it, the temp is+ * thrown away and the edit replayed on the fresh content, a bounded number of+ * times. `transform` returning null means there is nothing to change.+ *+ * A held lock lasts a few filesystem calls, so one older than STALE_LOCK_MS was+ * left by a process that died holding it, and is taken over. A writer that+ * cannot get the lock within `lockWaitMs` reports `contended` — told, not+ * silently dropped.+ *+ * ON WINDOWS, renaming over or reading a file another process holds at that+ * instant throws EPERM (EBUSY and EACCES are the same sharing violation by+ * other names). An indexer or editor reading the note is enough. That is a busy+ * file, not a broken one, so it is retried after a short pause, under its own+ * bound since it says nothing about our read being stale. Only on Windows,+ * where these codes are transient; on POSIX they mean a real permission+ * problem and are thrown at once.+ */+export function rewriteIfUnchanged(+ path: string,+ transform: (current: string) => string | null,+ attempts = 3,+ lockWaitMs = LOCK_WAIT_MS,+): RewriteResult {+ const lock = `${path}.lock`;+ if (!acquireLock(lock, lockWaitMs)) return "contended";+ try {+ return replaceIfUnchanged(path, transform, attempts);+ } finally {+ releaseLock(lock);+ }+}++/** Longest a writer waits for another's lock before reporting `contended`. */+const LOCK_WAIT_MS = 5_000;+/** Age past which a lock is a dead process's leftover rather than a writer. */+const STALE_LOCK_MS = 10_000;+/** How many sharing violations one rewrite or release sits out before throwing. */+const BUSY_RETRIES = 50;++function replaceIfUnchanged(+ path: string,+ transform: (current: string) => string | null,+ attempts: number,+): RewriteResult {+ let busy = 0;+ for (let attempt = 0; attempt < attempts; ) {+ let tmp: string | null = null;+ try {+ const before = readFileSync(path, "utf8");+ const next = transform(before);+ if (next === null || next === before) return "unchanged";+ tmp = `${path}.${process.pid}-${++tmpSeq}.tmp`;+ writeFileSync(tmp, next, "utf8");+ if (readFileSync(path, "utf8") !== before) {+ attempt++; // changed under us: retry on fresh content+ continue;+ }+ renameSync(tmp, path);+ tmp = null;+ return "written";+ } catch (err) {+ if (!isSharingViolation(err) || ++busy > BUSY_RETRIES) throw err;+ pauseMs(1 + Math.random() * 4);+ } finally {+ if (tmp) {+ try {+ unlinkSync(tmp);+ } catch {+ /* nothing to clean up */+ }+ }+ }+ }+ return "contended";+}++function acquireLock(lock: string, waitMs: number): boolean {+ const deadline = Date.now() + waitMs;+ for (;;) {+ try {+ closeSync(openSync(lock, "wx"));+ return true;+ } catch (err) {+ const code = (err as NodeJS.ErrnoException)?.code;+ if (code !== "EEXIST" && !isSharingViolation(err)) throw err;+ }+ if (lockIsStale(lock)) {+ try {+ unlinkSync(lock);+ } catch {+ /* another writer took it over first */+ }+ continue;+ }+ if (Date.now() >= deadline) return false;+ pauseMs(1 + Math.random() * 4); // jittered, so waiting writers stop meeting+ }+}++function lockIsStale(lock: string): boolean {+ try {+ return Date.now() - statSync(lock).mtimeMs > STALE_LOCK_MS;+ } catch {+ return false; // gone already: the next exclusive create decides+ }+}++function releaseLock(lock: string): void {+ for (let busy = 0; ; busy++) {+ try {+ unlinkSync(lock);+ return;+ } catch (err) {+ // Left in place, it would stall every writer until it went stale.+ if (!isSharingViolation(err) || busy >= BUSY_RETRIES) return;+ pauseMs(1 + Math.random() * 4);+ }+ }+}++/** Another process holding the file at that instant, as Windows reports it. */+function isSharingViolation(err: unknown): boolean {+ if (process.platform !== "win32") return false;+ const code = (err as NodeJS.ErrnoException)?.code;+ return code === "EPERM" || code === "EBUSY" || code === "EACCES";+}++const pauseCell = new Int32Array(new SharedArrayBuffer(4));++/** Block for `ms`: the rewrite is synchronous, and so are its callers. */+function pauseMs(ms: number): void {+ Atomics.wait(pauseCell, 0, 0, ms);+}diff --git a/.claude/scripts/lib/memory-supersede.ts b/.claude/scripts/lib/memory-supersede.tsindex 763b911..935894d 100644--- a/.claude/scripts/lib/memory-supersede.ts+++ b/.claude/scripts/lib/memory-supersede.ts@@ -26,9 +26,10 @@* reorders. The blast radius of a bug here is one extra line.*/-import { readFileSync, writeFileSync } from "node:fs";import { join } from "node:path";+import { rewriteIfUnchanged } from "./atomic-write.ts";+import { MEMORY_SOURCE } from "./memory-write.ts";/** The only `source` value this module is permitted to modify. */@@ -110,27 +111,39 @@ function frontmatterScalar(md: string, key: string): string | null {*/export function markSuperseded(vaultRoot: string, rel: string, supersedingTitle: string): SupersedeResult {const full = join(vaultRoot, rel);- let md: string;+ let reason = "";try {- md = readFileSync(full, "utf8");- } catch {- return { ok: false, reason: `cannot read ${rel}` };- }-- if (frontmatterScalar(md, "source") !== OWNED_BY_SERVER) {- return {- ok: false,- reason: `refusing to modify ${rel}: not an agent-written memory (source is not ${OWNED_BY_SERVER})`,- };+ const outcome = rewriteIfUnchanged(full, (current) => {+ // Re-checked on every attempt: the file may have been replaced between+ // our read and our write, and a note we no longer own is never touched.+ if (frontmatterScalar(current, "source") !== OWNED_BY_SERVER) {+ throw new SupersedeRefused(+ `refusing to modify ${rel}: not an agent-written memory (source is not ${OWNED_BY_SERVER})`,+ );+ }+ const result = addToFrontmatterList(current, "superseded_by", supersedingTitle);+ reason = result.reason;+ return result.changed ? result.text : null;+ });+ if (outcome === "contended") {+ return {+ ok: false,+ reason: `could not mark ${rel} superseded: contended, it kept changing or another writer held it`,+ };+ }+ return { ok: true, changed: outcome === "written", reason };+ } catch (err) {+ if (err instanceof SupersedeRefused) return { ok: false, reason: err.message };+ // The read and the rename both land here, so name which failure it was.+ // The code, not the message: the message carries the vault's absolute path.+ const code = (err as NodeJS.ErrnoException).code ?? (err as Error).name;+ return { ok: false, reason: `cannot read or rewrite ${rel}: ${code}` };}-- const result = addToFrontmatterList(md, "superseded_by", supersedingTitle);- if (!result.changed) return { ok: true, changed: false, reason: result.reason };-- writeFileSync(full, result.text, "utf8");- return { ok: true, changed: true, reason: result.reason };}+/** A refusal surfaced from inside the guarded rewrite. Never a write. */+class SupersedeRefused extends Error {}+/*** Resolve the memories a capture claims to supersede.*diff --git a/.claude/scripts/tests/_race-child.ts b/.claude/scripts/tests/_race-child.tsnew file mode 100644index 0000000..84ad31d--- /dev/null+++ b/.claude/scripts/tests/_race-child.ts@@ -0,0 +1,36 @@+/**+ * One concurrent writer for memory-supersede-race.test.ts.+ *+ * For each round it waits for that round's go file, spins to the start instant+ * written in it, and makes exactly ONE markSuperseded call on the round's+ * memory with its own title. No retry and no check afterwards: a lost update+ * has to show up as a missing title, not be quietly put back by the writer.+ *+ * Run as a PROCESS, not a worker thread: the bug under test lives in the gap+ * between one process's read and its rename, and only a separate process can+ * stand in another's gap.+ */++import { existsSync, readFileSync, realpathSync, writeFileSync } from "node:fs";+import { join } from "node:path";++import { writeFileAtomic } from "../lib/atomic-write.ts";+import { markSuperseded } from "../lib/memory-supersede.ts";++const [vaultRaw, syncDir, procId, roundsRaw] = process.argv.slice(2) as [string, string, string, string];+const vault = realpathSync(vaultRaw);+const rounds = Number(roundsRaw);++writeFileSync(join(syncDir, `ready-${procId}`), "1");++for (let r = 0; r < rounds; r++) {+ const go = join(syncDir, `go-${r}`);+ // Spin, not setTimeout: a timer's granularity is wider than the race window.+ while (!existsSync(go)) {}+ const round = JSON.parse(readFileSync(go, "utf8")) as { startAt: number; rel: string; titles: string[] };+ // The system clock, not performance.now(): every writer reads the same one.+ while (Date.now() < round.startAt) {}++ const result = markSuperseded(vault, round.rel, round.titles[Number(procId)]!);+ writeFileAtomic(join(syncDir, `done-${procId}-${r}`), JSON.stringify(result));+}diff --git a/.claude/scripts/tests/atomic-write.test.ts b/.claude/scripts/tests/atomic-write.test.tsindex 99c8b06..c35840e 100644--- a/.claude/scripts/tests/atomic-write.test.ts+++ b/.claude/scripts/tests/atomic-write.test.ts@@ -1,13 +1,14 @@/*** lib/atomic-write.ts writeFileAtomic: replace a file whole, or leave nothing behind.+ * rewriteIfUnchanged: edit an existing file without losing another writer's edit.*/import { test, describe, beforeEach, afterEach } from "node:test";import assert from "node:assert/strict";-import { mkdirSync, mkdtempSync, readdirSync, readFileSync, writeFileSync } from "node:fs";+import { mkdirSync, mkdtempSync, readdirSync, readFileSync, utimesSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";-import { writeFileAtomic } from "../lib/atomic-write.ts";+import { writeFileAtomic, rewriteIfUnchanged } from "../lib/atomic-write.ts";import { rmTemp } from "./_helpers.ts";describe("writeFileAtomic", () => {@@ -36,3 +37,100 @@ describe("writeFileAtomic", () => {assert.deepEqual(readdirSync(dir), ["taken"], "the temp file was removed");});});++describe("rewriteIfUnchanged", () => {+ let dir = "";+ let path = "";+ beforeEach(() => {+ dir = mkdtempSync(join(tmpdir(), "atomic-write-"));+ path = join(dir, "note.md");+ });+ afterEach(() => {+ rmTemp(dir);+ });++ test("writes when the file still holds what transform read", () => {+ writeFileSync(path, "v1", "utf8");+ const outcome = rewriteIfUnchanged(path, (current) => current + "v2");+ assert.equal(outcome, "written");+ assert.equal(readFileSync(path, "utf8"), "v1v2");+ assert.deepEqual(readdirSync(dir), ["note.md"], "no temp file left behind");+ });++ test("reports unchanged and writes nothing when there is nothing to change", () => {+ writeFileSync(path, "same", "utf8");+ assert.equal(rewriteIfUnchanged(path, () => null), "unchanged");+ assert.equal(rewriteIfUnchanged(path, (current) => current), "unchanged");+ assert.equal(readFileSync(path, "utf8"), "same");+ });++ test("a file that changes under us is retried on fresh content, both edits kept", () => {+ // The issue's contention path: a competing writer lands between our read+ // and our replace. The first attempt must NOT clobber their edit.+ writeFileSync(path, "v1", "utf8");+ let calls = 0;+ const outcome = rewriteIfUnchanged(path, (current) => {+ calls++;+ if (calls === 1) {+ // Simulate the other writer: it read the same v1 and added its line.+ writeFileSync(path, current.replace("v1", "v1\nother"), "utf8");+ }+ return current + "\nmine";+ });+ assert.equal(outcome, "written");+ assert.equal(readFileSync(path, "utf8"), "v1\nother\nmine", "the competing edit must survive");+ assert.equal(calls, 2, "the edit was replayed on the fresh content");+ assert.deepEqual(readdirSync(dir), ["note.md"], "no temp file left behind");+ });++ test("a file that changes on every attempt reports contended, unwritten", () => {+ writeFileSync(path, "base", "utf8");+ let calls = 0;+ const outcome = rewriteIfUnchanged(+ path,+ (current) => {+ calls++;+ // A writer that lands during every window: the file never matches.+ writeFileSync(path, current + `x`, "utf8");+ return current + "mine";+ },+ 3,+ );+ assert.equal(outcome, "contended");+ assert.equal(calls, 3, "gave up after the bounded attempts");+ assert.ok(!readFileSync(path, "utf8").includes("mine"), "nothing of ours was written");+ assert.deepEqual(readdirSync(dir), ["note.md"], "no temp file left behind");+ });+ test("a writer holding the lock makes the next one report contended, the lock left alone", () => {+ writeFileSync(path, "v1", "utf8");+ writeFileSync(`${path}.lock`, "", "utf8");+ let calls = 0;+ const outcome = rewriteIfUnchanged(+ path,+ (current) => {+ calls++;+ return current + "mine";+ },+ 3,+ 50,+ );+ assert.equal(outcome, "contended");+ assert.equal(calls, 0, "nothing is read for an edit before the lock is held");+ assert.equal(readFileSync(path, "utf8"), "v1");+ assert.deepEqual(+ readdirSync(dir).sort(),+ ["note.md", "note.md.lock"],+ "another writer's lock is not ours to remove",+ );+ });++ test("a lock left by a dead writer is taken over", () => {+ writeFileSync(path, "v1", "utf8");+ writeFileSync(`${path}.lock`, "", "utf8");+ const longAgo = new Date(Date.now() - 60_000);+ utimesSync(`${path}.lock`, longAgo, longAgo);+ assert.equal(rewriteIfUnchanged(path, (current) => current + "v2", 3, 50), "written");+ assert.equal(readFileSync(path, "utf8"), "v1v2");+ assert.deepEqual(readdirSync(dir), ["note.md"], "the stale lock is gone, and ours released");+ });+});diff --git a/.claude/scripts/tests/memory-supersede-race.test.ts b/.claude/scripts/tests/memory-supersede-race.test.tsnew file mode 100644index 0000000..cf1f0d8--- /dev/null+++ b/.claude/scripts/tests/memory-supersede-race.test.ts@@ -0,0 +1,97 @@+/**+ * markSuperseded under real concurrency: several PROCESSES marking the same+ * memory at the same instant, every title must land.+ *+ * This is the shape the deployment names — one server process per consuming+ * repo, all writing into one vault — and the protocol #328 measured: each+ * round, every writer makes ONE call on the same memory with its own title,+ * and afterwards every title must be in the file. The unguarded+ * read-modify-write lost titles while reporting success: 50 of 50 rounds in+ * #328's harness, and some round in every run of this one on Windows.+ *+ * No writer retries and nothing is repaired before the assertion, so a lost+ * update fails the round it happened in. A single-process test cannot stand a+ * writer in the gap between markSuperseded's read and its rename, so the+ * writers are child processes started together from a go file per round.+ */++import { test, describe } from "node:test";+import assert from "node:assert/strict";+import { spawn } from "node:child_process";+import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { dirname, join } from "node:path";+import { fileURLToPath } from "node:url";++import { rmTemp, waitFor } from "./_helpers.ts";++import { writeFileAtomic } from "../lib/atomic-write.ts";+import type { SupersedeResult } from "../lib/memory-supersede.ts";++const PROCS = 4;+const ROUNDS = 50;+// Long enough for every spinning writer to have read the go file before it fires.+const START_DELAY_MS = 30;++const MEMORY = "---\ndate: 2026-07-26\nsource: mcp-capture\nscope: general\n---\n\n# old\n\nbody\n";++describe("markSuperseded race", () => {+ test("writers marking one memory at the same instant all land their title", { timeout: 120_000 }, async () => {+ const dir = mkdtempSync(join(tmpdir(), "sup-race-"));+ const vault = join(dir, "vault");+ const sync = join(dir, "sync");+ mkdirSync(sync, { recursive: true });+ const kids: ReturnType<typeof spawn>[] = [];+ try {+ const rounds = Array.from({ length: ROUNDS }, (_, r) => {+ const rel = `memories/2026/07/2026-07-26 race ${r}.md`;+ mkdirSync(dirname(join(vault, rel)), { recursive: true });+ writeFileSync(join(vault, rel), MEMORY, "utf8");+ return { rel, titles: Array.from({ length: PROCS }, (_, p) => `Writer ${p} round ${r}`) };+ });++ const childPath = fileURLToPath(new URL("./_race-child.ts", import.meta.url));+ const exits = Array.from({ length: PROCS }, (_, p) => {+ const proc = spawn(+ process.execPath,+ ["--experimental-strip-types", childPath, vault, sync, String(p), String(ROUNDS)],+ { stdio: ["ignore", "ignore", "inherit"] },+ );+ kids.push(proc);+ return new Promise<number>((res) => proc.on("exit", (c) => res(c ?? -1)));+ });+ const done = (p: number, r: number) => join(sync, `done-${p}-${r}`);++ // Only once every writer is loaded and spinning: the start must be simultaneous.+ await waitFor(() => kids.every((_, p) => existsSync(join(sync, `ready-${p}`))), 30_000);++ const lost: { round: number; title: string; outcome: string }[] = [];+ for (const [r, round] of rounds.entries()) {+ const startAt = Date.now() + START_DELAY_MS;+ // Atomic: a writer spinning on the go file must never read it half-written.+ writeFileAtomic(join(sync, `go-${r}`), JSON.stringify({ ...round, startAt }));+ await waitFor(() => kids.every((_, p) => existsSync(done(p, r))), 30_000);++ const text = readFileSync(join(vault, round.rel), "utf8");+ for (const [p, title] of round.titles.entries()) {+ if (text.includes(JSON.stringify(title))) continue;+ // Say how it went missing: silently is #328, loudly is a refusal to explain.+ const result = JSON.parse(readFileSync(done(p, r), "utf8")) as SupersedeResult;+ lost.push({ round: r, title, outcome: result.ok ? "reported success" : result.reason });+ }+ }++ (await Promise.all(exits)).forEach((code, p) => assert.equal(code, 0, `writer ${p} crashed`));+ const lostRounds = new Set(lost.map((l) => l.round)).size;+ assert.deepEqual(+ lost,+ [],+ `${lost.length} of ${PROCS * ROUNDS} titles lost, in ${lostRounds} of ${ROUNDS} rounds: ` ++ lost.map((l) => `${l.title} (${l.outcome})`).join("; "),+ );+ } finally {+ for (const k of kids) k.kill();+ rmTemp(dir);+ }+ });+});diff --git a/.claude/scripts/tests/memory-supersede.test.ts b/.claude/scripts/tests/memory-supersede.test.tsindex 1fa92f5..8499ba3 100644--- a/.claude/scripts/tests/memory-supersede.test.ts+++ b/.claude/scripts/tests/memory-supersede.test.ts@@ -7,9 +7,10 @@* merely that a flag came back false.*/-import { test, describe } from "node:test";+import { test, describe, mock } from "node:test";import assert from "node:assert/strict";-import { mkdtempSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";+import fs, { mkdtempSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";+import { syncBuiltinESMExports } from "node:module";import { tmpdir } from "node:os";import { join, dirname } from "node:path";@@ -110,6 +111,39 @@ describe("markSuperseded", () => {}});+ test("a memory that changes on every attempt reports ok: false, unwritten", () => {+ // #328's exhaustion case: someone saves the note between each read and+ // its re-check, so no attempt's edit is safe to land. Reporting success+ // here is the lie the guarded rewrite exists to stop. The save is made+ // inside readFileSync, synced into the ESM bindings the library imported.+ const { dir, rel } = vaultWith(MD);+ const full = join(dir, rel);+ let saves = 0;+ // Captured first: once synced, this file's own readFileSync is the mock too.+ const realRead = fs.readFileSync;+ const realWrite = fs.writeFileSync;+ mock.method(fs, "readFileSync", (...args: Parameters<typeof readFileSync>) => {+ if (args[0] === full) realWrite(full, `${realRead(full, "utf8")}saved ${++saves}+`, "utf8");+ return realRead(...args);+ });+ syncBuiltinESMExports();+ try {+ const r = markSuperseded(dir, rel, "The Correction");+ assert.equal(r.ok, false);+ assert.match(r.reason, /contended/);+ mock.restoreAll();+ syncBuiltinESMExports();+ const after = readFileSync(full, "utf8");+ assert.ok(!after.includes("The Correction"), "nothing of ours was written");+ assert.ok(after.includes(`saved ${saves}`), "every save the other writer made survives");+ } finally {+ mock.restoreAll();+ syncBuiltinESMExports();+ rmTemp(dir);+ }+ });+test("a missing file returns a refusal rather than throwing", () => {const dir = mkdtempSync(join(tmpdir(), "sup-"));try {