Skip to content

A self-organizing Obsidian vault that gives AI coding agents persistent memory.

fix: guard markSuperseded against concurrent writers (#342)

akushonkamenfae7be467fc5
diff --git a/.claude/scripts/lib/atomic-write.ts b/.claude/scripts/lib/atomic-write.ts
index fa4a04f..74c5bb7 100644
--- a/.claude/scripts/lib/atomic-write.ts
+++ b/.claude/scripts/lib/atomic-write.ts
@@ -18,7 +18,17 @@
* half-written note for the indexer to pick up.
*/
-import { writeFileSync, copyFileSync, renameSync, unlinkSync, constants } from "node:fs";
+import {
+ writeFileSync,
+ readFileSync,
+ copyFileSync,
+ renameSync,
+ unlinkSync,
+ openSync,
+ closeSync,
+ statSync,
+ constants,
+} from "node:fs";
import { join } from "node:path";
/** Per-process counter, so two in-flight writes cannot share a temp name. */
@@ -100,3 +110,162 @@ export function writeFileAtomic(path: string, text: string): void {
throw err;
}
}
+
+/** What a guarded rewrite ended up doing. */
+export type RewriteResult = "written" | "unchanged" | "contended";
+
+/**
+ * Rewrite an existing file only if it is unchanged since `transform` read it.
+ *
+ * WHY NOT just read, edit, writeFileAtomic
+ *
+ * That is a read-modify-write with nothing between the read and the write that
+ * notices the file changed. Two writers each read the original, each applies
+ * its own edit, and the second rename erases the first — both reported as
+ * written. markSuperseded lost 90 of 200 titles exactly this way (#328).
+ * `writeFileAtomic` makes each replacement invisible to readers; it does
+ * nothing about the lost update between them.
+ *
+ * WHY A LOCK FILE, and not only a re-read before the rename
+ *
+ * Re-reading just before replacing shrinks the window from the whole edit to
+ * the gap between that read and the rename, but does not close it: four
+ * processes marking one memory together still lost a title in most runs of 50
+ * rounds, each reported as written. So writers going through this function
+ * first take `<file>.lock` by exclusive create, the same primitive `claimFile`
+ * relies on, and only one of them is ever between its read and its rename.
+ *
+ * The re-read stays, for writers that take no lock — a person saving the note
+ * in Obsidian. If the file changed since `transform` read it, the temp is
+ * thrown away and the edit replayed on the fresh content, a bounded number of
+ * times. `transform` returning null means there is nothing to change.
+ *
+ * A held lock lasts a few filesystem calls, so one older than STALE_LOCK_MS was
+ * left by a process that died holding it, and is taken over. A writer that
+ * cannot get the lock within `lockWaitMs` reports `contended` — told, not
+ * silently dropped.
+ *
+ * ON WINDOWS, renaming over or reading a file another process holds at that
+ * instant throws EPERM (EBUSY and EACCES are the same sharing violation by
+ * other names). An indexer or editor reading the note is enough. That is a busy
+ * file, not a broken one, so it is retried after a short pause, under its own
+ * bound since it says nothing about our read being stale. Only on Windows,
+ * where these codes are transient; on POSIX they mean a real permission
+ * problem and are thrown at once.
+ */
+export function rewriteIfUnchanged(
+ path: string,
+ transform: (current: string) => string | null,
+ attempts = 3,
+ lockWaitMs = LOCK_WAIT_MS,
+): RewriteResult {
+ const lock = `${path}.lock`;
+ if (!acquireLock(lock, lockWaitMs)) return "contended";
+ try {
+ return replaceIfUnchanged(path, transform, attempts);
+ } finally {
+ releaseLock(lock);
+ }
+}
+
+/** Longest a writer waits for another's lock before reporting `contended`. */
+const LOCK_WAIT_MS = 5_000;
+/** Age past which a lock is a dead process's leftover rather than a writer. */
+const STALE_LOCK_MS = 10_000;
+/** How many sharing violations one rewrite or release sits out before throwing. */
+const BUSY_RETRIES = 50;
+
+function replaceIfUnchanged(
+ path: string,
+ transform: (current: string) => string | null,
+ attempts: number,
+): RewriteResult {
+ let busy = 0;
+ for (let attempt = 0; attempt < attempts; ) {
+ let tmp: string | null = null;
+ try {
+ const before = readFileSync(path, "utf8");
+ const next = transform(before);
+ if (next === null || next === before) return "unchanged";
+ tmp = `${path}.${process.pid}-${++tmpSeq}.tmp`;
+ writeFileSync(tmp, next, "utf8");
+ if (readFileSync(path, "utf8") !== before) {
+ attempt++; // changed under us: retry on fresh content
+ continue;
+ }
+ renameSync(tmp, path);
+ tmp = null;
+ return "written";
+ } catch (err) {
+ if (!isSharingViolation(err) || ++busy > BUSY_RETRIES) throw err;
+ pauseMs(1 + Math.random() * 4);
+ } finally {
+ if (tmp) {
+ try {
+ unlinkSync(tmp);
+ } catch {
+ /* nothing to clean up */
+ }
+ }
+ }
+ }
+ return "contended";
+}
+
+function acquireLock(lock: string, waitMs: number): boolean {
+ const deadline = Date.now() + waitMs;
+ for (;;) {
+ try {
+ closeSync(openSync(lock, "wx"));
+ return true;
+ } catch (err) {
+ const code = (err as NodeJS.ErrnoException)?.code;
+ if (code !== "EEXIST" && !isSharingViolation(err)) throw err;
+ }
+ if (lockIsStale(lock)) {
+ try {
+ unlinkSync(lock);
+ } catch {
+ /* another writer took it over first */
+ }
+ continue;
+ }
+ if (Date.now() >= deadline) return false;
+ pauseMs(1 + Math.random() * 4); // jittered, so waiting writers stop meeting
+ }
+}
+
+function lockIsStale(lock: string): boolean {
+ try {
+ return Date.now() - statSync(lock).mtimeMs > STALE_LOCK_MS;
+ } catch {
+ return false; // gone already: the next exclusive create decides
+ }
+}
+
+function releaseLock(lock: string): void {
+ for (let busy = 0; ; busy++) {
+ try {
+ unlinkSync(lock);
+ return;
+ } catch (err) {
+ // Left in place, it would stall every writer until it went stale.
+ if (!isSharingViolation(err) || busy >= BUSY_RETRIES) return;
+ pauseMs(1 + Math.random() * 4);
+ }
+ }
+}
+
+/** Another process holding the file at that instant, as Windows reports it. */
+function isSharingViolation(err: unknown): boolean {
+ if (process.platform !== "win32") return false;
+ const code = (err as NodeJS.ErrnoException)?.code;
+ return code === "EPERM" || code === "EBUSY" || code === "EACCES";
+}
+
+const pauseCell = new Int32Array(new SharedArrayBuffer(4));
+
+/** Block for `ms`: the rewrite is synchronous, and so are its callers. */
+function pauseMs(ms: number): void {
+ Atomics.wait(pauseCell, 0, 0, ms);
+}
diff --git a/.claude/scripts/lib/memory-supersede.ts b/.claude/scripts/lib/memory-supersede.ts
index 763b911..935894d 100644
--- a/.claude/scripts/lib/memory-supersede.ts
+++ b/.claude/scripts/lib/memory-supersede.ts
@@ -26,9 +26,10 @@
* reorders. The blast radius of a bug here is one extra line.
*/
-import { readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
+import { rewriteIfUnchanged } from "./atomic-write.ts";
+
import { MEMORY_SOURCE } from "./memory-write.ts";
/** The only `source` value this module is permitted to modify. */
@@ -110,27 +111,39 @@ function frontmatterScalar(md: string, key: string): string | null {
*/
export function markSuperseded(vaultRoot: string, rel: string, supersedingTitle: string): SupersedeResult {
const full = join(vaultRoot, rel);
- let md: string;
+ let reason = "";
try {
- md = readFileSync(full, "utf8");
- } catch {
- return { ok: false, reason: `cannot read ${rel}` };
- }
-
- if (frontmatterScalar(md, "source") !== OWNED_BY_SERVER) {
- return {
- ok: false,
- reason: `refusing to modify ${rel}: not an agent-written memory (source is not ${OWNED_BY_SERVER})`,
- };
+ const outcome = rewriteIfUnchanged(full, (current) => {
+ // Re-checked on every attempt: the file may have been replaced between
+ // our read and our write, and a note we no longer own is never touched.
+ if (frontmatterScalar(current, "source") !== OWNED_BY_SERVER) {
+ throw new SupersedeRefused(
+ `refusing to modify ${rel}: not an agent-written memory (source is not ${OWNED_BY_SERVER})`,
+ );
+ }
+ const result = addToFrontmatterList(current, "superseded_by", supersedingTitle);
+ reason = result.reason;
+ return result.changed ? result.text : null;
+ });
+ if (outcome === "contended") {
+ return {
+ ok: false,
+ reason: `could not mark ${rel} superseded: contended, it kept changing or another writer held it`,
+ };
+ }
+ return { ok: true, changed: outcome === "written", reason };
+ } catch (err) {
+ if (err instanceof SupersedeRefused) return { ok: false, reason: err.message };
+ // The read and the rename both land here, so name which failure it was.
+ // The code, not the message: the message carries the vault's absolute path.
+ const code = (err as NodeJS.ErrnoException).code ?? (err as Error).name;
+ return { ok: false, reason: `cannot read or rewrite ${rel}: ${code}` };
}
-
- const result = addToFrontmatterList(md, "superseded_by", supersedingTitle);
- if (!result.changed) return { ok: true, changed: false, reason: result.reason };
-
- writeFileSync(full, result.text, "utf8");
- return { ok: true, changed: true, reason: result.reason };
}
+/** A refusal surfaced from inside the guarded rewrite. Never a write. */
+class SupersedeRefused extends Error {}
+
/**
* Resolve the memories a capture claims to supersede.
*
diff --git a/.claude/scripts/tests/_race-child.ts b/.claude/scripts/tests/_race-child.ts
new file mode 100644
index 0000000..84ad31d
--- /dev/null
+++ b/.claude/scripts/tests/_race-child.ts
@@ -0,0 +1,36 @@
+/**
+ * One concurrent writer for memory-supersede-race.test.ts.
+ *
+ * For each round it waits for that round's go file, spins to the start instant
+ * written in it, and makes exactly ONE markSuperseded call on the round's
+ * memory with its own title. No retry and no check afterwards: a lost update
+ * has to show up as a missing title, not be quietly put back by the writer.
+ *
+ * Run as a PROCESS, not a worker thread: the bug under test lives in the gap
+ * between one process's read and its rename, and only a separate process can
+ * stand in another's gap.
+ */
+
+import { existsSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { writeFileAtomic } from "../lib/atomic-write.ts";
+import { markSuperseded } from "../lib/memory-supersede.ts";
+
+const [vaultRaw, syncDir, procId, roundsRaw] = process.argv.slice(2) as [string, string, string, string];
+const vault = realpathSync(vaultRaw);
+const rounds = Number(roundsRaw);
+
+writeFileSync(join(syncDir, `ready-${procId}`), "1");
+
+for (let r = 0; r < rounds; r++) {
+ const go = join(syncDir, `go-${r}`);
+ // Spin, not setTimeout: a timer's granularity is wider than the race window.
+ while (!existsSync(go)) {}
+ const round = JSON.parse(readFileSync(go, "utf8")) as { startAt: number; rel: string; titles: string[] };
+ // The system clock, not performance.now(): every writer reads the same one.
+ while (Date.now() < round.startAt) {}
+
+ const result = markSuperseded(vault, round.rel, round.titles[Number(procId)]!);
+ writeFileAtomic(join(syncDir, `done-${procId}-${r}`), JSON.stringify(result));
+}
diff --git a/.claude/scripts/tests/atomic-write.test.ts b/.claude/scripts/tests/atomic-write.test.ts
index 99c8b06..c35840e 100644
--- a/.claude/scripts/tests/atomic-write.test.ts
+++ b/.claude/scripts/tests/atomic-write.test.ts
@@ -1,13 +1,14 @@
/**
* lib/atomic-write.ts writeFileAtomic: replace a file whole, or leave nothing behind.
+ * rewriteIfUnchanged: edit an existing file without losing another writer's edit.
*/
import { test, describe, beforeEach, afterEach } from "node:test";
import assert from "node:assert/strict";
-import { mkdirSync, mkdtempSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { mkdirSync, mkdtempSync, readdirSync, readFileSync, utimesSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
-import { writeFileAtomic } from "../lib/atomic-write.ts";
+import { writeFileAtomic, rewriteIfUnchanged } from "../lib/atomic-write.ts";
import { rmTemp } from "./_helpers.ts";
describe("writeFileAtomic", () => {
@@ -36,3 +37,100 @@ describe("writeFileAtomic", () => {
assert.deepEqual(readdirSync(dir), ["taken"], "the temp file was removed");
});
});
+
+describe("rewriteIfUnchanged", () => {
+ let dir = "";
+ let path = "";
+ beforeEach(() => {
+ dir = mkdtempSync(join(tmpdir(), "atomic-write-"));
+ path = join(dir, "note.md");
+ });
+ afterEach(() => {
+ rmTemp(dir);
+ });
+
+ test("writes when the file still holds what transform read", () => {
+ writeFileSync(path, "v1", "utf8");
+ const outcome = rewriteIfUnchanged(path, (current) => current + "v2");
+ assert.equal(outcome, "written");
+ assert.equal(readFileSync(path, "utf8"), "v1v2");
+ assert.deepEqual(readdirSync(dir), ["note.md"], "no temp file left behind");
+ });
+
+ test("reports unchanged and writes nothing when there is nothing to change", () => {
+ writeFileSync(path, "same", "utf8");
+ assert.equal(rewriteIfUnchanged(path, () => null), "unchanged");
+ assert.equal(rewriteIfUnchanged(path, (current) => current), "unchanged");
+ assert.equal(readFileSync(path, "utf8"), "same");
+ });
+
+ test("a file that changes under us is retried on fresh content, both edits kept", () => {
+ // The issue's contention path: a competing writer lands between our read
+ // and our replace. The first attempt must NOT clobber their edit.
+ writeFileSync(path, "v1", "utf8");
+ let calls = 0;
+ const outcome = rewriteIfUnchanged(path, (current) => {
+ calls++;
+ if (calls === 1) {
+ // Simulate the other writer: it read the same v1 and added its line.
+ writeFileSync(path, current.replace("v1", "v1\nother"), "utf8");
+ }
+ return current + "\nmine";
+ });
+ assert.equal(outcome, "written");
+ assert.equal(readFileSync(path, "utf8"), "v1\nother\nmine", "the competing edit must survive");
+ assert.equal(calls, 2, "the edit was replayed on the fresh content");
+ assert.deepEqual(readdirSync(dir), ["note.md"], "no temp file left behind");
+ });
+
+ test("a file that changes on every attempt reports contended, unwritten", () => {
+ writeFileSync(path, "base", "utf8");
+ let calls = 0;
+ const outcome = rewriteIfUnchanged(
+ path,
+ (current) => {
+ calls++;
+ // A writer that lands during every window: the file never matches.
+ writeFileSync(path, current + `x`, "utf8");
+ return current + "mine";
+ },
+ 3,
+ );
+ assert.equal(outcome, "contended");
+ assert.equal(calls, 3, "gave up after the bounded attempts");
+ assert.ok(!readFileSync(path, "utf8").includes("mine"), "nothing of ours was written");
+ assert.deepEqual(readdirSync(dir), ["note.md"], "no temp file left behind");
+ });
+ test("a writer holding the lock makes the next one report contended, the lock left alone", () => {
+ writeFileSync(path, "v1", "utf8");
+ writeFileSync(`${path}.lock`, "", "utf8");
+ let calls = 0;
+ const outcome = rewriteIfUnchanged(
+ path,
+ (current) => {
+ calls++;
+ return current + "mine";
+ },
+ 3,
+ 50,
+ );
+ assert.equal(outcome, "contended");
+ assert.equal(calls, 0, "nothing is read for an edit before the lock is held");
+ assert.equal(readFileSync(path, "utf8"), "v1");
+ assert.deepEqual(
+ readdirSync(dir).sort(),
+ ["note.md", "note.md.lock"],
+ "another writer's lock is not ours to remove",
+ );
+ });
+
+ test("a lock left by a dead writer is taken over", () => {
+ writeFileSync(path, "v1", "utf8");
+ writeFileSync(`${path}.lock`, "", "utf8");
+ const longAgo = new Date(Date.now() - 60_000);
+ utimesSync(`${path}.lock`, longAgo, longAgo);
+ assert.equal(rewriteIfUnchanged(path, (current) => current + "v2", 3, 50), "written");
+ assert.equal(readFileSync(path, "utf8"), "v1v2");
+ assert.deepEqual(readdirSync(dir), ["note.md"], "the stale lock is gone, and ours released");
+ });
+});
diff --git a/.claude/scripts/tests/memory-supersede-race.test.ts b/.claude/scripts/tests/memory-supersede-race.test.ts
new file mode 100644
index 0000000..cf1f0d8
--- /dev/null
+++ b/.claude/scripts/tests/memory-supersede-race.test.ts
@@ -0,0 +1,97 @@
+/**
+ * markSuperseded under real concurrency: several PROCESSES marking the same
+ * memory at the same instant, every title must land.
+ *
+ * This is the shape the deployment names — one server process per consuming
+ * repo, all writing into one vault — and the protocol #328 measured: each
+ * round, every writer makes ONE call on the same memory with its own title,
+ * and afterwards every title must be in the file. The unguarded
+ * read-modify-write lost titles while reporting success: 50 of 50 rounds in
+ * #328's harness, and some round in every run of this one on Windows.
+ *
+ * No writer retries and nothing is repaired before the assertion, so a lost
+ * update fails the round it happened in. A single-process test cannot stand a
+ * writer in the gap between markSuperseded's read and its rename, so the
+ * writers are child processes started together from a go file per round.
+ */
+
+import { test, describe } from "node:test";
+import assert from "node:assert/strict";
+import { spawn } from "node:child_process";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { rmTemp, waitFor } from "./_helpers.ts";
+
+import { writeFileAtomic } from "../lib/atomic-write.ts";
+import type { SupersedeResult } from "../lib/memory-supersede.ts";
+
+const PROCS = 4;
+const ROUNDS = 50;
+// Long enough for every spinning writer to have read the go file before it fires.
+const START_DELAY_MS = 30;
+
+const MEMORY = "---\ndate: 2026-07-26\nsource: mcp-capture\nscope: general\n---\n\n# old\n\nbody\n";
+
+describe("markSuperseded race", () => {
+ test("writers marking one memory at the same instant all land their title", { timeout: 120_000 }, async () => {
+ const dir = mkdtempSync(join(tmpdir(), "sup-race-"));
+ const vault = join(dir, "vault");
+ const sync = join(dir, "sync");
+ mkdirSync(sync, { recursive: true });
+ const kids: ReturnType<typeof spawn>[] = [];
+ try {
+ const rounds = Array.from({ length: ROUNDS }, (_, r) => {
+ const rel = `memories/2026/07/2026-07-26 race ${r}.md`;
+ mkdirSync(dirname(join(vault, rel)), { recursive: true });
+ writeFileSync(join(vault, rel), MEMORY, "utf8");
+ return { rel, titles: Array.from({ length: PROCS }, (_, p) => `Writer ${p} round ${r}`) };
+ });
+
+ const childPath = fileURLToPath(new URL("./_race-child.ts", import.meta.url));
+ const exits = Array.from({ length: PROCS }, (_, p) => {
+ const proc = spawn(
+ process.execPath,
+ ["--experimental-strip-types", childPath, vault, sync, String(p), String(ROUNDS)],
+ { stdio: ["ignore", "ignore", "inherit"] },
+ );
+ kids.push(proc);
+ return new Promise<number>((res) => proc.on("exit", (c) => res(c ?? -1)));
+ });
+ const done = (p: number, r: number) => join(sync, `done-${p}-${r}`);
+
+ // Only once every writer is loaded and spinning: the start must be simultaneous.
+ await waitFor(() => kids.every((_, p) => existsSync(join(sync, `ready-${p}`))), 30_000);
+
+ const lost: { round: number; title: string; outcome: string }[] = [];
+ for (const [r, round] of rounds.entries()) {
+ const startAt = Date.now() + START_DELAY_MS;
+ // Atomic: a writer spinning on the go file must never read it half-written.
+ writeFileAtomic(join(sync, `go-${r}`), JSON.stringify({ ...round, startAt }));
+ await waitFor(() => kids.every((_, p) => existsSync(done(p, r))), 30_000);
+
+ const text = readFileSync(join(vault, round.rel), "utf8");
+ for (const [p, title] of round.titles.entries()) {
+ if (text.includes(JSON.stringify(title))) continue;
+ // Say how it went missing: silently is #328, loudly is a refusal to explain.
+ const result = JSON.parse(readFileSync(done(p, r), "utf8")) as SupersedeResult;
+ lost.push({ round: r, title, outcome: result.ok ? "reported success" : result.reason });
+ }
+ }
+
+ (await Promise.all(exits)).forEach((code, p) => assert.equal(code, 0, `writer ${p} crashed`));
+ const lostRounds = new Set(lost.map((l) => l.round)).size;
+ assert.deepEqual(
+ lost,
+ [],
+ `${lost.length} of ${PROCS * ROUNDS} titles lost, in ${lostRounds} of ${ROUNDS} rounds: ` +
+ lost.map((l) => `${l.title} (${l.outcome})`).join("; "),
+ );
+ } finally {
+ for (const k of kids) k.kill();
+ rmTemp(dir);
+ }
+ });
+});
diff --git a/.claude/scripts/tests/memory-supersede.test.ts b/.claude/scripts/tests/memory-supersede.test.ts
index 1fa92f5..8499ba3 100644
--- a/.claude/scripts/tests/memory-supersede.test.ts
+++ b/.claude/scripts/tests/memory-supersede.test.ts
@@ -7,9 +7,10 @@
* merely that a flag came back false.
*/
-import { test, describe } from "node:test";
+import { test, describe, mock } from "node:test";
import assert from "node:assert/strict";
-import { mkdtempSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import fs, { mkdtempSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import { syncBuiltinESMExports } from "node:module";
import { tmpdir } from "node:os";
import { join, dirname } from "node:path";
@@ -110,6 +111,39 @@ describe("markSuperseded", () => {
}
});
+ test("a memory that changes on every attempt reports ok: false, unwritten", () => {
+ // #328's exhaustion case: someone saves the note between each read and
+ // its re-check, so no attempt's edit is safe to land. Reporting success
+ // here is the lie the guarded rewrite exists to stop. The save is made
+ // inside readFileSync, synced into the ESM bindings the library imported.
+ const { dir, rel } = vaultWith(MD);
+ const full = join(dir, rel);
+ let saves = 0;
+ // Captured first: once synced, this file's own readFileSync is the mock too.
+ const realRead = fs.readFileSync;
+ const realWrite = fs.writeFileSync;
+ mock.method(fs, "readFileSync", (...args: Parameters<typeof readFileSync>) => {
+ if (args[0] === full) realWrite(full, `${realRead(full, "utf8")}saved ${++saves}
+`, "utf8");
+ return realRead(...args);
+ });
+ syncBuiltinESMExports();
+ try {
+ const r = markSuperseded(dir, rel, "The Correction");
+ assert.equal(r.ok, false);
+ assert.match(r.reason, /contended/);
+ mock.restoreAll();
+ syncBuiltinESMExports();
+ const after = readFileSync(full, "utf8");
+ assert.ok(!after.includes("The Correction"), "nothing of ours was written");
+ assert.ok(after.includes(`saved ${saves}`), "every save the other writer made survives");
+ } finally {
+ mock.restoreAll();
+ syncBuiltinESMExports();
+ rmTemp(dir);
+ }
+ });
+
test("a missing file returns a refusal rather than throwing", () => {
const dir = mkdtempSync(join(tmpdir(), "sup-"));
try {