Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

fix(relay): the relay lock is a SQLite lease taken in one write [H5]

Corey Olsonf68b30123a78
diff --git a/src/relay-lock.js b/src/relay-lock.js
index 6cd5af7a..83be21e1 100644
--- a/src/relay-lock.js
+++ b/src/relay-lock.js
@@ -5,10 +5,10 @@ import {
mkdirSync,
readFileSync,
readdirSync,
- renameSync,
rmSync,
writeFileSync,
} from 'node:fs';
+import { DatabaseSync } from 'node:sqlite';
import {
dirname,
join,
@@ -33,73 +33,56 @@ function gone(pid) {
catch (error) { return error.code === 'ESRCH'; }
}
-/** Whether this process holds the reap lock right now. */
-function holdsReaper(reaper) {
- try { return ownerOf(reaper) === process.pid; } catch { return false; }
-}
-
/**
- * Take the reap lock, or clear one whose reaper died. A reaper publishes the lock with its pid already inside, by
- * renaming its own directory into place, so the lock always names a live or dead owner and is never taken twice.
- * A dead reaper's lock is moved aside in one step and deleted (#1119).
- *
- * @returns {boolean} Whether this process now holds the reap lock.
+ * The lease that decides which command holds a board's relay lock (#1119). Taking it is one SQLite write
+ * transaction that checks the holder and replaces it only when that holder's process is gone, so of any number of
+ * waiters exactly one holds it, and a live holder is never displaced: there is no separate removal step to race.
*/
-function takeReaper(reaper) {
- const mine = `${reaper}.${process.pid}.${randomUUID()}`;
- mkdirSync(mine, { mode: 0o700 });
- writeFileSync(join(mine, 'pid'), String(process.pid));
- try { renameSync(mine, reaper); return true; }
- catch (error) {
- rmSync(mine, { recursive: true, force: true });
- if (!['EEXIST', 'ENOTEMPTY', 'EPERM'].includes(error.code)) throw error;
- }
- let holder;
- try { holder = ownerOf(reaper); } catch { return false; }
- if (holder === null || !gone(holder)) return false;
- // Move it aside and delete it. If a live reaper took the path between our look and our move, it is the one
- // moved: it finds it no longer holds the reap lock and does nothing. It is never put back, since by then it may
- // have finished and would hold the path for ever.
- const aside = `${reaper}.dead.${process.pid}.${randomUUID()}`;
- try { renameSync(reaper, aside); } catch { return false; }
- rmSync(aside, { recursive: true, force: true });
- return false;
+function openLease(file) {
+ const db = new DatabaseSync(`${file}.lease.sqlite`);
+ db.exec(`PRAGMA busy_timeout = 10000;
+ CREATE TABLE IF NOT EXISTS relay_lease(name TEXT PRIMARY KEY, pid INTEGER NOT NULL, token TEXT NOT NULL);`);
+ return db;
}
-/**
- * Remove a lock whose owner is gone, as the only waiter allowed to (#1119). Waiters that found the same dead owner
- * compete for one short reap lock; the holder checks again that the lock still has that dead owner, or still none
- * after the stale age, and that it still holds the reap lock, and only then removes it. Nothing else removes
- * another's lock, so a live owner's lock is never removed, and the lock it frees is then won by exactly one mkdir.
- *
- * @returns {boolean} Whether this waiter cleared the lock.
- */
-function reap(lock, seenOwner) {
- const reaper = lock + '.reap';
- if (!takeReaper(reaper)) return false;
+/** Take the lease for this process when it is free or its holder has died; true when this process holds it. */
+function takeLease(db, token) {
+ db.exec('BEGIN IMMEDIATE');
try {
- let owner;
- try { owner = ownerOf(lock); } catch (error) { if (error.code === 'ENOENT') return false; owner = null; }
- const dead = seenOwner === null
- ? owner === null && Date.now() - lstatSync(lock).mtimeMs > UNOWNED_STALE_MS
- : owner === seenOwner && gone(owner);
- if (!dead || !holdsReaper(reaper)) return false;
- rmSync(lock, { recursive: true, force: true });
- return true;
+ const holder = db.prepare("SELECT pid, token FROM relay_lease WHERE name = 'relay'").get();
+ const free = !holder || holder.token === token || gone(holder.pid);
+ if (free) db.prepare("INSERT OR REPLACE INTO relay_lease(name, pid, token) VALUES ('relay', ?, ?)").run(process.pid, token);
+ db.exec('COMMIT');
+ return free;
} catch (error) {
- if (error.code === 'ENOENT') return false;
+ db.exec('ROLLBACK');
throw error;
- } finally {
- // Move the reap lock aside in one step before deleting it: emptied in place, its path would briefly accept
- // another reaper's rename, which the deletion would then remove.
- if (holdsReaper(reaper)) {
- const done = `${reaper}.done.${process.pid}.${randomUUID()}`;
- try { renameSync(reaper, done); } catch (error) { if (error.code !== 'ENOENT') throw error; }
- rmSync(done, { recursive: true, force: true });
- }
}
}
+/** Give the lease up, only while it is still this process's own. */
+function releaseLease(db, token) {
+ db.prepare("DELETE FROM relay_lease WHERE name = 'relay' AND token = ?").run(token);
+}
+
+/**
+ * Take the directory lock older pullboard versions still use, as the lease holder. Only the lease holder clears a
+ * dead owner's directory, so no two current commands race on it.
+ *
+ * @returns {boolean} Whether this process now holds the directory lock too.
+ */
+function takeDirectory(lock) {
+ try { mkdirSync(lock, { mode: 0o700 }); writeFileSync(join(lock, 'pid'), String(process.pid)); return true; }
+ catch (error) { if (error.code !== 'EEXIST') throw error; }
+ let owner = null;
+ try { owner = ownerOf(lock); } catch (error) { if (error.code === 'ENOENT') return false; }
+ let stale;
+ try { stale = owner !== null ? gone(owner) : Date.now() - lstatSync(lock).mtimeMs > UNOWNED_STALE_MS; }
+ catch (error) { if (error.code === 'ENOENT') return false; throw error; }
+ if (stale) rmSync(lock, { recursive: true, force: true });
+ return false;
+}
+
/**
* Wait for another worktree's upload, recovering a lock only when its owner is gone.
* Why: one large baseline PUT can hold the lock past any short limit, and a board move that gave up
@@ -112,22 +95,10 @@ export async function locked(file, work, io) {
const deadline = startedAt + (io?.relayLockWaitMs ?? LOCK_WAIT_MS);
const progressEvery = io?.relayLockProgressMs ?? LOCK_PROGRESS_MS;
let nextProgress = startedAt + progressEvery;
- while (true) {
- try { mkdirSync(lock, { mode: 0o700 }); writeFileSync(join(lock, 'pid'), String(process.pid)); break; }
- catch (error) {
- if (error.code !== 'EEXIST') throw error;
- let owner;
- try { owner = ownerOf(lock); }
- catch (unread) {
- if (unread.code === 'ENOENT') {
- try { lstatSync(lock); } catch (missing) { if (missing.code === 'ENOENT') continue; throw missing; }
- }
- owner = null;
- }
- let stale;
- try { stale = owner !== null ? gone(owner) : Date.now() - lstatSync(lock).mtimeMs > UNOWNED_STALE_MS; }
- catch (vanished) { if (vanished.code === 'ENOENT') continue; throw vanished; }
- if (stale && reap(lock, owner)) continue;
+ const lease = openLease(file);
+ const token = randomUUID();
+ try {
+ while (!(takeLease(lease, token) && takeDirectory(lock))) {
if (Date.now() >= deadline) throw new Refused('RELAY_BUSY', `another worktree was still sending this board after ${Math.round((Date.now() - startedAt) / 1000)} s; nothing was lost, so run the command again`);
if (io?.err && Date.now() >= nextProgress) {
nextProgress = Date.now() + progressEvery;
@@ -135,17 +106,20 @@ export async function locked(file, work, io) {
}
await new Promise((resolve) => setTimeout(resolve, 100));
}
- }
- try {
- // Only the current lock holder can have a live link write; remove crashed legacy bearer fragments.
- for (const name of readdirSync(dirname(file))) {
- if (/^relay\.json\.[0-9a-f-]+\.tmp$/u.test(name)) rmSync(join(dirname(file), name), { force: true });
+ try {
+ // Only the current lock holder can have a live link write; remove crashed legacy bearer fragments.
+ for (const name of readdirSync(dirname(file))) {
+ if (/^relay\.json\.[0-9a-f-]+\.tmp$/u.test(name)) rmSync(join(dirname(file), name), { force: true });
+ }
+ return await work();
+ } finally {
+ // Remove only this command's own directory lock; another's is never ours to free.
+ let owner = null;
+ try { owner = ownerOf(lock); } catch { /* Already gone, or never named: nothing of ours to keep. */ }
+ if (owner === process.pid) rmSync(lock, { recursive: true, force: true });
}
- return await work();
} finally {
- // Remove only this command's own lock; another's is never ours to free.
- let owner = null;
- try { owner = ownerOf(lock); } catch { /* Already gone, or never named: nothing of ours to keep. */ }
- if (owner === null || owner === process.pid) rmSync(lock, { recursive: true, force: true });
+ releaseLease(lease, token);
+ lease.close();
}
}
diff --git a/test/relay-lock.test.js b/test/relay-lock.test.js
index a855dbe3..5d4c546a 100644
--- a/test/relay-lock.test.js
+++ b/test/relay-lock.test.js
@@ -4,6 +4,7 @@ import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'nod
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { spawn } from 'node:child_process';
+import { DatabaseSync } from 'node:sqlite';
import { test } from 'node:test';
const LOCK_MODULE = new URL('../src/relay-lock.js', import.meta.url).href;
@@ -30,18 +31,20 @@ await locked(file, async () => {
});
`;
-/** Run six waiters against a dead owner's lock, optionally with a dead reaper's reap lock too, and check one holder. */
-async function round(root, index, { deadReaper = false } = {}) {
+/** Run six waiters against a dead owner's lock, optionally with a dead holder's lease too, and check one holder. */
+async function round(root, index, { deadLease = false } = {}) {
const file = join(root, `relay-${index}.json`);
const log = join(root, `log-${index}`);
writeFileSync(log, '');
// The lock a crashed command left behind: its owner's process is gone.
mkdirSync(`${file}.lock`);
writeFileSync(join(`${file}.lock`, 'pid'), String(deadPid()));
- if (deadReaper) {
- // And the reap lock of a waiter that died while clearing it.
- mkdirSync(`${file}.lock.reap`);
- writeFileSync(join(`${file}.lock.reap`, 'pid'), String(deadPid()));
+ if (deadLease) {
+ // And the lease of a command that died holding it.
+ const db = new DatabaseSync(`${file}.lease.sqlite`);
+ db.exec('CREATE TABLE IF NOT EXISTS relay_lease(name TEXT PRIMARY KEY, pid INTEGER NOT NULL, token TEXT NOT NULL)');
+ db.prepare("INSERT INTO relay_lease(name, pid, token) VALUES ('relay', ?, 'dead')").run(deadPid());
+ db.close();
}
const waiters = Array.from({ length: 6 }, () => new Promise((done, fail) => {
const child = spawn(process.execPath, ['--input-type=module', '-e', WAITER, LOCK_MODULE, log, file], { stdio: ['ignore', 'ignore', 'pipe'] });
@@ -65,8 +68,8 @@ test('waiters on a dead owner leave exactly one holder at a time [H5]', async (t
for (let index = 0; index < 15; index += 1) await round(root, index);
});
-test('a reaper that died clearing a lock is recovered, and still one waiter holds the lock at a time [H5]', async (t) => {
- const root = mkdtempSync(join(tmpdir(), 'pullboard-relay-lock-reaper-'));
+test('a lease left by a dead holder is taken over by one waiter at a time [H5]', async (t) => {
+ const root = mkdtempSync(join(tmpdir(), 'pullboard-relay-lock-lease-'));
t.after(() => rmSync(root, { recursive: true, force: true }));
- for (let index = 0; index < 10; index += 1) await round(root, index, { deadReaper: true });
+ for (let index = 0; index < 10; index += 1) await round(root, index, { deadLease: true });
});