Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

fix(relay): a dead reaper's reap lock is cleared, never restored [H5]

Corey Olson8ab3037d669b
diff --git a/src/relay-lock.js b/src/relay-lock.js
index 39b01ac7..6cd5af7a 100644
--- a/src/relay-lock.js
+++ b/src/relay-lock.js
@@ -1,9 +1,11 @@
/** The per-board link lock: one worktree sends at a time, and a sibling waits instead of failing (#919). */
+import { randomUUID } from 'node:crypto';
import {
lstatSync,
mkdirSync,
readFileSync,
readdirSync,
+ renameSync,
rmSync,
writeFileSync,
} from 'node:fs';
@@ -31,36 +33,71 @@ function gone(pid) {
catch (error) { return error.code === 'ESRCH'; }
}
+/** Whether this process holds the reap lock right now. */
+function holdsReaper(reaper) {
+ try { return ownerOf(reaper) === process.pid; } catch { return false; }
+}
+
+/**
+ * Take the reap lock, or clear one whose reaper died. A reaper publishes the lock with its pid already inside, by
+ * renaming its own directory into place, so the lock always names a live or dead owner and is never taken twice.
+ * A dead reaper's lock is moved aside in one step and deleted (#1119).
+ *
+ * @returns {boolean} Whether this process now holds the reap lock.
+ */
+function takeReaper(reaper) {
+ const mine = `${reaper}.${process.pid}.${randomUUID()}`;
+ mkdirSync(mine, { mode: 0o700 });
+ writeFileSync(join(mine, 'pid'), String(process.pid));
+ try { renameSync(mine, reaper); return true; }
+ catch (error) {
+ rmSync(mine, { recursive: true, force: true });
+ if (!['EEXIST', 'ENOTEMPTY', 'EPERM'].includes(error.code)) throw error;
+ }
+ let holder;
+ try { holder = ownerOf(reaper); } catch { return false; }
+ if (holder === null || !gone(holder)) return false;
+ // Move it aside and delete it. If a live reaper took the path between our look and our move, it is the one
+ // moved: it finds it no longer holds the reap lock and does nothing. It is never put back, since by then it may
+ // have finished and would hold the path for ever.
+ const aside = `${reaper}.dead.${process.pid}.${randomUUID()}`;
+ try { renameSync(reaper, aside); } catch { return false; }
+ rmSync(aside, { recursive: true, force: true });
+ return false;
+}
+
/**
* Remove a lock whose owner is gone, as the only waiter allowed to (#1119). Waiters that found the same dead owner
- * compete for one short reap lock; the winner checks again that the lock still has that dead owner, or still none
- * after the stale age, and only then removes it. Nothing else removes another's lock, so the check cannot be
- * overtaken: a live owner's lock is never removed, and the lock it frees is then won by exactly one mkdir.
+ * compete for one short reap lock; the holder checks again that the lock still has that dead owner, or still none
+ * after the stale age, and that it still holds the reap lock, and only then removes it. Nothing else removes
+ * another's lock, so a live owner's lock is never removed, and the lock it frees is then won by exactly one mkdir.
*
* @returns {boolean} Whether this waiter cleared the lock.
*/
function reap(lock, seenOwner) {
const reaper = lock + '.reap';
- try { mkdirSync(reaper, { mode: 0o700 }); }
- catch (error) {
- if (error.code !== 'EEXIST') throw error;
- // A reaper holds this only for a check and a removal; one this old died doing it.
- try { if (Date.now() - lstatSync(reaper).mtimeMs > UNOWNED_STALE_MS) rmSync(reaper, { recursive: true, force: true }); }
- catch (gone) { if (gone.code !== 'ENOENT') throw gone; }
- return false;
- }
+ if (!takeReaper(reaper)) return false;
try {
let owner;
try { owner = ownerOf(lock); } catch (error) { if (error.code === 'ENOENT') return false; owner = null; }
const dead = seenOwner === null
? owner === null && Date.now() - lstatSync(lock).mtimeMs > UNOWNED_STALE_MS
: owner === seenOwner && gone(owner);
- if (dead) rmSync(lock, { recursive: true, force: true });
- return dead;
+ if (!dead || !holdsReaper(reaper)) return false;
+ rmSync(lock, { recursive: true, force: true });
+ return true;
} catch (error) {
if (error.code === 'ENOENT') return false;
throw error;
- } finally { rmSync(reaper, { recursive: true, force: true }); }
+ } finally {
+ // Move the reap lock aside in one step before deleting it: emptied in place, its path would briefly accept
+ // another reaper's rename, which the deletion would then remove.
+ if (holdsReaper(reaper)) {
+ const done = `${reaper}.done.${process.pid}.${randomUUID()}`;
+ try { renameSync(reaper, done); } catch (error) { if (error.code !== 'ENOENT') throw error; }
+ rmSync(done, { recursive: true, force: true });
+ }
+ }
}
/**
diff --git a/test/relay-lock.test.js b/test/relay-lock.test.js
index 8e5edcdf..a855dbe3 100644
--- a/test/relay-lock.test.js
+++ b/test/relay-lock.test.js
@@ -3,14 +3,19 @@ import assert from 'node:assert/strict';
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
-import { spawn, spawnSync } from 'node:child_process';
+import { spawn } from 'node:child_process';
import { test } from 'node:test';
const LOCK_MODULE = new URL('../src/relay-lock.js', import.meta.url).href;
-/** A process number that belonged to a process that has already exited. */
+/**
+ * A process number no process can hold: far above any system's limit. Not one that just exited, nor a free one in the
+ * usual range: the system may hand either to one of the waiters, and the lock's owner would then look alive.
+ */
function deadPid() {
- return spawnSync(process.execPath, ['-e', '']).pid;
+ const pid = 2_147_483_000;
+ try { process.kill(pid, 0); } catch (error) { if (error.code === 'ESRCH') return pid; throw error; }
+ throw new Error('the dead owner\'s process number is in use');
}
/** One waiter: take the lock, note entry and exit in the shared log, and hold it briefly. */
@@ -25,29 +30,43 @@ await locked(file, async () => {
});
`;
+/** Run six waiters against a dead owner's lock, optionally with a dead reaper's reap lock too, and check one holder. */
+async function round(root, index, { deadReaper = false } = {}) {
+ const file = join(root, `relay-${index}.json`);
+ const log = join(root, `log-${index}`);
+ writeFileSync(log, '');
+ // The lock a crashed command left behind: its owner's process is gone.
+ mkdirSync(`${file}.lock`);
+ writeFileSync(join(`${file}.lock`, 'pid'), String(deadPid()));
+ if (deadReaper) {
+ // And the reap lock of a waiter that died while clearing it.
+ mkdirSync(`${file}.lock.reap`);
+ writeFileSync(join(`${file}.lock.reap`, 'pid'), String(deadPid()));
+ }
+ const waiters = Array.from({ length: 6 }, () => new Promise((done, fail) => {
+ const child = spawn(process.execPath, ['--input-type=module', '-e', WAITER, LOCK_MODULE, log, file], { stdio: ['ignore', 'ignore', 'pipe'] });
+ let stderr = '';
+ child.stderr.on('data', part => { stderr += part; });
+ child.once('exit', code => (code === 0 ? done() : fail(new Error(stderr))));
+ }));
+ await Promise.all(waiters);
+ let holders = 0;
+ const lines = readFileSync(log, 'utf8').trim().split('\n');
+ for (const line of lines) {
+ holders += line.startsWith('enter') ? 1 : -1;
+ assert.ok(holders <= 1, `round ${index}: two waiters held the lock at once:\n${lines.join('\n')}`);
+ }
+ assert.equal(lines.filter(line => line.startsWith('enter')).length, 6, `round ${index}: every waiter held the lock once`);
+}
+
test('waiters on a dead owner leave exactly one holder at a time [H5]', async (t) => {
const root = mkdtempSync(join(tmpdir(), 'pullboard-relay-lock-'));
t.after(() => rmSync(root, { recursive: true, force: true }));
- for (let round = 0; round < 15; round += 1) {
- const file = join(root, `relay-${round}.json`);
- const log = join(root, `log-${round}`);
- writeFileSync(log, '');
- // The lock a crashed command left behind: its owner's process is gone.
- mkdirSync(`${file}.lock`);
- writeFileSync(join(`${file}.lock`, 'pid'), String(deadPid()));
- const waiters = Array.from({ length: 6 }, () => new Promise((done, fail) => {
- const child = spawn(process.execPath, ['--input-type=module', '-e', WAITER, LOCK_MODULE, log, file], { stdio: ['ignore', 'ignore', 'pipe'] });
- let stderr = '';
- child.stderr.on('data', part => { stderr += part; });
- child.once('exit', code => (code === 0 ? done() : fail(new Error(stderr))));
- }));
- await Promise.all(waiters);
- let holders = 0;
- const lines = readFileSync(log, 'utf8').trim().split('\n');
- for (const line of lines) {
- holders += line.startsWith('enter') ? 1 : -1;
- assert.ok(holders <= 1, `round ${round}: two waiters held the lock at once:\n${lines.join('\n')}`);
- }
- assert.equal(lines.filter(line => line.startsWith('enter')).length, 6, `round ${round}: every waiter held the lock once`);
- }
+ for (let index = 0; index < 15; index += 1) await round(root, index);
+});
+
+test('a reaper that died clearing a lock is recovered, and still one waiter holds the lock at a time [H5]', async (t) => {
+ const root = mkdtempSync(join(tmpdir(), 'pullboard-relay-lock-reaper-'));
+ t.after(() => rmSync(root, { recursive: true, force: true }));
+ for (let index = 0; index < 10; index += 1) await round(root, index, { deadReaper: true });
});