Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

fix(land): fence stalled owners and failed guardians [Q1,Q3]

Corey Olson163fb13d77dd
diff --git a/src/land.js b/src/land.js
index e30ba1f2..cca86a75 100644
--- a/src/land.js
+++ b/src/land.js
@@ -690,7 +690,7 @@ export async function land(options) {
adopt: options.adopt, fence: lease.fence };
if (batchOptions.maximum === null) delete batchOptions.maximum;
return await runLandingWorker(batchOptions, { write, checkLease, checkLeaseNow: lease.assertCurrent,
- bindWorker: lease.bindWorker, onWait: options.onWait, say: options.say, signal: controller.signal });
+ bindWorker: lease.bindWorker, watchDeadline: lease.watchDeadline, onWait: options.onWait, say: options.say, signal: controller.signal });
} catch (error) { failed = true; throw error; }
finally {
try { await lease.release(); }
diff --git a/src/landing-resource.js b/src/landing-resource.js
index 8a9cd604..5b66c8d9 100644
--- a/src/landing-resource.js
+++ b/src/landing-resource.js
@@ -26,8 +26,14 @@ export async function takeLandingResource({ board, write, branch, onWait, onLost
let timer;
let watchdog;
let deadline = 0;
+ const deadlineListeners = new Set();
let pending = Promise.resolve();
let lost = null;
+ /** Send only acknowledged deadlines to the independent guardian. */
+ const confirmedUntil = until => {
+ deadline = Date.parse(until);
+ for (const listener of deadlineListeners) listener(deadline);
+ };
/** Serialize heartbeats and explicit checks, retaining a failure until the command stops. */
const assertHeld = () => {
const task = pending.then(async () => {
@@ -35,8 +41,8 @@ export async function takeLandingResource({ board, write, branch, onWait, onLost
try {
const result = await write('landingLeaseMove', [{ ...parameters, action: 'renew', generation }]);
if (lost) throw lost;
- assertLandingLease(board, { token, branch, generation }); // A delayed recovered receipt is not a fresh grant.
- deadline = Date.parse(result.holder.until);
+ const holder = assertLandingLease(board, { token, branch, generation }); // A delayed recovered receipt is not a fresh grant.
+ confirmedUntil(holder.until);
return result;
}
catch (error) { lost = error; onLost?.(error); throw error; }
@@ -49,7 +55,7 @@ export async function takeLandingResource({ board, write, branch, onWait, onLost
const holder = landingLeaseState(board).holder;
if (deadLocalOwner(holder)) await write('landingLeaseMove', [{ agentId: 'coordinator', action: 'release', token: holder.token, generation: holder.generation }]);
const result = await write('landingLeaseMove', [{ ...parameters, action: 'request' }]);
- if (result.acquired) { generation = result.generation; deadline = Date.parse(result.holder.until); break; }
+ if (result.acquired) { generation = result.generation; confirmedUntil(result.holder.until); break; }
await onWait?.({ holders: result.holder ? [result.holder] : [], position: result.position });
await pause(1000);
}
@@ -67,10 +73,12 @@ export async function takeLandingResource({ board, write, branch, onWait, onLost
async bindWorker(guardianPid) {
await assertHeld();
if (lost) throw lost;
- const result = await write('landingLeaseMove', [{ ...parameters, action: 'bind', generation, guardianPid }]);
+ await write('landingLeaseMove', [{ ...parameters, action: 'bind', generation, guardianPid }]);
if (lost) throw lost;
- deadline = Date.parse(result.holder.until);
+ confirmedUntil(assertLandingLease(board, { token, branch, generation }).until);
},
+ /** Keep the guardian's own deadline current even if this parent later stops responding. */
+ watchDeadline(listener) { deadlineListeners.add(listener); listener(deadline); return () => deadlineListeners.delete(listener); },
/** Check the current wall-time fence without adding an await before a physical action. */
assertCurrent() { return assertLandingLease(board, { token, branch, generation }); },
/** End renewals before releasing; a stale generation never clears its successor. */
diff --git a/src/landing-worker.js b/src/landing-worker.js
index 23a7293a..b9ab8fbd 100644
--- a/src/landing-worker.js
+++ b/src/landing-worker.js
@@ -1,5 +1,5 @@
/** Isolate blocking Git and gates so the coordinator keeps renewing and fencing its board lease [Q1,Q3]. */
-import { spawn, execFileSync } from 'node:child_process';
+import { spawn, spawnSync } from 'node:child_process';
import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
import * as store from './board.js';
@@ -12,38 +12,54 @@ const FILE = fileURLToPath(import.meta.url);
/** Preserve one typed refusal prefix while passing it through more than one process. */
function messageText(error) { return String(error.message).replace(/^\[[A-Z][A-Z0-9_]*\] /u, ''); }
-/** Stop only the process group this invocation created; never signal another landing's processes. */
-function stop(child) {
- if (!child.pid) return;
- // Sharded gates start their own groups; collect them while their parent ancestry is still present.
+/** Read process identities before attributing a live worker or its detached gate descendants. */
+function census() {
try {
- const rows = parseProcessCensus(execFileSync('/bin/ps', ['-axo', 'pid=,ppid=,pgid=,lstart=,stat=,comm='], { encoding: 'utf8', env: { ...process.env, LC_ALL: 'C', TZ: 'UTC' } }));
- if (rows) {
- const owned = new Set([child.pid]);
- let changed;
- do {
- changed = false;
- for (const row of rows.values()) if (owned.has(row.ppid) && !owned.has(row.pid)) { owned.add(row.pid); changed = true; }
- } while (changed);
- for (const pid of [...owned].reverse()) if (pid !== child.pid) {
- try { process.kill(pid, 'SIGKILL'); } catch (error) { if (error.code !== 'ESRCH') throw error; }
- }
- }
- } catch { /* The worker's own group is still known; never broaden a failed census to the machine. */ }
+ const result = spawnSync('/bin/ps', ['-axo', 'pid=,ppid=,pgid=,lstart=,stat=,comm='], { encoding: 'utf8', env: { ...process.env, LC_ALL: 'C', TZ: 'UTC' } });
+ if (result.status !== 0) return null;
+ const rows = parseProcessCensus(result.stdout);
+ rows?.delete(result.pid); // This already-exited observer is never a cleanup target.
+ return rows;
+ } catch { return null; }
+}
+
+/** Stop a currently owned group; a recorded worker identity permits cleanup after guardian death without trusting a reused PID. */
+function stop(child, member) {
+ if (!child.pid) return;
+ const rows = census();
+ const root = rows?.get(child.pid);
+ const known = member && rows?.get(member.pid);
+ const ownsRoot = root && (child.pid === process.pid || root.ppid === process.pid && child.exitCode === null && child.signalCode === null);
+ const ownsMember = known && known.pgid === child.pid && known.started === member.started && known.processSha256 === member.processSha256;
+ if (!ownsRoot && !ownsMember) return;
+ // Shards have separate groups; traverse actual descendant ancestry before stopping the known group.
+ const owned = new Set([ownsRoot ? child.pid : member.pid]);
+ let changed;
+ do {
+ changed = false;
+ for (const row of rows.values()) if (owned.has(row.ppid) && !owned.has(row.pid)) { owned.add(row.pid); changed = true; }
+ } while (changed);
+ const current = census();
+ for (const pid of [...owned].reverse()) if (pid !== child.pid && current?.get(pid)?.started === rows.get(pid)?.started
+ && current.get(pid)?.processSha256 === rows.get(pid)?.processSha256) {
+ try { process.kill(pid, 'SIGKILL'); } catch (error) { if (error.code !== 'ESRCH') throw error; }
+ }
try { process.kill(-child.pid, 'SIGKILL'); }
catch (error) { if (error.code !== 'ESRCH') throw error; }
}
/** Run blocking batch work under a live parent that owns all canonical writes and lease renewal. */
-export async function runLandingWorker(options, { write, checkLease, checkLeaseNow, bindWorker, onWait, say, signal }) {
+export async function runLandingWorker(options, { write, checkLease, checkLeaseNow, bindWorker, watchDeadline, onWait, say, signal }) {
const child = spawn(process.execPath, [FILE, 'guardian'], { cwd: options.root, env: process.env, detached: true, stdio: ['ignore', 'ignore', 'pipe', 'ipc'] });
const inFlight = new Set();
let result;
let failure;
let stderr = '';
+ let member;
+ const unwatch = watchDeadline(until => { if (child.connected) child.send({ type: 'deadline', until }, () => {}); });
child.stderr.setEncoding('utf8').on('data', part => { stderr = (stderr + part).slice(-65536); });
/** A failed authority check stops the whole owned group before another command can adopt its worktree. */
- const abort = () => { failure ??= signal.reason; if (child.exitCode === null && child.signalCode === null) stop(child); };
+ const abort = () => { failure ??= signal.reason; stop(child, member); };
signal.addEventListener('abort', abort, { once: true });
/** Reply only over this owned child's channel, preserving the original typed refusal. */
const answer = (id, value, error) => {
@@ -51,13 +67,27 @@ export async function runLandingWorker(options, { write, checkLease, checkLeaseN
child.send({ type: 'answer', id, value, error: error ? { code: error.code ?? 'LAND_WORKER', message: messageText(error) } : null }, () => {});
};
child.on('message', message => {
+ if (message.type === 'worker') {
+ if (signal.aborted || failure) { stop(child, member); return; }
+ try { checkLeaseNow(); }
+ catch (error) { failure = error; stop(child, member); return; }
+ const row = census()?.get(message.pid);
+ if (!row || row.ppid !== child.pid || row.pgid !== child.pid) {
+ failure = new Refused('LAND_WORKER', 'the landing worker identity could not be confirmed; inspect pullboard resume before adopting');
+ stop(child);
+ return;
+ }
+ member = row;
+ child.send({ type: 'worker-start', options: { ...options, guardianPid: child.pid } }, () => {});
+ return;
+ }
if (message.type === 'say') { say(message.text); return; }
if (message.type === 'result') { result = message.value; return; }
if (message.type === 'failure') { failure = new Refused(message.code, message.message); return; }
if (message.type !== 'call') return;
/** Permit only the batch worker's narrow protocol, retaining all authority in this parent. */
const act = async () => {
- if (signal.aborted) throw signal.reason;
+ if (signal.aborted || failure) throw signal.reason ?? failure;
if (message.operation === 'check') { await checkLease(); checkLeaseNow(); return null; }
if (message.operation === 'wait') return await onWait?.(message.args[0]);
if (!['recordLandingBatch', 'finishLandingBatch', 'recordLandingFlake', 'recordFlakeObservation', 'recordGateRun', 'reopen', 'appendFact'].includes(message.operation)) {
@@ -72,6 +102,12 @@ export async function runLandingWorker(options, { write, checkLease, checkLeaseN
try {
const terminal = new Promise(resolveClose => {
child.once('error', error => { failure = error; });
+ child.once('exit', (code, terminatedBy) => {
+ if (code !== 0 || terminatedBy || result === undefined) {
+ failure ??= new Refused('LAND_WORKER', 'the landing guardian stopped unexpectedly; inspect pullboard resume before adopting');
+ stop(child, member);
+ }
+ });
child.once('close', (code, terminatedBy) => resolveClose({ code, signal: terminatedBy }));
});
await new Promise((resolveSpawn, reject) => { child.once('spawn', resolveSpawn); child.once('error', reject); });
@@ -84,13 +120,14 @@ export async function runLandingWorker(options, { write, checkLease, checkLeaseN
return result;
} finally {
signal.removeEventListener('abort', abort);
- if (child.exitCode === null && child.signalCode === null) stop(child);
+ unwatch();
+ if (child.exitCode === null && child.signalCode === null) stop(child, member);
await Promise.allSettled(inFlight);
}
}
/** Make one checked request to the live parent without giving the worker independent board-write authority. */
-function channel() {
+function channel(group) {
let next = 0;
let finished = false;
const waiting = new Map();
@@ -102,7 +139,7 @@ function channel() {
if (message.error) pair.reject(new Refused(message.error.code, message.error.message));
else pair.resolve(message.value);
});
- process.once('disconnect', () => { if (!finished) { try { process.kill(-process.pid, 'SIGKILL'); } catch { process.exit(1); } } });
+ process.once('disconnect', () => { if (!finished) { try { process.kill(-group, 'SIGKILL'); } catch { process.exit(1); } } });
const call = (operation, args = []) => new Promise((resolve, reject) => {
const id = ++next;
waiting.set(id, { resolve, reject });
@@ -113,7 +150,7 @@ function channel() {
/** Enter batch execution only through the parent's private process channel. */
async function workerMain(options) {
- const pipe = channel();
+ const pipe = channel(options.guardianPid);
const { call } = pipe;
const board = store.openBoard(join(repoInfo(options.root).commonDir, 'pullboard', 'board.sqlite'));
try {
@@ -143,13 +180,28 @@ function guardianMain() {
let finished = false;
let closed = false;
let pending = 0;
+ let deadline = 0;
+ let expiring = false;
+ /** Fail closed independently of a suspended CLI, using only its last acknowledged authority deadline. */
+ const expire = () => {
+ if (finished || expiring) return;
+ expiring = true;
+ if (process.connected) process.send({ type: 'failure', code: 'LAND_LEASE_LOST', message: 'the guardian stopped its owned worker before the acknowledged landing lease expired' }, () => stop({ pid: process.pid }));
+ else stop({ pid: process.pid });
+ };
+ const watchdog = setInterval(() => { if (deadline && Date.now() >= deadline - 1500) expire(); }, 100);
/** Flush result and refusal messages before ending this private channel normally. */
const finish = () => {
if (!closed || pending) return;
finished = true;
+ clearInterval(watchdog);
if (process.connected) process.disconnect();
};
process.on('message', message => {
+ if (message.type === 'deadline') {
+ if (Number.isFinite(message.until)) deadline = message.until;
+ return;
+ }
if (message.type === 'start' && !worker) {
// The worker stays in this guardian's detached group; its gates inherit that group too.
worker = spawn(process.execPath, [FILE, 'worker'], { cwd: message.options.root, env: process.env,
@@ -167,7 +219,10 @@ function guardianMain() {
process.exitCode = code ?? (signal ? 1 : 0);
finish();
});
- worker.send(message);
+ worker.once('spawn', () => process.send({ type: 'worker', pid: worker.pid }));
+ } else if (message.type === 'worker-start' && worker?.connected) {
+ if (!deadline || Date.now() >= deadline - 1500) { expire(); return; }
+ worker.send({ type: 'start', options: message.options });
} else if (worker?.connected) worker.send(message, () => {});
});
process.once('disconnect', () => { if (!finished) stop({ pid: process.pid }); });
diff --git a/test/landing-queue-fixture.js b/test/landing-queue-fixture.js
index 6850c481..ba7abcd0 100644
--- a/test/landing-queue-fixture.js
+++ b/test/landing-queue-fixture.js
@@ -56,7 +56,7 @@ function acceptedPair(helpers, box, staleAfterFirst) {
}
/** Prove named waiting, exclusion and verdict order with two real overlapping land commands. */
-export async function landingQueueCase(t, helpers, { staleAfterFirst = false, killParent = false } = {}) {
+export async function landingQueueCase(t, helpers, { staleAfterFirst = false, killParent = false, killGuardian = false, suspendParent = false } = {}) {
const sandbox = helpers.sandbox();
const arm = join(sandbox.dir, 'landing-armed');
const entered = join(sandbox.dir, 'landing-entered');
@@ -66,7 +66,9 @@ export async function landingQueueCase(t, helpers, { staleAfterFirst = false, ki
writeFileSync(gateFile, [
"const fs = require('node:fs'); const assert = require('node:assert/strict');",
`const [arm,entered,release,gatePid] = ${JSON.stringify([arm, entered, release, gatePid])};`,
- "if (fs.existsSync(arm)) { fs.writeFileSync(gatePid, String(process.pid)); fs.writeFileSync(entered, process.cwd()); const end = Date.now() + 20000; while (!fs.existsSync(release) && Date.now() < end) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,20); assert.ok(fs.existsSync(release), 'owning test releases the gate'); }",
+ ...(suspendParent ? ["if (fs.existsSync(gatePid)) { const old = Number(fs.readFileSync(gatePid,'utf8')); try { process.kill(old,0); throw new Error('the previous landing gate is still running when the successor gate starts'); } catch(error) { if (error.code !== 'ESRCH') throw error; } }"] : []),
+ // Budget: the suspended-parent control holds beyond the real 30s lease; 65s is a runaway ceiling, not a speed target.
+ `if (fs.existsSync(arm)) { fs.writeFileSync(gatePid, String(process.pid)); fs.writeFileSync(entered, process.cwd()); const end = Date.now() + ${suspendParent ? 65000 : 20000}; while (!fs.existsSync(release) && Date.now() < end) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,20); assert.ok(fs.existsSync(release), 'owning test releases the gate'); }`,
"const files = fs.existsSync('web') ? fs.readdirSync('web').filter(file => /^queue-\\d+\\.txt$/.test(file)) : [];",
"for (const file of files) assert.match(fs.readFileSync('web/' + file,'utf8'), /^works/);",
"console.log('TAP version 13\\nok 1 - landed queue files keep their behavior\\n1..1\\n# tests 1\\n# pass 1\\n# fail 0');",
@@ -82,7 +84,7 @@ export async function landingQueueCase(t, helpers, { staleAfterFirst = false, ki
t.after(async () => {
writeFileSync(release, 'released');
for (const child of children) {
- if (child.exitCode === null && child.signalCode === null) child.kill('SIGTERM');
+ if (child.exitCode === null && child.signalCode === null) { child.kill('SIGCONT'); child.kill('SIGTERM'); }
await child.closed;
}
});
@@ -95,7 +97,8 @@ export async function landingQueueCase(t, helpers, { staleAfterFirst = false, ki
const firstBatch = readBoard(box, board => store.landingBatches(board).at(-1));
assert.equal(firstBatch.state, 'gating');
assert.deepEqual(firstBatch.items.map(entry => entry.id), [entries[1].id], 'verdict order precedes numeric item order');
- const second = helpers.launch(box, box.repo, process.execPath, [helpers.BIN, 'land', '--max', '1']);
+ if (suspendParent) rmSync(arm); // Only later gates become unblocked; the current one still requires its unreleased marker.
+ const second = helpers.launch(box, box.repo, process.execPath, [helpers.BIN, 'land', ...(suspendParent ? ['--adopt'] : ['--max', '1'])]);
children.push(second);
await helpers.waitFor(() => /landing waiting: .*held by .*; queued/u.test(second.stdoutText) || second.exitCode !== null,
'second landing names its holder and queues');
@@ -109,12 +112,29 @@ export async function landingQueueCase(t, helpers, { staleAfterFirst = false, ki
await helpers.waitFor(() => Date.parse(readBoard(box, board => store.landingLeaseState(board).holder?.until)) > Date.parse(holder.until),
'the independent parent renews while the worker is blocked inside its real synchronous gate');
assert.equal(readBoard(box, board => store.landingBatches(board).at(-1).state), 'gating');
- if (killParent) {
+ if (killParent || killGuardian || suspendParent) {
const blockedGatePid = Number(readFileSync(gatePid, 'utf8'));
assert.ok(Number.isSafeInteger(holder.guardianPid));
- first.kill('SIGKILL'); // Kill only the CLI, leaving its separately grouped guardian responsible for the blocked gate.
- assert.equal((await first.closed).signal, 'SIGKILL');
- await helpers.waitFor(() => !alive(holder.guardianPid) && !alive(blockedGatePid), 'the guardian and blocked gate stop without releasing their marker');
+ if (killParent) {
+ first.kill('SIGKILL'); // Only the CLI dies; its separately grouped guardian owns the blocked gate.
+ assert.equal((await first.closed).signal, 'SIGKILL');
+ } else if (killGuardian) {
+ process.kill(holder.guardianPid, 'SIGKILL'); // This exact PID came from this private command's bound guardian.
+ } else first.kill('SIGSTOP'); // Keep IPC connected while the authority owner cannot renew or run its own watchdog.
+ await helpers.waitFor(() => !alive(holder.guardianPid) && !alive(blockedGatePid), 'the guardian and blocked gate stop without releasing their marker',
+ suspendParent ? 45_000 : 8_000); // Budget: actual 30s lease expiry plus owned-process drain; not a speed threshold.
+ if (suspendParent) {
+ const successor = await second.closed;
+ assert.equal(successor.code, 0, successor.failure ?? 'the queued adopter lands only after the previous gate is dead');
+ assert.equal(existsSync(release), false, 'no test release made the old gate look drained');
+ assert.equal(readBoard(box, board => store.landingBatches(board).at(-1).state), 'landed');
+ first.kill('SIGCONT');
+ assert.equal((await first.closed).code, 1, 'the resumed old holder cannot use its expired generation');
+ run(box, box.repo, 'land', '--max', '1');
+ assert.equal(readBoard(box, board => store.landingBatches(board).length), 2);
+ return;
+ }
+ if (!killParent) assert.equal((await first.closed).code, 1, 'a surviving authority owner refuses after its worker stops');
assert.equal(existsSync(release), false, 'the owning test did not let the abandoned gate proceed');
const waiting = await second.closed;
assert.equal(waiting.code, 1);
diff --git a/test/team.test.js b/test/team.test.js
index ef2612e2..f2bd33bb 100644
--- a/test/team.test.js
+++ b/test/team.test.js
@@ -40,6 +40,12 @@ test('a queued landing rechecks freshness after its holder moves main [E8,Q1,Q2]
test('an abandoned landing drains its blocked gate before adoption [Q1,Q3,B3]', { timeout: 120_000 }, // budget: private CLI death, owned-process cleanup, interrupted-batch adoption and Git; no speed target.
async t => landingQueueCase(t, helpers, { killParent: true }));
+test('a dead landing guardian cannot leave its blocked worker running [Q1,Q3,B3]', { timeout: 120_000 }, // budget: private guardian death, identity-bound cleanup and explicit adoption; no speed target.
+ async t => landingQueueCase(t, helpers, { killGuardian: true }));
+
+test('a suspended landing owner is fenced before the queued adopter runs [Q1,Q3]', { timeout: 120_000 }, // budget: real 30s authority expiry, independent guardian watchdog, cleanup and adoption; no speed target.
+ async t => landingQueueCase(t, helpers, { suspendParent: true }));
+
/** Open a fixture's independent board and always close it after its observation or public move. */
function withBoard(root, action) {
const board = store.openBoard(join(root, '.git', 'pullboard', 'board.sqlite'));