Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

fix(runner): a job's detached processes end with it [C7]

Corey Olsonbf753756be73
diff --git a/src/gate.js b/src/gate.js
index ead1127d..948a2985 100644
--- a/src/gate.js
+++ b/src/gate.js
@@ -566,11 +566,13 @@ export async function withGateSlot(root, action, { landing = false, itemCheck =
timed(lease).tracker.mark('actionAt');
try { return await action(lease); }
catch (error) {
- if (!(error instanceof RunnerDropped) || !lease.runner || !lease.requeueRunner) throw error;
- const runnerUnavailable = { host: lease.runner.host, reason: 'transport lost before a complete result', until: 'the host is checked and re-registered' };
+ // A cleanup the runner could not verify is the runner's failure, not the run's: rerun it locally (#1063).
+ const cleanup = error instanceof Refused && error.code === 'RUNNER_CLEANUP_TIMEOUT';
+ if (!(error instanceof RunnerDropped || cleanup) || !lease.runner || !lease.requeueRunner) throw error;
+ const runnerUnavailable = { host: lease.runner.host, reason: cleanup ? error.message : 'transport lost before a complete result', until: 'the host is checked and re-registered' };
if (onWait) await onWait({ holders: [], line: [], position: 0, runnerUnavailable });
- else process.stderr.write(`runner ${runnerUnavailable.host} unavailable: ${runnerUnavailable.reason}; until ${runnerUnavailable.until}; requeueing the run\n`);
- lease = await lease.requeueRunner('transport-lost');
+ else process.stderr.write(`runner ${runnerUnavailable.host} unavailable: ${runnerUnavailable.reason}; until ${runnerUnavailable.until}; ${cleanup ? 'rerunning it locally' : 'requeueing the run'}\n`);
+ lease = await lease.requeueRunner(cleanup ? 'cleanup-unverified' : 'transport-lost', { local: cleanup });
}
}
} finally {
diff --git a/src/process-census.js b/src/process-census.js
index a3df00ec..2c919c94 100644
--- a/src/process-census.js
+++ b/src/process-census.js
@@ -28,3 +28,22 @@ export function parseProcessCensus(stdout) {
}
return rows.size ? rows : null;
}
+
+const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
+
+/**
+ * Whether a census row's process started at the given moment, from a census read with TZ=UTC. The census gives
+ * whole seconds, and a process starts at most a moment before the record of it, so the window is one second before
+ * that second to three after, for a loaded host. Hosts hand out pids in order, so a reused pid cannot start inside it.
+ *
+ * @param {{started: string}} row - A row from parseProcessCensus.
+ * @param {number} ms - When the job's own record says the process started, in epoch milliseconds.
+ * @returns {boolean}
+ */
+export function bornNear(row, ms) {
+ const match = /^[A-Z][a-z]{2} ([A-Z][a-z]{2}) (\d{1,2}) (\d{2}):(\d{2}):(\d{2}) (\d{4})$/u.exec(row?.started ?? '');
+ const month = match ? MONTHS.indexOf(match[1]) : -1;
+ if (month < 0 || !Number.isFinite(ms)) return false;
+ const second = Date.UTC(Number(match[6]), month, Number(match[2]), Number(match[3]), Number(match[4]), Number(match[5]));
+ return second - 1_000 <= ms && ms < second + 3_000;
+}
diff --git a/src/resources.js b/src/resources.js
index 75f54ac2..307856cd 100644
--- a/src/resources.js
+++ b/src/resources.js
@@ -600,7 +600,7 @@ function pause(ms) { return new Promise((resolvePromise) => setTimeout(resolvePr
* memory, when given, is read before each admission: a held verdict keeps the whole queue
* waiting in place, local, remote and short slots alike, and an unchecked one is passed to onWait once
* so the caller can warn.
- * @returns {Promise<{ name: string, scope: string, token: string, pool: string, browser: boolean, slotClass: 'gate'|'short', runner?: {host:string,slots:number,browser:number}, waitMs: number, release: () => void, renew: () => void, requeueRunner?: () => Promise<object> }>}
+ * @returns {Promise<{ name: string, scope: string, token: string, pool: string, browser: boolean, slotClass: 'gate'|'short', runner?: {host:string,slots:number,browser:number}, waitMs: number, release: () => void, renew: () => void, requeueRunner?: (reason?: string, options?: { local?: boolean }) => Promise<object> }>}
*/
export async function takeResource(options) {
const { name, capacity, scope = 'machine', root = process.cwd(), allowIdleCapacityUpdate = false } = options;
@@ -788,8 +788,8 @@ export async function takeResource(options) {
throw new Refused('RESOURCE_LEASE_LOST', `resource lease for "${name}" is no longer held`);
}
};
- /** Quarantine a failed remote host and retry this exact ticket on another executor. */
- const requeueRunner = async () => {
+ /** Quarantine a failed remote host and retry this exact ticket on another executor, or only locally. */
+ const requeueRunner = async (reason = 'transport-lost', { local = false } = {}) => {
if (!acquired.runner) throw new Refused('LOCAL_RUNNER_REQUEUE', 'only a remote lease can be returned to the runner queue');
if (released || transferred || lost) throw new Refused('RESOURCE_LEASE_LOST', `resource lease for "${name}" is no longer held`);
transferred = true;
@@ -799,7 +799,7 @@ export async function takeResource(options) {
try {
update(db, () => {
db.prepare('INSERT INTO runner_unavailable(host, since, detail) VALUES (?, ?, ?) ON CONFLICT(host) DO UPDATE SET since = excluded.since, detail = excluded.detail')
- .run(acquired.runner.host, new Date().toISOString(), 'TRANSPORT_LOST');
+ .run(acquired.runner.host, new Date().toISOString(), reason === 'cleanup-unverified' ? 'CLEANUP_UNVERIFIED' : 'TRANSPORT_LOST');
const holderName = runnerResourceName(acquired.runner.host);
const holder = db.prepare('SELECT token FROM holder WHERE token = ? AND name = ? AND runner = ?').get(token, holderName, acquired.runner.host);
if (!holder) throw new Refused('RESOURCE_LEASE_LOST', `remote lease for "${name}" is no longer held`);
@@ -820,7 +820,7 @@ export async function takeResource(options) {
}
db.prepare('DELETE FROM holder WHERE token = ?').run(token);
db.prepare('INSERT INTO waiter(ticket, token, name, pid, started, agent, repo, since, heartbeat, landing, item_check, diagnostic, browser, item, kind, remote, classed, release_aware, board_key, release_rank, queue_ticket, short_eligible) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')
- .run(requeueTicket, token, name, pid, started, agent, repo, queuedAt, Date.now(), Number(landing), Number(itemCheck || diagnostic), Number(diagnostic), Number(browser), item, kind, Number(remote), marker, 1, boardKey, releaseRank, requeueOpportunity, Number(shortEligible));
+ .run(requeueTicket, token, name, pid, started, agent, repo, queuedAt, Date.now(), Number(landing), Number(itemCheck || diagnostic), Number(diagnostic), Number(browser), item, kind, Number(remote && !local), marker, 1, boardKey, releaseRank, requeueOpportunity, Number(shortEligible));
});
} catch (error) {
transferred = false;
@@ -828,7 +828,7 @@ export async function takeResource(options) {
throw error;
}
db.close();
- return takeResource({ ...options, _resume: { token, ticket: requeueTicket ?? acquired.ticket, queueTicket: requeueOpportunity,
+ return takeResource({ ...options, ...(local ? { remote: false } : {}), _resume: { token, ticket: requeueTicket ?? acquired.ticket, queueTicket: requeueOpportunity,
queuedAt, requestedAt, priorWaitMs: acquiredWaitMs, waitStartedAt: Date.now() } });
};
return { name, scope, token, pool, browser, slotClass: acquired.shortSlot ? 'short' : 'gate', ...(acquired.runner ? { runner: acquired.runner } : {}),
diff --git a/src/runner-worker.js b/src/runner-worker.js
index 083898fc..e0d7b279 100644
--- a/src/runner-worker.js
+++ b/src/runner-worker.js
@@ -37,6 +37,7 @@ import {
pathToFileURL,
} from 'node:url';
import * as resourceApi from './resources.js';
+import { bornNear } from './process-census.js';
import { parseProcessCensus } from './process-census.js';
import {
gateStages,
@@ -340,7 +341,7 @@ function record(value) {
overflow = true;
}
}
-record({ type: 'boot', pid: process.pid, ppid: process.ppid, instance });
+record({ type: 'boot', pid: process.pid, ppid: process.ppid, instance, started: Math.round(performance.timeOrigin) });
const channel = diagnostics.channel('child_process');
const seen = new WeakSet();
channel.subscribe(message => {
@@ -351,7 +352,7 @@ channel.subscribe(message => {
const writeReceipt = () => {
if (!Number.isSafeInteger(child.pid) || child.pid < 1) return;
const commandSha256 = createHash('sha256').update(JSON.stringify({ command: child.spawnfile ?? '', args: child.spawnargs ?? [] })).digest('hex');
- record({ type: 'spawn', pid: child.pid, ppid: process.pid, instance, commandSha256 });
+ record({ type: 'spawn', pid: child.pid, ppid: process.pid, instance, commandSha256, at: Date.now() });
};
if (Number.isSafeInteger(child.pid)) writeReceipt();
else child.once('spawn', writeReceipt);
@@ -384,6 +385,8 @@ function createDescendantTracker(directory) {
const offsets = new Map();
const partial = new Map();
const instancePids = new Map();
+ // Each job Node process's own boot record: its pid and start time, written into this job's private journal.
+ const boots = new Map();
const overflowPath = journal.overflowPath;
let current = baseline;
let uncertain = false;
@@ -402,6 +405,20 @@ function createDescendantTracker(directory) {
reason ||= message;
}
+ /**
+ * Take custody of a live process proven to be this job's, and of the group it runs in. A group lives inside one
+ * session, and a session holds only descendants of the job process that made it, so that group is the job's too.
+ * A group led by a process that was here before the job never becomes the job's (#1063).
+ */
+ function adopt(row, commandSha256, ownedPids) {
+ known.set(row.pid, { ...row, commandSha256, root: false, group: row.pgid });
+ const leader = current.get(row.pgid);
+ const foreign = leader && baseline.get(row.pgid)?.birth === leader.birth;
+ if (!detachedGroups.has(row.pgid) && !foreign) detachedGroups.set(row.pgid, leader?.birth ?? '');
+ pendingIdentities.delete(row.pid);
+ ownedPids.add(row.pid);
+ }
+
function readJournal() {
if (existsSync(overflowPath)) { markUncertain('child spawn journal overflow or write failure'); }
let total = 0;
@@ -439,6 +456,8 @@ function createDescendantTracker(directory) {
try { record = JSON.parse(line); } catch { markUncertain('child spawn journal record is invalid'); continue; }
if (record.type === 'boot' && Number.isSafeInteger(record.pid) && record.pid > 0 && typeof record.instance === 'string') {
instancePids.set(record.instance, record.pid);
+ // A boot record without its start time still binds receipts; it only proves nothing by itself.
+ if (Number.isFinite(record.started)) boots.set(record.pid, [...(boots.get(record.pid) ?? []), record.started].slice(-4));
} else if (record.type === 'spawn' && Number.isSafeInteger(record.pid) && record.pid > 0
&& Number.isSafeInteger(record.ppid) && record.ppid > 0
&& typeof record.instance === 'string' && /^[a-f0-9]{64}$/u.test(record.commandSha256)) {
@@ -491,8 +510,14 @@ function createDescendantTracker(directory) {
if (retired.has(`${pid}:${row.birth}`)) { markUncertain('a retired process identity reappeared'); continue; }
if (!launched) rootLaunchIdentities.set(pid, row);
known.set(pid, { ...row, commandSha256: prior?.commandSha256 ?? '', root: true, group: pid });
+ if (!detachedGroups.has(pid)) detachedGroups.set(pid, row.birth);
ownedPids.add(pid);
}
+ // Every identity the job already proved stays owned while it lives, even after its parent exits (#1063).
+ for (const [pid, identity] of known) {
+ const row = current.get(pid);
+ if (row && row.birth === identity.birth && !/^Z/u.test(row.state) && !retired.has(`${pid}:${row.birth}`)) ownedPids.add(pid);
+ }
for (const row of current.values()) {
if (/^Z/u.test(row.state)) continue;
@@ -506,6 +531,13 @@ function createDescendantTracker(directory) {
if (retired.has(`${row.pid}:${row.birth}`)) { markUncertain('a retired process identity reappeared'); continue; }
if (prior && prior.birth === row.birth) { consumedEvents.add(event.eventId); ownedPids.add(row.pid); continue; }
if (!prior && baseline.get(row.pid)?.birth === row.birth) { consumedEvents.add(event.eventId); continue; }
+ // The receipt and the census both carry a start time. A process born when a job process spawned it is that
+ // child, even if its parent exited before any census saw the two together; a reused pid starts later (#1063).
+ if ((row.ppid !== event.ppid || !ownedPids.has(event.ppid)) && bornNear(row, event.at)) {
+ adopt(row, event.commandSha256, ownedPids);
+ consumedEvents.add(event.eventId);
+ continue;
+ }
// A spawn receipt must be observed while its exact parent instance still owns it.
// A same-numbered PID first seen after reparenting is ambiguous and never adopted.
if (row.ppid !== event.ppid) {
@@ -539,6 +571,16 @@ function createDescendantTracker(directory) {
}
}
+ // A Node process that wrote its own boot record into this job's private journal is the job's, once the
+ // census shows it born when it said it was (#1063).
+ for (const [pid, starts] of boots) {
+ const row = current.get(pid);
+ if (!row || /^Z/u.test(row.state) || ownedPids.has(pid) || retired.has(`${pid}:${row.birth}`)) continue;
+ if (baseline.get(pid)?.birth === row.birth) continue;
+ if (known.get(pid)?.birth === row.birth) ownedPids.add(pid);
+ else if (starts.some(started => bornNear(row, started))) adopt(row, '', ownedPids);
+ }
+
// Process ancestry captures non-Node children; journal receipts retain ownership after reparenting.
let changed = true;
while (changed) {
@@ -562,28 +604,24 @@ function createDescendantTracker(directory) {
ownedPids.add(row.pid); changed = true;
}
}
- // Detached children establish a private process group; census every member even
- // after the original launcher is reparented or exits.
- for (const row of current.values()) {
- if (/^Z/u.test(row.state)) continue;
- const leaderStarted = detachedGroups.get(row.pgid);
- if (!leaderStarted) continue;
- const leader = current.get(row.pgid);
- if (leader && leader.birth !== leaderStarted) {
+ // Every member of a group the job owns is the job's (see adopt), so census each one even after its launcher
+ // exited. The kernel reuses a leader's number for a new group only once the old group is gone, so a reused
+ // leader ends that custody; a proven member still inside it would contradict that, and stays uncertain (#1063).
+ for (const [pgid, leaderStarted] of detachedGroups) {
+ const leader = current.get(pgid);
+ if (!leader || !leaderStarted || leader.birth === leaderStarted) continue;
+ if ([...current.values()].some(row => row.pgid === pgid && row.pid !== pgid && !/^Z/u.test(row.state) && known.get(row.pid)?.birth === row.birth)) {
markUncertain('detached process group identity was reused');
- continue;
}
+ detachedGroups.delete(pgid);
+ }
+ for (const row of current.values()) {
+ if (/^Z/u.test(row.state) || !detachedGroups.has(row.pgid)) continue;
const prior = known.get(row.pid);
if (retired.has(`${row.pid}:${row.birth}`)) { markUncertain('a retired process identity reappeared'); continue; }
if (prior && prior.birth === row.birth) { ownedPids.add(row.pid); continue; }
- if (!prior && baseline.get(row.pid)?.birth === row.birth) continue;
- if ((!prior || prior.birth !== row.birth) && !ownedPids.has(row.ppid)) {
- pendingIdentities.set(row.pid, { birth: row.birth, eventIds: new Set() });
- reason ||= 'detached process group contains an unidentified member';
- continue;
- }
- known.set(row.pid, { ...row, commandSha256: '', root: false, group: row.pgid });
- ownedPids.add(row.pid);
+ if (baseline.get(row.pid)?.birth === row.birth) continue;
+ adopt(row, '', ownedPids);
}
// A recorded PGID does not prove an unseen member belongs to this job. Keep
// the lease fail-closed when a root group contains a process lacking identity evidence.
diff --git a/test/runner-orphan-fixture.js b/test/runner-orphan-fixture.js
new file mode 100644
index 00000000..acb3d0b2
--- /dev/null
+++ b/test/runner-orphan-fixture.js
@@ -0,0 +1,141 @@
+/** A runner job's detached processes end with the job, and a cleanup it cannot verify reruns locally [C7]. */
+import assert from 'node:assert/strict';
+import { once } from 'node:events';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { DatabaseSync } from 'node:sqlite';
+import { join } from 'node:path';
+import { takeResource } from '../src/resources.js';
+import { runFixtureChild } from './fixture-child.js';
+import { readRemoteGateHolders, waitForCondition } from './runner-fixture-state.js';
+
+/** The committed job: on the runner it leaves a detached child and grandchild behind, or overflows its spawn journal. */
+const JOB = `import { spawn } from 'node:child_process';
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+const [out, remoteRoot] = process.argv.slice(2);
+const mode = readFileSync(new URL('./mode.txt', import.meta.url), 'utf8').trim();
+const option = /(?:^|\\s)--import=(?:"([^"]+)"|'([^']+)'|(\\S+))/u.exec(process.env.NODE_OPTIONS || '');
+const preload = option ? fileURLToPath(option[1] || option[2] || option[3]) : '';
+if (!preload.startsWith(remoteRoot)) { writeFileSync(join(out, mode + '-local'), 'local\\n'); console.log('ran locally'); process.exit(0); }
+writeFileSync(join(out, mode + '-remote'), 'remote\\n');
+if (mode === 'overflow') { writeFileSync(join(dirname(preload), 'child-journal', 'overflow'), 'fixture overflow\\n'); process.exit(0); }
+// A launcher that exits as soon as it spawns, so no census sees the detached child beside its parent.
+spawn(process.execPath, ['-e', "require('node:child_process').spawn(process.execPath, [process.argv[1], process.argv[2]], { detached: true, stdio: 'ignore' }).unref(); process.exit(0);",
+ fileURLToPath(new URL('./orphan-child.mjs', import.meta.url)), out], { stdio: 'ignore' });
+const deadline = Date.now() + 10_000;
+while (!existsSync(join(out, 'grandchild.pid')) && Date.now() < deadline) await new Promise(done => setTimeout(done, 20));
+console.log('left a detached child and grandchild');
+`;
+const CHILD = `import { spawn } from 'node:child_process';
+import { renameSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+const [out] = process.argv.slice(2);
+const grandchild = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { detached: true, stdio: 'ignore' });
+grandchild.unref();
+writeFileSync(join(out, 'child.pid'), String(process.pid));
+writeFileSync(join(out, 'grandchild.tmp'), String(grandchild.pid));
+renameSync(join(out, 'grandchild.tmp'), join(out, 'grandchild.pid'));
+process.on('SIGTERM', () => {});
+setInterval(() => {}, 1000);
+`;
+
+/**
+ * Run the gate twice on a stand-in SSH runner while the only local slot is held. The first job leaves a detached
+ * child and grandchild that no census sees beside their exited launcher; the runner must still end both, verify
+ * that, and return green. The second job makes its cleanup unverifiable; the run must rerun locally, not fail.
+ */
+export async function runnerOrphanFixture(t, tools) {
+ const { cli, git, installSshStandIn, observe, startToolingCli, processIsAlive, HOST } = tools;
+ const dir = realpathSync(mkdtempSync(join(tmpdir(), 'pullboard-runner-orphans-')));
+ const home = join(dir, 'home');
+ const machineHome = join(home, '.pullboard');
+ const remoteHome = join(dir, 'worker-home');
+ const remoteStateRoot = join(remoteHome, '.pullboard-runner');
+ const remoteMachineHome = join(remoteHome, '.pullboard');
+ const remoteDatabase = join(remoteMachineHome, 'resources.sqlite');
+ const repo = join(dir, 'repo');
+ const out = join(dir, 'out');
+ const worker = join(dir, 'second-machine');
+ const bin = join(dir, 'bin');
+ const starts = join(dir, 'ssh-starts.jsonl');
+ const workerLog = join(dir, 'worker-stderr.log');
+ const pidIn = name => existsSync(join(out, name)) ? Number(readFileSync(join(out, name), 'utf8')) : 0;
+ let running;
+ let held;
+ const previous = [process.env.PULLBOARD_HOME, process.env.PULLBOARD_MACHINE_HOME];
+ t.after(async () => {
+ if (running && running.exitCode === null && running.signalCode === null) {
+ try { process.kill(-running.pid, 'SIGTERM'); } catch (error) { if (error.code !== 'ESRCH') t.diagnostic(`fixture group: ${error.message}`); }
+ await once(running, 'close').catch(error => t.diagnostic(`fixture child close: ${error.message}`));
+ }
+ // A failing run must not leave its own orphans behind either.
+ for (const pid of [pidIn('child.pid'), pidIn('grandchild.pid')]) {
+ if (pid > 0) try { process.kill(pid, 'SIGKILL'); } catch (error) { if (error.code !== 'ESRCH') t.diagnostic(`fixture orphan: ${error.message}`); }
+ }
+ held?.release();
+ for (const [key, value] of [['PULLBOARD_HOME', previous[0]], ['PULLBOARD_MACHINE_HOME', previous[1]]]) {
+ if (value === undefined) delete process.env[key];
+ else process.env[key] = value;
+ }
+ rmSync(dir, { recursive: true, force: true });
+ });
+ for (const folder of [repo, out, worker, bin, home, machineHome, remoteHome, remoteStateRoot, remoteMachineHome]) mkdirSync(folder, { recursive: true });
+ writeFileSync(join(machineHome, 'settings.json'), JSON.stringify({ gateSlots: 1 }));
+ const env = {
+ ...process.env, HOME: home, PULLBOARD_HOME: home, PULLBOARD_MACHINE_HOME: machineHome,
+ PATH: `${bin}:${process.env.PATH}`, GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_NOSYSTEM: '1',
+ GIT_AUTHOR_NAME: 'Runner Orphan Fixture', GIT_AUTHOR_EMAIL: '[email protected]',
+ GIT_COMMITTER_NAME: 'Runner Orphan Fixture', GIT_COMMITTER_EMAIL: '[email protected]',
+ };
+ const initialized = runFixtureChild('git', ['init', '-q', '-b', 'main'], { cwd: repo, env, encoding: 'utf8' });
+ assert.equal(initialized.status, 0, initialized.context);
+ writeFileSync(join(repo, 'pullboard.json'), JSON.stringify({ gate: `node orphan-job.mjs ${JSON.stringify(out)} ${JSON.stringify(remoteStateRoot)}` }));
+ writeFileSync(join(repo, 'orphan-job.mjs'), JOB);
+ writeFileSync(join(repo, 'orphan-child.mjs'), CHILD);
+ writeFileSync(join(repo, 'mode.txt'), 'detached\n');
+ git(repo, env, 'add', 'pullboard.json', 'orphan-job.mjs', 'orphan-child.mjs', 'mode.txt');
+ git(repo, env, 'commit', '-q', '-m', 'test: a job that leaves detached processes behind');
+ installSshStandIn({ bin, worker, home: remoteHome, machineHome: remoteMachineHome, requests: join(dir, 'ssh-requests.jsonl'), starts, stderrLog: workerLog });
+ const added = cli(repo, env, 'runner', 'add', HOST, '--slots', '1', '--browser', '1', '--json');
+ assert.equal(added.status, 0, added.context);
+ process.env.PULLBOARD_HOME = home;
+ process.env.PULLBOARD_MACHINE_HOME = machineHome;
+ held = await takeResource({ name: 'gate', capacity: 1, scope: 'machine', root: repo,
+ agent: 'orphan-local-holder', item: 'orphan-local-holder', kind: 'fixture-hold', browser: true });
+ /** One gate run through the real CLI; `whenRemote` runs once the runner's job body has started. */
+ const gate = async (whenRemote) => {
+ running = startToolingCli(repo, env, ['gate', '--json'], { detached: true });
+ const run = observe(running);
+ if (whenRemote) { await waitForCondition(() => existsSync(join(out, 'overflow-remote')), 'the overflow job ran on the runner'); whenRemote(); }
+ const completed = await run.closed;
+ return { ...completed, context: `${completed.stderr}\n${completed.stdout}` };
+ };
+
+ const first = await gate();
+ assert.equal(first.status, 0, first.context);
+ assert.ok(existsSync(join(out, 'detached-remote')) && !existsSync(join(out, 'detached-local')), 'the first job ran on the runner only');
+ const [child, grandchild] = [pidIn('child.pid'), pidIn('grandchild.pid')];
+ assert.ok(child > 0 && grandchild > 0 && child !== grandchild, 'the job started a detached child and grandchild');
+ await waitForCondition(() => !processIsAlive(child) && !processIsAlive(grandchild), 'the runner ended the detached child and grandchild',
+ { state: () => ({ child: processIsAlive(child), grandchild: processIsAlive(grandchild) }) });
+ const log = existsSync(workerLog) ? readFileSync(workerLog, 'utf8') : '';
+ assert.doesNotMatch(log, /PULLBOARD_RUNNER_CLEANUP_(?:UNVERIFIED|TIMEOUT)/u, 'the runner verified its cleanup from its own record');
+
+ writeFileSync(join(repo, 'mode.txt'), 'overflow\n');
+ git(repo, env, 'commit', '-q', '-m', 'test: a job whose cleanup cannot be verified', '--', 'mode.txt');
+ const second = await gate(() => { held.release(); held = null; });
+ assert.equal(second.status, 0, second.context);
+ const local = new DatabaseSync(join(machineHome, 'resources.sqlite'), { readOnly: true });
+ try {
+ assert.equal(local.prepare('SELECT detail FROM runner_unavailable WHERE host = ?').get(HOST)?.detail, 'CLEANUP_UNVERIFIED',
+ 'the runner whose cleanup was unverified is out of the queue until checked, by name');
+ } finally { local.close(); }
+ assert.ok(existsSync(join(out, 'overflow-local')), 'the run whose runner cleanup was unverified reran locally');
+ assert.equal(readFileSync(starts, 'utf8').trim().split('\n').length, 2, 'each run reached the runner once and was never retried there');
+ const retained = readdirSync(join(remoteStateRoot, 'jobs')).filter(name => existsSync(join(remoteStateRoot, 'jobs', name, 'cleanup-timeout.json')));
+ assert.equal(retained.length, 1, 'the runner kept the unverified job for inspection');
+ assert.match(readFileSync(join(remoteStateRoot, 'jobs', retained[0], 'cleanup-timeout.json'), 'utf8'), /child spawn journal overflow/u);
+ await waitForCondition(() => readRemoteGateHolders(remoteDatabase).length === 0, 'the runner gate holder releases after its cleanup timeout');
+}
diff --git a/test/runners.test.js b/test/runners.test.js
index d09eb06d..cba7f951 100644
--- a/test/runners.test.js
+++ b/test/runners.test.js
@@ -16,6 +16,7 @@ import { readRemoteGateHolders, waitForCondition } from './runner-fixture-state.
import { assertAttemptDeadlineReset, assertRedispatchDeadlineAccounting, runPairedRerun, startRunnerCpuStandIn } from './runner-paired-rerun-fixture.js';
import { assertFixtureGroupEpermControls, stopGroup } from './runner-process-group.js';
import { parentDeathDrainFixture } from './runner-parent-death-fixture.js';
+import { runnerOrphanFixture } from './runner-orphan-fixture.js';
import { createE2eHelpers } from './e2e-helpers.js';
import { refuseSandboxBlocked } from '../src/gate.js';
import { SECRET, HOST, shellQuote, installSshStandIn } from './runner-ssh-fixture.js';
@@ -1503,114 +1504,9 @@ test('a queued run executes on a registered runner and its result comes back [C7
test('a queued run executes on a registered runner and its result comes back [C7] (parent death drains a child that leaves its group after the first census)',
{ timeout: 90_000 }, t => parentDeathDrainFixture(t, parentDeathTools, { lateGroup: true })); // budget: same fixture ceiling as the ordinary parent-death case.
-test('a queued run executes on a registered runner and its result comes back [C7] (unverified cleanup times out as a named red result)', { timeout: 90_000 }, async (t) => { // budget: bounded 30-second remote cleanup timeout plus startup, transport and fixture cleanup ceiling.
- const dir = realpathSync(mkdtempSync(join(tmpdir(), 'pullboard-runner-cleanup-timeout-')));
- const home = join(dir, 'home');
- const machineHome = join(home, '.pullboard');
- const remoteHome = join(dir, 'worker-home');
- const remoteStateRoot = join(remoteHome, '.pullboard-runner');
- const remoteMachineHome = join(remoteHome, '.pullboard');
- const remoteDatabase = join(remoteMachineHome, 'resources.sqlite');
- const repo = join(dir, 'repo');
- const worker = join(dir, 'second-machine');
- const bin = join(dir, 'bin');
- const requests = join(dir, 'ssh-requests.jsonl');
- const bodyRan = join(remoteStateRoot, 'cleanup-timeout-body-ran');
- const starts = join(dir, 'ssh-starts.jsonl');
- let commandChild;
- t.after(async () => {
- if (commandChild && commandChild.exitCode === null && commandChild.signalCode === null) {
- try { process.kill(-commandChild.pid, 'SIGTERM'); }
- catch (error) { if (error.code !== 'ESRCH') t.diagnostic(`fixture process-group cleanup: ${error.message}`); }
- await once(commandChild, 'close').catch(error => t.diagnostic(`fixture child close: ${error.message}`));
- }
- if (existsSync(remoteDatabase)) {
- try { await waitForCondition(() => readRemoteGateHolders(remoteDatabase).length === 0,
- 'fixture remote physical gate holder cleanup'); }
- catch (error) { t.diagnostic(error.message); }
- }
- rmSync(dir, { recursive: true, force: true });
- });
- for (const folder of [repo, worker, bin, home, machineHome, remoteHome, remoteStateRoot, remoteMachineHome]) {
- mkdirSync(folder, { recursive: true });
- }
- writeFileSync(join(machineHome, 'settings.json'), JSON.stringify({ gateSlots: 1 }));
- const env = {
- ...process.env, HOME: home, PULLBOARD_HOME: home, PULLBOARD_MACHINE_HOME: machineHome,
- PATH: `${bin}:${process.env.PATH}`, GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_NOSYSTEM: '1',
- GIT_AUTHOR_NAME: 'Runner Cleanup Fixture', GIT_AUTHOR_EMAIL: '[email protected]',
- GIT_COMMITTER_NAME: 'Runner Cleanup Fixture', GIT_COMMITTER_EMAIL: '[email protected]',
- };
- const initialized = runFixtureChild('git', ['init', '-q', '-b', 'main'], { cwd: repo, env, encoding: 'utf8' });
- assert.equal(initialized.status, 0, initialized.context);
- writeFileSync(join(repo, 'pullboard.json'), JSON.stringify({ gate: 'node runner-cleanup-timeout.js' }));
- writeFileSync(join(repo, 'tracked.txt'), 'cleanup timeout fixture\n');
- writeFileSync(join(repo, 'runner-cleanup-timeout.js'), `import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
-import { dirname, join } from 'node:path';
-import { fileURLToPath } from 'node:url';
-const option = /(?:^|\\s)--import=(?:"([^"]+)"|'([^']+)'|(\\S+))/u.exec(process.env.NODE_OPTIONS || '');
-if (!option) throw new Error('runner preload path was not passed to the committed job');
-const preload = fileURLToPath(option[1] || option[2] || option[3]);
-const runDirectory = dirname(preload);
-const journalDirectory = join(runDirectory, 'child-journal');
-const runnerRoot = dirname(dirname(runDirectory));
-if (!existsSync(journalDirectory)) throw new Error('private child journal is missing');
-mkdirSync(runnerRoot, { recursive: true });
-writeFileSync(join(journalDirectory, 'overflow'), 'fixture journal overflow\\n');
-writeFileSync(join(runnerRoot, 'cleanup-timeout-body-ran'), 'ran\\n');
-console.log('RUNNER_CLEANUP_TIMEOUT_BODY_RAN');
-`);
- git(repo, env, 'add', 'pullboard.json', 'tracked.txt', 'runner-cleanup-timeout.js');
- git(repo, env, 'commit', '-q', '-m', 'test: prepare runner cleanup timeout fixture');
- installSshStandIn({ bin, worker, home: remoteHome, machineHome: remoteMachineHome, requests, starts });
- const added = cli(repo, env, 'runner', 'add', HOST, '--slots', '1', '--browser', '1', '--json');
- assert.equal(added.status, 0, added.context);
-
- const previousHome = process.env.PULLBOARD_HOME;
- const previousMachineHome = process.env.PULLBOARD_MACHINE_HOME;
- process.env.PULLBOARD_HOME = home;
- process.env.PULLBOARD_MACHINE_HOME = machineHome;
- let held;
- try {
- held = await takeResource({ name: 'gate', capacity: 1, scope: 'machine', root: repo,
- agent: 'cleanup-timeout-local-holder', item: 'cleanup-timeout-local-holder', kind: 'fixture-hold', browser: true });
- // The real CLI must refuse this one response as RUNNER_CLEANUP_TIMEOUT; a missing or
- // incomplete transport result would take the retry path and could incorrectly run locally.
- commandChild = startToolingCli(repo, env, ['gate', '--json'], { detached: true });
- const command = observe(commandChild);
- const completed = await command.closed;
- assert.equal(completed.status, 1, `${completed.stderr}\n${completed.stdout}`);
- const response = JSON.parse(completed.stdout.trim());
- assert.equal(response.error?.code, 'RUNNER_CLEANUP_TIMEOUT', JSON.stringify(response));
- assert.match(response.error.message, /child spawn journal overflow or write failure/u,
- 'the named refusal retains the worker reason instead of losing it at the controller');
- const jobId = /inspect job ([A-Za-z0-9_-]+)/u.exec(response.error.message)?.[1];
- assert.ok(jobId, 'the refusal identifies the exact failed job for inspection');
- const retained = readdirSync(join(remoteStateRoot, 'jobs')).find(name => name.startsWith(jobId + '-'));
- assert.ok(retained, 'a cleanup timeout retains its private job diagnostics');
- const diagnosis = JSON.parse(readFileSync(join(remoteStateRoot, 'jobs', retained, 'cleanup-timeout.json'), 'utf8'));
- assert.equal(diagnosis.id, jobId);
- assert.match(diagnosis.reason, /child spawn journal overflow or write failure/u);
- assert.ok(diagnosis.stage, 'the timeout records which cleanup step was running');
- assert.ok(existsSync(bodyRan), 'the committed command ran to completion and wrote its own overflow marker');
- assert.equal(readFileSync(starts, 'utf8').trim().split('\n').length, 1, 'the refused cleanup result is not retried or sent to a local runner');
- await waitForCondition(() => readRemoteGateHolders(remoteDatabase).length === 0,
- 'remote physical gate holder releases after the named cleanup timeout');
- assert.equal(readRemoteGateHolders(join(machineHome, 'resources.sqlite')).length, 1,
- 'the separate local fixture holder remains intact until explicitly released');
- } finally {
- if (commandChild && commandChild.exitCode === null && commandChild.signalCode === null) {
- try { process.kill(-commandChild.pid, 'SIGTERM'); }
- catch (error) { if (error.code !== 'ESRCH') throw error; }
- await once(commandChild, 'close').catch(() => {});
- }
- if (held) held.release();
- if (previousHome === undefined) delete process.env.PULLBOARD_HOME;
- else process.env.PULLBOARD_HOME = previousHome;
- if (previousMachineHome === undefined) delete process.env.PULLBOARD_MACHINE_HOME;
- else process.env.PULLBOARD_MACHINE_HOME = previousMachineHome;
- }
-});
+const orphanTools = { cli, git, installSshStandIn, observe, startToolingCli, processIsAlive, HOST };
+test('a runner job leaves no process behind, detached ones included [C7]',
+ { timeout: 150_000 }, t => runnerOrphanFixture(t, orphanTools)); // budget: one plain runner round trip, then the declared 30-second remote cleanup timeout and a local rerun.
/** Read one machine pool's browser row through the public resource API, selecting that machine's home. */
function browserPoolAt(machineHome, root) {