Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

Merge branch 'main' into core/205

Corey Olson08c83d62edae
diff --cc src/child-journal-preload.js
index 00000000,72eb7098..861eb493
mode 000000,100644..100644
--- a/src/child-journal-preload.js
+++ b/src/child-journal-preload.js
@@@ -1,0 -1,66 +1,109 @@@
+ /** The SSH runner's spawn-journal preload, built from its directory alone so its content proves what it is [C7]. */
++import { spawnSync } from 'node:child_process';
++import { readFileSync } from 'node:fs';
+ import { join } from 'node:path';
+
++// Print a process's kernel start time to the microsecond through sysctl KERN_PROC_PID (kinfo_proc opens with it).
++const PERL_START = `my $mib = pack('l4', 1, 14, 1, $ARGV[0]); my $buf = "\\0" x 1024; my $len = pack('Q', length $buf);
++exit 1 if syscall(202, $mib, 4, $buf, $len, 0, 0) != 0 || unpack('Q', $len) < 16;
++my ($sec, $usec) = unpack('q l', substr($buf, 0, 16)); printf("%d.%06d\\n", $sec, $usec);`;
++
++/**
++ * A process's exact kernel start time, or '' when it cannot be read: procfs start ticks on Linux, microseconds through
++ * sysctl on macOS. With its pid it names one process, so a reused pid never matches (#1063).
++ *
++ * @param {number} pid
++ * @returns {string}
++ */
++export function exactStart(pid) {
++ if (!Number.isSafeInteger(pid) || pid < 1) return '';
++ try {
++ const stat = readFileSync(`/proc/${pid}/stat`, 'utf8');
++ const close = stat.lastIndexOf(')');
++ const field = stat.slice(close + 2).trim().split(/\s+/u)[19];
++ return close > 0 && /^\d+$/u.test(field ?? '') ? `proc:${field}` : '';
++ } catch (error) {
++ if (process.platform !== 'darwin' || error.code !== 'ENOENT') return '';
++ }
++ const result = spawnSync('/usr/bin/perl', ['-e', PERL_START, String(pid)], { encoding: 'utf8', timeout: 2_000, env: { PATH: '/usr/bin:/bin', LANG: 'C' } });
++ const text = typeof result.stdout === 'string' ? result.stdout.trim() : '';
++ return !result.error && result.status === 0 && /^\d+\.\d{6}$/u.test(text) ? `mac:${text}` : '';
++}
++
+ /**
+ * Source of the preload an SSH runner puts in NODE_OPTIONS for a job: it records only child identity and a one-way
+ * command digest. It depends on nothing but its directory, so a classifier can rebuild it and compare byte for byte.
++ * Each spawn receipt also carries the child's exactStart, read in the spawn tick or at this process's exit: the
++ * child is not yet reaped then, so its pid is still its own (#1063).
+ *
+ * @param {string} directory the job directory that holds child-spawn-preload.mjs and child-journal/
+ * @returns {string}
+ */
+ export function childJournalPreloadSource(directory) {
+ const journalDirectory = join(directory, 'child-journal');
+ const overflowPath = join(journalDirectory, 'overflow');
+ return `
+ import diagnostics from 'node:diagnostics_channel';
-import { closeSync, openSync, statSync, writeSync, writeFileSync } from 'node:fs';
++import { spawnSync } from 'node:child_process';
++import { closeSync, openSync, readFileSync, statSync, writeSync, writeFileSync } from 'node:fs';
+ import { join } from 'node:path';
+ import { createHash, randomUUID } from 'node:crypto';
++const PERL_START = ${JSON.stringify(PERL_START)};
++${exactStart.toString()}
+ const directory = ${JSON.stringify(journalDirectory)};
+ const overflowPath = ${JSON.stringify(overflowPath)};
+ const instance = randomUUID();
+ process.env.PULLBOARD_RUNNER_INSTANCE = instance;
+ const file = join(directory, 'children-' + process.pid + '.jsonl');
+ const fd = openSync(file, 'a', 0o600);
+ let bytes = statSync(file).size;
+ let overflow = false;
+ const limit = 131072;
+ function record(value) {
+ if (overflow) return;
+ try {
+ const line = JSON.stringify(value) + '\\n';
+ const size = Buffer.byteLength(line);
+ if (bytes + size > limit) {
+ overflow = true;
+ try { writeFileSync(overflowPath, 'journal limit reached\\n', { flag: 'wx', mode: 0o600 }); } catch {}
+ return;
+ }
+ writeSync(fd, line);
+ bytes += size;
+ } catch {
+ try { writeFileSync(overflowPath, 'journal write failed\\n', { flag: 'wx', mode: 0o600 }); } catch {}
+ overflow = true;
+ }
+ }
+ record({ type: 'boot', pid: process.pid, ppid: process.ppid, instance });
+ const channel = diagnostics.channel('child_process');
+ const seen = new WeakSet();
++// The spawn event comes a tick later; a parent that exits in the same tick still owes those children a receipt.
++const unrecorded = new Map();
+ channel.subscribe(message => {
+ try {
+ const child = message?.process;
+ if (!child || typeof child.once !== 'function' || seen.has(child)) return;
+ seen.add(child);
+ const writeReceipt = () => {
- if (!Number.isSafeInteger(child.pid) || child.pid < 1) return;
++ if (!unrecorded.delete(child) || !Number.isSafeInteger(child.pid) || child.pid < 1) return;
+ const commandSha256 = createHash('sha256').update(JSON.stringify({ command: child.spawnfile ?? '', args: child.spawnargs ?? [] })).digest('hex');
- record({ type: 'spawn', pid: child.pid, ppid: process.pid, instance, commandSha256 });
++ const start = exactStart(child.pid);
++ record({ type: 'spawn', pid: child.pid, ppid: process.pid, instance, commandSha256, ...(start ? { start } : {}) });
+ };
++ unrecorded.set(child, writeReceipt);
++ child.once('error', () => unrecorded.delete(child));
+ if (Number.isSafeInteger(child.pid)) writeReceipt();
+ else child.once('spawn', writeReceipt);
+ } catch {
+ try { writeFileSync(overflowPath, 'child receipt could not be recorded\\n', { flag: 'wx', mode: 0o600 }); } catch {}
+ }
+ });
-process.once('exit', () => { try { closeSync(fd); } catch {} });
++process.once('exit', () => {
++ // Still unreaped while this process exits, so each pid is still its child's own.
++ for (const writeReceipt of [...unrecorded.values()]) writeReceipt();
++ try { closeSync(fd); } catch {}
++});
+ `;
+ }
diff --cc src/runner-custody.js
index 004b6ba7,00000000..ef5ea3ae
mode 100644,000000..100644
--- a/src/runner-custody.js
+++ b/src/runner-custody.js
@@@ -1,615 -1,0 +1,528 @@@
+/** Custody of a runner job's processes: who the job started, proven from its own record, and stopping them [C7]. */
+import { spawnSync } from 'node:child_process';
+import {
+ closeSync,
+ existsSync,
+ mkdirSync,
+ openSync,
+ readFileSync,
+ readdirSync,
+ readSync,
+ statSync,
+ writeFileSync,
+} from 'node:fs';
+import {
+ delimiter,
+ dirname,
+ join,
+} from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { parseProcessCensus } from './process-census.js';
++import { childJournalPreloadSource } from './child-journal-preload.js';
++import { exactStart } from './child-journal-preload.js';
+
+const safePath = [dirname(process.execPath), process.env.PATH || '/usr/local/bin:/usr/bin:/bin'].join(delimiter);
+
+/** Read one all-process POSIX census; a failed or malformed census is never interpreted as an empty machine. */
+export function processCensus() {
+ const result = spawnSync('/bin/ps', ['-axo', 'pid=,ppid=,pgid=,lstart=,stat=,comm='], {
+ encoding: 'utf8', timeout: 2_000, maxBuffer: 8 * 1024 * 1024,
+ env: { PATH: safePath, LANG: 'C', LC_ALL: 'C', TZ: 'UTC' },
+ });
+ if (result.error || result.status !== 0 || typeof result.stdout !== 'string') return null;
+ return parseProcessCensus(result.stdout);
+}
+
+/** Add a kernel start-time identity when procfs is available; otherwise retain a conservative census tuple. */
+export function processIdentity(row) {
+ let birth = '';
+ let exact = false;
+ try {
+ const stat = readFileSync(`/proc/${row.pid}/stat`, 'utf8');
+ const close = stat.lastIndexOf(')');
+ const fields = stat.slice(close + 2).trim().split(/\s+/u);
+ if (close > 0 && Number(fields[1]) === row.ppid && Number(fields[2]) === row.pgid && /^\d+$/u.test(fields[19] ?? '')) {
+ birth = `proc:${fields[19]}`;
+ exact = true;
+ }
+ } catch { /* macOS and other POSIX hosts use the validated census tuple below. */ }
+ if (!birth) birth = `ps:${row.started}`;
+ return { ...row, birth, exact };
+}
+
- // Print a process's kernel start time to the microsecond through sysctl KERN_PROC_PID (kinfo_proc opens with it).
- const PERL_START = `my $mib = pack('l4', 1, 14, 1, $ARGV[0]); my $buf = "\\0" x 1024; my $len = pack('Q', length $buf);
- exit 1 if syscall(202, $mib, 4, $buf, $len, 0, 0) != 0 || unpack('Q', $len) < 16;
- my ($sec, $usec) = unpack('q l', substr($buf, 0, 16)); printf("%d.%06d\\n", $sec, $usec);`;
-
- /**
- * A process's exact kernel start time, or '' when it cannot be read: procfs start ticks on Linux, microseconds through
- * sysctl on macOS. With its pid it names one process, so a reused pid never matches (#1063).
- *
- * @param {number} pid
- * @returns {string}
- */
- export function exactStart(pid) {
- if (!Number.isSafeInteger(pid) || pid < 1) return '';
- try {
- const stat = readFileSync(`/proc/${pid}/stat`, 'utf8');
- const close = stat.lastIndexOf(')');
- const field = stat.slice(close + 2).trim().split(/\s+/u)[19];
- return close > 0 && /^\d+$/u.test(field ?? '') ? `proc:${field}` : '';
- } catch (error) {
- if (process.platform !== 'darwin' || error.code !== 'ENOENT') return '';
- }
- const result = spawnSync('/usr/bin/perl', ['-e', PERL_START, String(pid)], { encoding: 'utf8', timeout: 2_000, env: { PATH: '/usr/bin:/bin', LANG: 'C' } });
- const text = typeof result.stdout === 'string' ? result.stdout.trim() : '';
- return !result.error && result.status === 0 && /^\d+\.\d{6}$/u.test(text) ? `mac:${text}` : '';
- }
-
+/**
+ * Make a private Node preload that records only child identity and a one-way command digest. Each spawn receipt also
+ * carries the child's exact start, read in the spawn tick: the child is not yet reaped, so its pid is still its own.
+ */
+function writeChildJournalPreload(directory) {
+ const journalDirectory = join(directory, 'child-journal');
+ mkdirSync(journalDirectory, { mode: 0o700 });
+ const overflowPath = join(journalDirectory, 'overflow');
- const source = `
- import diagnostics from 'node:diagnostics_channel';
- import { closeSync, openSync, statSync, writeSync, writeFileSync } from 'node:fs';
- import { join } from 'node:path';
- import { createHash, randomUUID } from 'node:crypto';
- import { exactStart } from ${JSON.stringify(import.meta.url)};
- const directory = ${JSON.stringify(journalDirectory)};
- const overflowPath = ${JSON.stringify(overflowPath)};
- const instance = randomUUID();
- process.env.PULLBOARD_RUNNER_INSTANCE = instance;
- const file = join(directory, 'children-' + process.pid + '.jsonl');
- const fd = openSync(file, 'a', 0o600);
- let bytes = statSync(file).size;
- let overflow = false;
- const limit = 131072;
- function record(value) {
- if (overflow) return;
- try {
- const line = JSON.stringify(value) + '\\n';
- const size = Buffer.byteLength(line);
- if (bytes + size > limit) {
- overflow = true;
- try { writeFileSync(overflowPath, 'journal limit reached\\n', { flag: 'wx', mode: 0o600 }); } catch {}
- return;
- }
- writeSync(fd, line);
- bytes += size;
- } catch {
- try { writeFileSync(overflowPath, 'journal write failed\\n', { flag: 'wx', mode: 0o600 }); } catch {}
- overflow = true;
- }
- }
- record({ type: 'boot', pid: process.pid, ppid: process.ppid, instance });
- const channel = diagnostics.channel('child_process');
- const seen = new WeakSet();
- // The spawn event comes a tick later; a parent that exits in the same tick still owes those children a receipt.
- const unrecorded = new Map();
- channel.subscribe(message => {
- try {
- const child = message?.process;
- if (!child || typeof child.once !== 'function' || seen.has(child)) return;
- seen.add(child);
- const writeReceipt = () => {
- if (!unrecorded.delete(child) || !Number.isSafeInteger(child.pid) || child.pid < 1) return;
- const commandSha256 = createHash('sha256').update(JSON.stringify({ command: child.spawnfile ?? '', args: child.spawnargs ?? [] })).digest('hex');
- const start = exactStart(child.pid);
- record({ type: 'spawn', pid: child.pid, ppid: process.pid, instance, commandSha256, ...(start ? { start } : {}) });
- };
- unrecorded.set(child, writeReceipt);
- child.once('error', () => unrecorded.delete(child));
- if (Number.isSafeInteger(child.pid)) writeReceipt();
- else child.once('spawn', writeReceipt);
- } catch {
- try { writeFileSync(overflowPath, 'child receipt could not be recorded\\n', { flag: 'wx', mode: 0o600 }); } catch {}
- }
- });
- process.once('exit', () => {
- // Still unreaped while this process exits, so each pid is still its child's own.
- for (const writeReceipt of [...unrecorded.values()]) writeReceipt();
- try { closeSync(fd); } catch {}
- });
- `;
+ const preload = join(directory, 'child-spawn-preload.mjs');
- writeFileSync(preload, source, { mode: 0o600, flag: 'wx' });
++ writeFileSync(preload, childJournalPreloadSource(directory), { mode: 0o600, flag: 'wx' });
+ return { preload, journalDirectory, overflowPath, nodeOptions: `--import=${pathToFileURL(preload).href}` };
+}
+
+/**
+ * Track job descendants by PID plus OS start identity, joining spawn receipts with fresh census rows. The hooks
+ * default to this host; a test passes a scripted census and a recording kill (#1063).
+ *
+ * @param {string} directory - The job's private run directory, where the child journal lives.
+ * @param {{ census?: () => Map<number, object>|null, kill?: (pid: number, signal: string|number) => void, selfPid?: number,
+ * tick?: () => void, timedOut?: () => boolean, warn?: (reason: string) => void, intervalMs?: number,
+ * startOf?: (pid: number) => string }} [hooks]
+ */
+export function createDescendantTracker(directory, { census: processCensusHook = processCensus, kill = (pid, signal) => process.kill(pid, signal),
+ selfPid = process.pid, tick = () => {}, timedOut = () => false, warn = () => {}, intervalMs = 250, startOf = exactStart } = {}) {
+ const initial = processCensusHook();
+ if (!initial) throw new Error('initial process identity census failed');
+ const baseline = new Map([...initial].map(([pid, row]) => [pid, processIdentity(row)]));
+ const journal = writeChildJournalPreload(directory);
+ const roots = new Set();
+ const rootLaunchIdentities = new Map();
+ const known = new Map();
+ const retired = new Set();
+ const eventsByChild = new Map();
+ const pendingIdentities = new Map();
+ const consumedEvents = new Set();
+ let eventCount = 0;
+ const detachedGroups = new Map();
+ const offsets = new Map();
+ const partial = new Map();
+ const instancePids = new Map();
+ const overflowPath = journal.overflowPath;
+ let current = baseline;
+ let uncertain = false;
+ let censusFailure = false;
+ let reason = '';
+ let uncertainReason = '';
+ let busy = false;
+ let warned = false;
+ let timer = null;
+ let stoppingPromise = null;
+
+ /** Keep permanent custody failures distinct from earlier, potentially resolved pending observations. */
+ function markUncertain(message) {
+ uncertain = true;
+ uncertainReason ||= message;
+ reason ||= message;
+ }
+
+ /**
+ * Take custody of a live member of a group this census proves is the job's (groupProven). A group lives inside one
+ * session, and a session holds only descendants of the job process that made it, and the kernel never reuses a
+ * group's number while a member lives, so the member is the job's. It is marked `byGroup`: membership is its only
+ * proof, so it never proves the group in turn (#1063).
+ */
+ function adoptMember(row, ownedPids) {
+ known.set(row.pid, { ...row, commandSha256: '', root: false, group: row.pgid, byGroup: true });
+ pendingIdentities.delete(row.pid);
+ ownedPids.add(row.pid);
+ }
+
+ /**
+ * Whether this census proves group `pgid` is the job's: a live member is a process the job proved itself, as the
+ * runner's own launch, a spawn receipt matched while its parent was the job's, or live ancestry. A recorded start
+ * time never proves a group, since a census reads start times only to the second (#1063).
+ */
+ function groupProven(pgid) {
+ for (const row of current.values()) {
+ if (row.pgid !== pgid || /^Z/u.test(row.state)) continue;
+ const identity = known.get(row.pid);
+ if (identity && !identity.byGroup && identity.birth === row.birth && !retired.has(`${row.pid}:${row.birth}`)) return true;
+ }
+ return false;
+ }
+
+ function readJournal() {
+ if (existsSync(overflowPath)) { markUncertain('child spawn journal overflow or write failure'); }
+ let total = 0;
+ const files = readdirSync(journal.journalDirectory).filter(name => /^children-\d+\.jsonl$/u.test(name)).sort();
+ // A full suite creates one journal per Node process; align this count with
+ // the event and identity bounds while retaining the byte bounds below.
+ if (files.length > 100_000) { markUncertain('child spawn journal file limit exceeded'); return; }
+ for (const name of files) {
+ const file = join(journal.journalDirectory, name);
+ let size;
+ try { size = statSync(file).size; } catch { markUncertain('child spawn journal could not be inspected'); continue; }
+ total += size;
+ if (!Number.isSafeInteger(size) || size > 131_072 || total > 8 * 1024 * 1024) {
+ markUncertain('child spawn journal size limit exceeded'); continue;
+ }
+ const offset = offsets.get(name) ?? 0;
+ if (size < offset) { markUncertain('child spawn journal changed unexpectedly'); continue; }
+ if (size === offset) continue;
+ const length = size - offset;
+ const fd = openSync(file, 'r');
+ let text;
+ try {
+ const bytes = Buffer.alloc(length);
+ const read = readSync(fd, bytes, 0, length, offset);
+ if (read !== length) { markUncertain('child spawn journal read was incomplete'); continue; }
+ text = (partial.get(name) ?? '') + bytes.toString('utf8');
+ } finally { closeSync(fd); }
+ offsets.set(name, size);
+ const lines = text.split('\n');
+ partial.set(name, lines.pop() ?? '');
+ for (const line of lines) {
+ if (!line) continue;
+ if (line.length > 4_096) { markUncertain('child spawn journal record is oversized'); continue; }
+ let record;
+ try { record = JSON.parse(line); } catch { markUncertain('child spawn journal record is invalid'); continue; }
+ if (record.type === 'boot' && Number.isSafeInteger(record.pid) && record.pid > 0 && typeof record.instance === 'string') {
+ instancePids.set(record.instance, record.pid);
+ } else if (record.type === 'spawn' && Number.isSafeInteger(record.pid) && record.pid > 0
+ && Number.isSafeInteger(record.ppid) && record.ppid > 0
+ && typeof record.instance === 'string' && /^[a-f0-9]{64}$/u.test(record.commandSha256)
+ && (record.start === undefined || /^(?:proc|mac):\d+(?:\.\d{6})?$/u.test(record.start))) {
+ const childEvents = eventsByChild.get(record.pid) ?? [];
+ eventCount += 1;
+ childEvents.push({ ...record, eventId: `${name}:${eventCount}` });
+ eventsByChild.set(record.pid, childEvents);
+ if (eventCount > 100_000) { markUncertain('child spawn journal event limit exceeded'); return; }
+ } else { markUncertain('child spawn journal identity is invalid'); }
+ }
+ }
+ }
+
+ function observe() {
+ if (busy) return current;
+ busy = true;
+ try {
+ readJournal();
+ const rawCensus = processCensusHook();
+ if (!rawCensus) { censusFailure = true; return null; }
+ censusFailure = false;
+ const census = new Map([...rawCensus].map(([pid, row]) => [pid, processIdentity(row)]));
+ current = census;
+ // A process first seen without its recorded parent stays unowned while alive.
+ // Once a later census proves that exact birth identity gone, retire its receipts
+ // so a reused PID cannot be adopted from stale launch evidence.
+ for (const [pid, pending] of pendingIdentities) {
+ const row = current.get(pid);
+ if (row?.birth === pending.birth && !/^Z/u.test(row.state)) continue;
+ for (const eventId of pending.eventIds) consumedEvents.add(eventId);
+ pendingIdentities.delete(pid);
+ }
+ for (const [pid, identity] of known) {
+ const row = current.get(pid);
+ if (!row || row.birth !== identity.birth) retired.add(`${pid}:${identity.birth}`);
+ else if (retired.has(`${pid}:${identity.birth}`)) {
+ markUncertain('a previously observed process identity reappeared');
+ }
+ }
+ const ownedPids = new Set();
+ for (const pid of roots) {
+ const row = current.get(pid);
+ if (!row) continue;
+ const prior = known.get(pid);
+ const launched = rootLaunchIdentities.get(pid);
+ if (prior && prior.birth !== row.birth) continue;
+ if (launched && launched.birth !== row.birth) continue;
+ if (!launched && row.ppid !== selfPid) continue;
+ if (!launched && row.pgid !== pid) continue;
+ if (retired.has(`${pid}:${row.birth}`)) { markUncertain('a retired process identity reappeared'); continue; }
+ if (!launched) rootLaunchIdentities.set(pid, row);
+ known.set(pid, { ...row, commandSha256: prior?.commandSha256 ?? '', root: true, group: pid });
+ if (!detachedGroups.has(pid)) detachedGroups.set(pid, row.birth);
+ ownedPids.add(pid);
+ }
+ // Every identity the job already proved stays owned while it lives, even after its parent exits (#1063).
+ for (const [pid, identity] of known) {
+ const row = current.get(pid);
+ if (row && row.birth === identity.birth && !/^Z/u.test(row.state) && !retired.has(`${pid}:${row.birth}`)) ownedPids.add(pid);
+ }
+
+ for (const row of current.values()) {
+ if (/^Z/u.test(row.state)) continue;
+ for (const event of eventsByChild.get(row.pid) ?? []) {
+ if (consumedEvents.has(event.eventId)) continue;
+ const parentPid = instancePids.get(event.instance);
+ // A preload instance exists only in the private job environment. The receipt
+ // binds its child to that exact process instance even after the parent exits.
+ if (!parentPid || parentPid !== event.ppid) continue;
+ const prior = known.get(row.pid);
+ if (retired.has(`${row.pid}:${row.birth}`)) { markUncertain('a retired process identity reappeared'); continue; }
+ if (prior && prior.birth === row.birth) { consumedEvents.add(event.eventId); ownedPids.add(row.pid); continue; }
+ if (!prior && baseline.get(row.pid)?.birth === row.birth) { consumedEvents.add(event.eventId); continue; }
+ // A receipt names its child by pid and exact start, read before the child could be reaped. The same pid with
+ // the same exact start is that child, even once its parent exited unseen; a reused pid starts later (#1063).
+ if (event.start && (row.ppid !== event.ppid || !ownedPids.has(event.ppid)) && startOf(row.pid) === event.start) {
+ if (row.pgid === row.pid) detachedGroups.set(row.pid, row.birth);
+ known.set(row.pid, { ...row, commandSha256: event.commandSha256, root: false, group: row.pgid });
+ consumedEvents.add(event.eventId);
+ pendingIdentities.delete(row.pid);
+ ownedPids.add(row.pid);
+ continue;
+ }
+ // A spawn receipt must be observed while its exact parent instance still owns it.
+ // A same-numbered PID first seen after reparenting is ambiguous and never adopted.
+ if (row.ppid !== event.ppid) {
+ const pending = pendingIdentities.get(row.pid);
+ if (!pending || pending.birth !== row.birth) pendingIdentities.set(row.pid, { birth: row.birth, eventIds: new Set() });
+ pendingIdentities.get(row.pid).eventIds.add(event.eventId);
+ consumedEvents.add(event.eventId);
+ reason ||= 'spawned process was first observed after reparenting';
+ continue;
+ }
+ if (!ownedPids.has(event.ppid)) {
+ pendingIdentities.set(row.pid, { birth: row.birth, eventIds: new Set([event.eventId]) });
+ consumedEvents.add(event.eventId);
+ reason ||= 'spawn receipt parent has not been independently identified';
+ continue;
+ }
+ const parent = known.get(event.ppid);
+ const inheritedGroup = parent?.group ?? [...roots].find(rootPid => known.get(rootPid)?.group === row.pgid);
+ const group = row.pgid === row.pid ? row.pid : inheritedGroup;
+ if (group === undefined) {
+ pendingIdentities.set(row.pid, { birth: row.birth, eventIds: new Set([event.eventId]) });
+ consumedEvents.add(event.eventId);
+ reason ||= 'spawned process group has no owned parent';
+ continue;
+ }
+ if (row.pgid === row.pid) detachedGroups.set(row.pid, row.birth);
+ known.set(row.pid, { ...row, commandSha256: event.commandSha256, root: false, group });
+ consumedEvents.add(event.eventId);
+ pendingIdentities.delete(row.pid);
+ ownedPids.add(row.pid);
+ }
+ }
+
+ // Process ancestry captures non-Node children; journal receipts retain ownership after reparenting.
+ let changed = true;
+ while (changed) {
+ changed = false;
+ for (const row of current.values()) {
+ if (!ownedPids.has(row.ppid)) continue;
+ const prior = known.get(row.pid);
+ if (retired.has(`${row.pid}:${row.birth}`)) { markUncertain('a retired process identity reappeared'); continue; }
+ if (prior && prior.birth === row.birth) { ownedPids.add(row.pid); continue; }
+ if (!prior && baseline.get(row.pid)?.birth === row.birth) continue;
+ const parent = known.get(row.ppid);
+ const group = row.pgid === row.pid ? row.pid : parent?.group;
+ if (group === undefined) {
+ pendingIdentities.set(row.pid, { birth: row.birth, eventIds: new Set() });
+ reason ||= 'descendant process group has no owned parent';
+ continue;
+ }
+ if (row.pgid === row.pid) detachedGroups.set(row.pid, row.birth);
+ known.set(row.pid, { ...row, commandSha256: '', root: false, group });
+ pendingIdentities.delete(row.pid);
+ ownedPids.add(row.pid); changed = true;
+ }
+ }
+ // Every member of a group the job owns is the job's (see adopt), so census each one even after its launcher
+ // exited, but only while this census still proves the group (groupProven). The kernel reuses a leader's number
+ // for a new group only once the old group is gone, so a reused leader ends custody; a self-proven member still
+ // inside it would contradict that, and stays uncertain. Custody that cannot be proven is dropped, never used:
+ // its group-only members turn unidentified, cleanup cannot verify, and the controller reruns locally (#1063).
+ for (const [pgid, leaderStarted] of detachedGroups) {
+ const leader = current.get(pgid);
+ const reused = leader && leader.birth !== leaderStarted;
+ if (reused && [...current.values()].some(row => row.pgid === pgid && row.pid !== pgid && !/^Z/u.test(row.state)
+ && !known.get(row.pid)?.byGroup && known.get(row.pid)?.birth === row.birth)) {
+ markUncertain('detached process group identity was reused');
+ }
+ if (!reused && groupProven(pgid)) continue;
+ detachedGroups.delete(pgid);
+ for (const row of current.values()) {
+ const identity = known.get(row.pid);
+ if (row.pgid !== pgid || !identity?.byGroup || identity.birth !== row.birth || /^Z/u.test(row.state)) continue;
+ known.delete(row.pid);
+ ownedPids.delete(row.pid);
+ pendingIdentities.set(row.pid, { birth: row.birth, eventIds: new Set() });
+ reason ||= 'process group custody could not be proven';
+ }
+ }
+ for (const row of current.values()) {
+ if (/^Z/u.test(row.state) || !detachedGroups.has(row.pgid)) continue;
+ const prior = known.get(row.pid);
+ if (retired.has(`${row.pid}:${row.birth}`)) { markUncertain('a retired process identity reappeared'); continue; }
+ if (prior && prior.birth === row.birth) { ownedPids.add(row.pid); continue; }
+ if (baseline.get(row.pid)?.birth === row.birth) continue;
+ adoptMember(row, ownedPids);
+ }
+ // A recorded PGID does not prove an unseen member belongs to this job. Keep
+ // the lease fail-closed when a root group contains a process lacking identity evidence.
+ for (const rootPid of roots) {
+ for (const row of current.values()) {
+ if (row.pgid !== rootPid || /^Z/u.test(row.state)) continue;
+ const identity = known.get(row.pid);
+ if (identity?.birth === row.birth) continue;
+ if (baseline.get(row.pid)?.birth === row.birth) continue;
+ pendingIdentities.set(row.pid, { birth: row.birth, eventIds: new Set() });
+ reason ||= 'owned process group contains an unidentified member';
+ }
+ }
+ if (known.size > 100_000) { markUncertain('owned process identity limit exceeded'); }
+ // Refresh live PGID/start data only when the PID still has the same start identity.
+ for (const pid of ownedPids) {
+ const row = current.get(pid);
+ const prior = known.get(pid);
+ if (!row || prior?.birth !== row.birth) continue;
+ Object.assign(prior, row);
+ // A detached child is forked into its parent's group and calls setsid a moment later. A census in that window
+ // records the parent's group; once this proven identity leads its own group, track that group instead (#888).
+ if (!prior.root && row.pgid === row.pid && prior.group !== row.pid) {
+ prior.group = row.pid;
+ detachedGroups.set(row.pid, row.birth);
+ }
+ }
+ return current;
+ } catch {
+ censusFailure = true; return null;
+ } finally { busy = false; }
+ }
+
+ function registerRoot(pid) {
+ if (!Number.isSafeInteger(pid) || pid < 1) return;
+ roots.add(pid);
+ // Bind the spawned leader while its actual parent is still this worker; later
+ // group membership alone is never sufficient to adopt an unknown process.
+ const census = processCensusHook();
+ const row = census?.get(pid);
+ if (row && row.ppid === selfPid && row.pgid === pid) rootLaunchIdentities.set(pid, processIdentity(row));
+ }
+
+ function liveDescendants() {
+ const active = [];
+ for (const [pid, identity] of known) {
+ if (identity.root) continue;
+ const row = current.get(pid);
+ if (row && row.birth === identity.birth && !retired.has(`${pid}:${identity.birth}`) && !/^Z/u.test(row.state)) active.push({ ...identity });
+ }
+ return active;
+ }
+
+ function signalOwnedGroup(pgid, signal) {
+ const census = observe();
+ tick();
+ if (!census || censusFailure) return null;
+ // A journal error does not revoke known ownership, but an unidentified live
+ // member prevents signaling the whole group until its identity is resolved.
+ if ([...pendingIdentities].some(([pid, identity]) => {
+ const row = current.get(pid);
+ return row?.pgid === pgid && row.birth === identity.birth && !/^Z/u.test(row.state);
+ })) return null;
+ // Signal a whole group only while this census proves it is still the job's, never on recorded custody (#1063).
+ if (!groupProven(pgid)) return false;
+ try { kill(-pgid, signal); }
+ catch (error) {
+ if (error.code === 'ESRCH') return false;
+ // macOS answers EPERM for a group whose members exited but are not yet reaped. Throwing here escaped cleanup and
+ // skipped the lease release (#888, and the load flake in #850); retry on a fresh census instead, and say so.
+ if (error.code === 'EPERM') {
+ process.stderr.write(`runner: signal to process group ${pgid} answered EPERM; its members have exited, so the next census decides\n`);
+ return null;
+ }
+ throw error;
+ }
+ return true;
+ }
+
+ async function stopDescendantsLoop() {
+ // Retry through ambiguous censuses rather than releasing the host lease on an unproven drain.
+ for (;;) {
+ const census = observe();
+ tick();
+ if (!census) {
+ if (!warned) { warn(reason); warned = true; }
+ await new Promise(resolvePause => setTimeout(resolvePause, 1_000));
+ continue;
+ }
+ if (censusFailure) { await new Promise(resolvePause => setTimeout(resolvePause, 250)); continue; }
+ const live = liveDescendants();
+ // Once the budget is spent and reported, an unidentified process is left alone, never waited on forever: the
+ // controller already reruns the work locally, and a worker left waiting is itself a process left behind (#1063).
+ if (!live.length && ((!pendingIdentities.size && !uncertain) || timedOut())) return;
+ if ((pendingIdentities.size || uncertain) && !warned) {
+ warn(reason || 'unresolved process identity remains live');
+ warned = true;
+ }
+ if (!live.length) { await new Promise(resolvePause => setTimeout(resolvePause, 250)); continue; }
+ // Signal each confirmed owned process group once; use PID signals only with exact kernel identity.
+ const signaledGroups = new Set();
+ for (const identity of live) {
+ const row = current.get(identity.pid);
+ if (!row || row.birth !== identity.birth || retired.has(`${identity.pid}:${identity.birth}`) || /^Z/u.test(row.state)) continue;
+ const ownedGroup = identity.group;
+ if (Number.isSafeInteger(ownedGroup) && row.pgid === ownedGroup && !signaledGroups.has(ownedGroup)) {
+ const result = signalOwnedGroup(ownedGroup, 'SIGTERM');
+ if (result === null) { censusFailure = true; break; }
+ if (result) signaledGroups.add(ownedGroup);
+ continue;
+ }
+ if (!identity.exact) { markUncertain('descendant lacks a safe process-group identity'); break; }
+ try { kill(identity.pid, 'SIGTERM'); }
+ catch (error) { if (error.code !== 'ESRCH') { markUncertain('owned descendant could not be stopped safely'); } }
+ }
+ const deadline = Date.now() + 2_000;
+ while (Date.now() < deadline) {
+ await new Promise(resolvePause => setTimeout(resolvePause, 50));
+ if (!observe()) break;
+ if (!liveDescendants().length) break;
+ }
+ if (!current || !liveDescendants().length) continue;
+ const killedGroups = new Set();
+ for (const identity of liveDescendants()) {
+ const row = current.get(identity.pid);
+ if (!row || row.birth !== identity.birth || retired.has(`${identity.pid}:${identity.birth}`) || /^Z/u.test(row.state)) continue;
+ const ownedGroup = identity.group;
+ if (Number.isSafeInteger(ownedGroup) && row.pgid === ownedGroup && !killedGroups.has(ownedGroup)) {
+ const result = signalOwnedGroup(ownedGroup, 'SIGKILL');
+ if (result === null) { censusFailure = true; break; }
+ if (result) killedGroups.add(ownedGroup);
+ continue;
+ }
+ if (!identity.exact) { markUncertain('descendant lacks a safe process-group identity'); break; }
+ try { kill(identity.pid, 'SIGKILL'); }
+ catch (error) { if (error.code !== 'ESRCH') { markUncertain('owned descendant could not be killed safely'); } }
+ }
+ await new Promise(resolvePause => setTimeout(resolvePause, 100));
+ }
+ }
+
+ function stopDescendants() {
+ if (!stoppingPromise) {
+ stoppingPromise = stopDescendantsLoop().finally(() => { stoppingPromise = null; });
+ }
+ return stoppingPromise;
+ }
+
+ async function verifyAfterGroups() {
+ for (;;) {
+ const census = observe();
+ tick();
+ if (census && (!uncertain || timedOut()) && !censusFailure) {
+ const liveRoots = [...roots].some(pid => {
+ const row = current.get(pid);
+ const knownRoot = known.get(pid);
+ return row && knownRoot?.birth === row.birth && !/^Z/u.test(row.state);
+ });
+ if (!liveRoots && !liveDescendants().length && (!pendingIdentities.size || timedOut())) return;
+ }
+ if (!warned) { warn(reason || 'owned process identity remains live'); warned = true; }
+ await new Promise(resolvePause => setTimeout(resolvePause, 1_000));
+ }
+ }
+
+ timer = setInterval(() => { if (!busy) observe(); }, intervalMs);
+ timer.unref();
+ return {
+ nodeOptions: journal.nodeOptions,
+ /** Describe the last observed cleanup state without treating it as a fresh ownership proof. */
+ cleanupState() {
+ return { reason: uncertainReason || reason, uncertain, censusFailure, knownCount: known.size,
+ pending: [...pendingIdentities].slice(0, 16).map(([pid, identity]) => ({ pid, birth: identity.birth })),
+ live: liveDescendants().slice(0, 16).map(({ pid, birth, group }) => ({ pid, birth, group })) };
+ },
+ registerRoot,
+ /** Take one census now; a test drives custody through it. */
+ observe,
+ stopDescendants,
+ /** Revalidate a process group immediately before signaling; null means census ambiguity. */
+ signalOwnedGroup,
+ verifyAfterGroups,
+ stopMonitoring() { clearInterval(timer); },
+ };
+}
diff --cc src/runner-worker.js
index 2fd9a042,99fd5ef2..be89c2fc
--- a/src/runner-worker.js
+++ b/src/runner-worker.js
@@@ -29,9 -32,14 +29,10 @@@ import
isAbsolute,
join,
} from 'node:path';
-import {
- fileURLToPath,
- pathToFileURL,
-} from 'node:url';
+import { fileURLToPath } from 'node:url';
import * as resourceApi from './resources.js';
+import { createDescendantTracker } from './runner-custody.js';
+ import { Refused } from './refused.js';
-import { parseProcessCensus } from './process-census.js';
-import { childJournalPreloadSource } from './child-journal-preload.js';
import {
gateStages,
stagedScript,
@@@ -196,7 -205,7 +198,7 @@@ async function readSpec()
tagNames.add(tag.name);
if (tag.chain.length ? tag.chain[0] !== tag.object : tag.object !== tag.peeledCommit) throw new Error('invalid tag manifest');
}
- const runtimeNames = ['runner-custody.js', 'resources.js', 'git.js', 'machine-home.js', 'refused.js', 'process-census.js', 'queue-settings.js', 'queue-order.js', 'sandbox-denial.js', 'gate-stages.js'];
- const runtimeNames = ['resources.js', 'git.js', 'machine-home.js', 'refused.js', 'process-census.js', 'queue-settings.js', 'queue-order.js', 'sandbox-denial.js', 'gate-stages.js', 'child-journal-preload.js'];
++ const runtimeNames = ['runner-custody.js', 'resources.js', 'git.js', 'machine-home.js', 'refused.js', 'process-census.js', 'queue-settings.js', 'queue-order.js', 'sandbox-denial.js', 'gate-stages.js', 'child-journal-preload.js'];
if (Object.keys(value.runtime.files).length !== runtimeNames.length
|| runtimeNames.some(name => typeof value.runtime.files[name] !== 'string')) throw new Error('invalid work specification');
if (createHash('sha256').update(JSON.stringify(value.runtime.files)).digest('hex') !== value.runtime.sha256) throw new Error('invalid trusted runtime');
diff --cc src/runners.js
index 15def19d,87fe2b32..3936fa28
--- a/src/runners.js
+++ b/src/runners.js
@@@ -63,7 -67,7 +67,7 @@@ const MAX_BLOB_BYTES = 8 * 1024 ** 2
const MAX_HISTORY_TAGS = 1_024;
const MAX_TAG_CHAIN = 8;
const TOOL_ROOT = fileURLToPath(new URL('../', import.meta.url));
- const RUNTIME_NAMES = ['runner-custody.js', 'resources.js', 'git.js', 'machine-home.js', 'refused.js', 'process-census.js', 'queue-settings.js', 'queue-order.js', 'sandbox-denial.js', 'gate-stages.js'];
-const RUNTIME_NAMES = ['resources.js', 'git.js', 'machine-home.js', 'refused.js', 'process-census.js', 'queue-settings.js', 'queue-order.js', 'sandbox-denial.js', 'gate-stages.js', 'child-journal-preload.js'];
++const RUNTIME_NAMES = ['runner-custody.js', 'resources.js', 'git.js', 'machine-home.js', 'refused.js', 'process-census.js', 'queue-settings.js', 'queue-order.js', 'sandbox-denial.js', 'gate-stages.js', 'child-journal-preload.js'];
/** Validate an SSH destination as one argument, never a command or an SSH option. */
export function validateRunner(value) {
diff --cc test/runners.test.js
index 0343377d,38f35782..e5e25795
--- a/test/runners.test.js
+++ b/test/runners.test.js
@@@ -1505,144 -1506,116 +1508,13 @@@ test('a queued run executes on a regist
test('a queued run executes on a registered runner and its result comes back [C7] (parent death drains a child that leaves its group after the first census)',
{ timeout: 90_000 }, t => parentDeathDrainFixture(t, parentDeathTools, { lateGroup: true })); // budget: same fixture ceiling as the ordinary parent-death case.
-test('a queued run executes on a registered runner and its result comes back [C7] (unverified cleanup times out as a named red result)', { timeout: 90_000 }, async (t) => { // budget: bounded 30-second remote cleanup timeout plus startup, transport and fixture cleanup ceiling.
- const dir = realpathSync(mkdtempSync(join(tmpdir(), 'pullboard-runner-cleanup-timeout-')));
- const home = join(dir, 'home');
- const machineHome = join(home, '.pullboard');
- const remoteHome = join(dir, 'worker-home');
- const remoteStateRoot = join(remoteHome, '.pullboard-runner');
- const remoteMachineHome = join(remoteHome, '.pullboard');
- const remoteDatabase = join(remoteMachineHome, 'resources.sqlite');
- const repo = join(dir, 'repo');
- const worker = join(dir, 'second-machine');
- const bin = join(dir, 'bin');
- const requests = join(dir, 'ssh-requests.jsonl');
- const bodyRan = join(remoteStateRoot, 'cleanup-timeout-body-ran');
- const starts = join(dir, 'ssh-starts.jsonl');
- let commandChild;
- t.after(async () => {
- if (commandChild && commandChild.exitCode === null && commandChild.signalCode === null) {
- try { process.kill(-commandChild.pid, 'SIGTERM'); }
- catch (error) { if (error.code !== 'ESRCH') t.diagnostic(`fixture process-group cleanup: ${error.message}`); }
- await once(commandChild, 'close').catch(error => t.diagnostic(`fixture child close: ${error.message}`));
- }
- if (existsSync(remoteDatabase)) {
- try { await waitForCondition(() => readRemoteGateHolders(remoteDatabase).length === 0,
- 'fixture remote physical gate holder cleanup'); }
- catch (error) { t.diagnostic(error.message); }
- }
- rmSync(dir, { recursive: true, force: true });
- });
- for (const folder of [repo, worker, bin, home, machineHome, remoteHome, remoteStateRoot, remoteMachineHome]) {
- mkdirSync(folder, { recursive: true });
- }
- writeFileSync(join(machineHome, 'settings.json'), JSON.stringify({ gateSlots: 1 }));
- const env = {
- ...process.env, HOME: home, PULLBOARD_HOME: home, PULLBOARD_MACHINE_HOME: machineHome,
- PATH: `${bin}:${process.env.PATH}`, GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_NOSYSTEM: '1',
- GIT_AUTHOR_NAME: 'Runner Cleanup Fixture', GIT_AUTHOR_EMAIL: '[email protected]',
- GIT_COMMITTER_NAME: 'Runner Cleanup Fixture', GIT_COMMITTER_EMAIL: '[email protected]',
- };
- const initialized = runFixtureChild('git', ['init', '-q', '-b', 'main'], { cwd: repo, env, encoding: 'utf8' });
- assert.equal(initialized.status, 0, initialized.context);
- writeFileSync(join(repo, 'pullboard.json'), JSON.stringify({ gate: 'node runner-cleanup-timeout.js' }));
- writeFileSync(join(repo, 'tracked.txt'), 'cleanup timeout fixture\n');
- writeFileSync(join(repo, 'runner-cleanup-timeout.js'), `import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
-import { dirname, join } from 'node:path';
-import { fileURLToPath } from 'node:url';
-const option = /(?:^|\\s)--import=(?:"([^"]+)"|'([^']+)'|(\\S+))/u.exec(process.env.NODE_OPTIONS || '');
-if (!option) throw new Error('runner preload path was not passed to the committed job');
-const preload = fileURLToPath(option[1] || option[2] || option[3]);
-const runDirectory = dirname(preload);
-const journalDirectory = join(runDirectory, 'child-journal');
-const runnerRoot = dirname(dirname(runDirectory));
-if (!existsSync(journalDirectory)) throw new Error('private child journal is missing');
-mkdirSync(runnerRoot, { recursive: true });
-writeFileSync(join(journalDirectory, 'overflow'), 'fixture journal overflow\\n');
-writeFileSync(join(runnerRoot, 'cleanup-timeout-body-ran'), 'ran\\n');
-console.log('RUNNER_CLEANUP_TIMEOUT_BODY_RAN');
-`);
- git(repo, env, 'add', 'pullboard.json', 'tracked.txt', 'runner-cleanup-timeout.js');
- git(repo, env, 'commit', '-q', '-m', 'test: prepare runner cleanup timeout fixture');
- installSshStandIn({ bin, worker, home: remoteHome, machineHome: remoteMachineHome, requests, starts });
- const added = cli(repo, env, 'runner', 'add', HOST, '--slots', '1', '--browser', '1', '--json');
- assert.equal(added.status, 0, added.context);
-
- const previousHome = process.env.PULLBOARD_HOME;
- const previousMachineHome = process.env.PULLBOARD_MACHINE_HOME;
- process.env.PULLBOARD_HOME = home;
- process.env.PULLBOARD_MACHINE_HOME = machineHome;
- let held;
- try {
- held = await takeResource({ name: 'gate', capacity: 1, scope: 'machine', root: repo,
- agent: 'cleanup-timeout-local-holder', item: 'cleanup-timeout-local-holder', kind: 'fixture-hold', browser: true });
- // The real CLI must refuse this one response as RUNNER_CLEANUP_TIMEOUT; a missing or
- // incomplete transport result would take the retry path and could incorrectly run locally.
- commandChild = startToolingCli(repo, env, ['gate', '--json'], { detached: true });
- const command = observe(commandChild);
- const completed = await command.closed;
- assert.equal(completed.status, 1, `${completed.stderr}\n${completed.stdout}`);
- const response = JSON.parse(completed.stdout.trim());
- assert.equal(response.error?.code, 'RUNNER_CLEANUP_TIMEOUT', JSON.stringify(response));
- assert.match(response.error.message, /child spawn journal overflow or write failure/u,
- 'the named refusal retains the worker reason instead of losing it at the controller');
- const jobId = /inspect job ([A-Za-z0-9_-]+)/u.exec(response.error.message)?.[1];
- assert.ok(jobId, 'the refusal identifies the exact failed job for inspection');
- const retained = readdirSync(join(remoteStateRoot, 'jobs')).find(name => name.startsWith(jobId + '-'));
- assert.ok(retained, 'a cleanup timeout retains its private job diagnostics');
- const diagnosis = JSON.parse(readFileSync(join(remoteStateRoot, 'jobs', retained, 'cleanup-timeout.json'), 'utf8'));
- assert.equal(diagnosis.id, jobId);
- assert.match(diagnosis.reason, /child spawn journal overflow or write failure/u);
- assert.ok(diagnosis.stage, 'the timeout records which cleanup step was running');
- assert.ok(existsSync(bodyRan), 'the committed command ran to completion and wrote its own overflow marker');
- assert.equal(readFileSync(starts, 'utf8').trim().split('\n').length, 1, 'the refused cleanup result is not retried or sent to a local runner');
- await waitForCondition(() => readRemoteGateHolders(remoteDatabase).length === 0,
- 'remote physical gate holder releases after the named cleanup timeout');
- assert.equal(readRemoteGateHolders(join(machineHome, 'resources.sqlite')).length, 1,
- 'the separate local fixture holder remains intact until explicitly released');
- } finally {
- if (commandChild && commandChild.exitCode === null && commandChild.signalCode === null) {
- try { process.kill(-commandChild.pid, 'SIGTERM'); }
- catch (error) { if (error.code !== 'ESRCH') throw error; }
- await once(commandChild, 'close').catch(() => {});
- }
- if (held) held.release();
- if (previousHome === undefined) delete process.env.PULLBOARD_HOME;
- else process.env.PULLBOARD_HOME = previousHome;
- if (previousMachineHome === undefined) delete process.env.PULLBOARD_MACHINE_HOME;
- else process.env.PULLBOARD_MACHINE_HOME = previousMachineHome;
- }
-});
+const orphanTools = { cli, git, installSshStandIn, observe, startToolingCli, processIsAlive, HOST };
+test('a runner job leaves no process behind, detached ones included [C7]',
+ { timeout: 150_000 }, t => runnerOrphanFixture(t, orphanTools)); // budget: one plain runner round trip, then the declared 30-second remote cleanup timeout and a local rerun.
+test('a runner job leaves no process behind, and never signals a process it cannot prove is the job\'s [C7]',
+ { timeout: 30_000 }, t => unprovenProcessFixture(t, { createDescendantTracker, parseProcessCensus })); // budget: scripted census, no real processes.
- /** Read one machine pool's browser row through the public resource API, selecting that machine's home. */
- function browserPoolAt(machineHome, root) {
- const previous = process.env.PULLBOARD_MACHINE_HOME;
- process.env.PULLBOARD_MACHINE_HOME = machineHome;
- try { return listResources({ scope: 'machine', root }).find(resource => resource.name === 'browser') ?? null; }
- finally {
- if (previous === undefined) delete process.env.PULLBOARD_MACHINE_HOME;
- else process.env.PULLBOARD_MACHINE_HOME = previous;
- }
- }
-
- /**
- * Poll an observable fixture state, failing at once with the controller's output if a run exits first.
- * Without the early exit a refused remote setup would only show up as an unexplained timeout.
- */
- async function waitForRunState(runs, check, label) {
- const deadline = Date.now() + 30_000; // budget: bound remote checkout plus admission for two real workers.
- while (true) {
- const state = check();
- if (state) return state;
- for (const run of runs) {
- if (run.result) throw new Error(`${label}: run ${run.item} exited first\n${run.result.stdout.slice(-4_000)}\n${run.result.stderr.slice(-4_000)}`);
- }
- if (Date.now() >= deadline) throw new Error(`${label}: state did not arrive`);
- await new Promise(resolve => setTimeout(resolve, 20)); // budget: polling interval for a public pool state.
- }
- }
-
- test('a runner admits as many browser runs as its machine allows [C7,O5]', async (t) => {
- const dir = realpathSync(mkdtempSync(join(tmpdir(), 'pullboard-runner-browser-slots-')));
- t.after(() => rmSync(dir, { recursive: true, force: true }));
- const home = join(dir, 'home');
- const machineHome = join(home, '.pullboard');
- const remoteHome = join(dir, 'worker-home');
- const remoteMachineHome = join(remoteHome, '.pullboard');
- const repo = join(dir, 'repo');
- const worker = join(dir, 'second-machine');
- const bin = join(dir, 'bin');
- const markers = join(dir, 'markers');
- const requests = join(dir, 'ssh-requests.jsonl');
- for (const folder of [repo, worker, bin, home, machineHome, remoteHome, remoteMachineHome, markers]) mkdirSync(folder, { recursive: true });
- writeFileSync(join(machineHome, 'settings.json'), JSON.stringify({ gateSlots: 1 }));
- const env = {
- ...process.env, HOME: home, PULLBOARD_HOME: home, PULLBOARD_MACHINE_HOME: machineHome,
- PATH: `${bin}:${process.env.PATH}`, GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_NOSYSTEM: '1',
- GIT_AUTHOR_NAME: 'Runner Fixture', GIT_AUTHOR_EMAIL: '[email protected]',
- GIT_COMMITTER_NAME: 'Runner Fixture', GIT_COMMITTER_EMAIL: '[email protected]',
- };
- const remoteEnv = { ...env, HOME: remoteHome, PULLBOARD_HOME: remoteHome, PULLBOARD_MACHINE_HOME: remoteMachineHome };
- const initialized = runFixtureChild('git', ['init', '-q', '-b', 'main'], { cwd: repo, env, encoding: 'utf8' });
- assert.equal(initialized.status, 0, initialized.context);
- writeFileSync(join(repo, 'pullboard.json'), JSON.stringify({ gate: 'true' }));
- writeFileSync(join(repo, 'tracked.txt'), 'committed runner source\n');
- git(repo, env, 'add', 'pullboard.json', 'tracked.txt');
- git(repo, env, 'commit', '-q', '-m', 'test: prepare runner browser capacity fixture');
- installSshStandIn({ bin, worker, home: remoteHome, machineHome: remoteMachineHome, requests });
- // The controller may send two browser runs at once, so only the runner's own pool can hold one back.
- const added = cli(repo, env, 'runner', 'add', HOST, '--slots', '2', '--browser', '2', '--json');
- assert.equal(added.status, 0, added.context);
-
- const previousHome = process.env.PULLBOARD_HOME;
- const previousMachineHome = process.env.PULLBOARD_MACHINE_HOME;
- process.env.PULLBOARD_HOME = home;
- process.env.PULLBOARD_MACHINE_HOME = machineHome;
- const runs = [];
- /** Start one browser run that holds its runner slot until the test writes its release marker. */
- const startHeld = (item) => {
- const started = join(markers, `${item}.started`);
- const release = join(markers, `${item}.release`);
- const code = `const fs=require('node:fs'); fs.writeFileSync(${JSON.stringify(started)}, 'started\\n');
- const timer=setInterval(()=>{ if (fs.existsSync(${JSON.stringify(release)})) clearInterval(timer); }, 5);`;
- const observed = observe(startQueueController(repo, env, `node -e ${shellQuote(code)}`, { item, queueMarker: null, firstItem: null }));
- const run = { item, started, release, ...observed, result: null };
- observed.closed.then(result => { run.result = result; });
- runs.push(run);
- return run;
- };
- /** Release runs and check that each one came back green from the runner. */
- const finish = async (pair) => {
- for (const run of pair) writeFileSync(run.release, 'release\n');
- for (const result of await Promise.all(pair.map(run => run.closed))) {
- assert.equal(result.status, 0, `${result.stdout}${result.stderr}`);
- assert.equal(gateResult(result.stdout).runner, HOST, 'the run executed on the runner');
- }
- };
- let held;
- try {
- // Fill the controller's own gate so every run below goes to the runner.
- held = await takeResource({ name: 'gate', capacity: 1, scope: 'machine', root: repo,
- agent: 'fixture-local-holder', item: 'local-holder', kind: 'force-registered-runner' });
-
- // A runner machine that set nothing admits one browser run; the second waits in its pool.
- const unset = [startHeld('unset-a'), startHeld('unset-b')];
- const queued = await waitForRunState(unset, () => {
- const browser = browserPoolAt(remoteMachineHome, repo);
- return browser?.holders.length === 1 && browser.line.length === 1 ? browser : null;
- }, 'unset runner: one browser holder and one browser waiter');
- assert.equal(queued.capacity, 1, 'a runner with no browserSlots setting keeps one browser slot');
- const [first, second] = queued.holders[0].item === 'unset-a' ? unset : [...unset].reverse();
- assert.equal(queued.line[0].item, second.item);
- await waitForRunState(unset, () => existsSync(first.started), 'unset runner: the admitted run starts');
- assert.equal(existsSync(second.started), false, 'the second browser run has not started while the first holds the only slot');
- writeFileSync(first.release, 'release\n');
- await waitForRunState([second], () => existsSync(second.started), 'unset runner: the waiting run starts after the first ends');
- await finish(unset);
-
- // The runner machine sets browserSlots 2: both browser runs hold its slots at the same time.
- const setting = cli(repo, remoteEnv, 'settings', 'browserSlots', '2', '--json');
- assert.equal(setting.status, 0, setting.context);
- const configured = [startHeld('two-a'), startHeld('two-b')];
- await waitForRunState(configured, () => configured.every(run => existsSync(run.started)), 'runner set to 2: both browser runs start');
- const both = browserPoolAt(remoteMachineHome, repo);
- assert.equal(both.capacity, 2, 'the runner pool uses its machine browserSlots setting');
- assert.deepEqual(both.holders.map(holder => holder.item).sort(), ['two-a', 'two-b']);
- assert.equal(both.line.length, 0, 'no browser run waits on a runner set to 2');
- await finish(configured);
- } finally {
- held?.release();
- for (const run of runs) writeFileSync(run.release, 'cleanup-release\n');
- // A remote setup that never answers must not hold the test file open: stop its controller.
- const settled = Promise.all(runs.map(run => run.closed));
- const ceiling = new Promise(resolve => { const timer = setTimeout(resolve, 20_000); timer.unref(); }); // budget: released runs finish well inside this.
- await Promise.race([settled, ceiling]);
- for (const run of runs) if (!run.result) run.child.kill('SIGTERM');
- await settled;
- if (previousHome === undefined) delete process.env.PULLBOARD_HOME;
- else process.env.PULLBOARD_HOME = previousHome;
- if (previousMachineHome === undefined) delete process.env.PULLBOARD_MACHINE_HOME;
- else process.env.PULLBOARD_MACHINE_HOME = previousMachineHome;
- }
- });
-
+ test('a runner uses the capacity it was registered with [C7,O5]', { timeout: 180_000 }, t => runnerUsesRegisteredCapacity(t, { cli, git, observe, startQueueController, gateResult })); // budget: two rounds of four concurrent runner round trips.
test('review655 a registered runner honors the same live person waiver at landing [B3,R3,C7]', { timeout: 120_000 }, async (t) => {
const { createE2eHelpers } = await import('./e2e-helpers.js');