Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

test(runners): cover capacity placement and timing isolation [C7,Q1]

Corey Olson08308c4a4d9e
diff --git a/test/runner-stage-fixture.js b/test/runner-stage-fixture.js
new file mode 100644
index 00000000..031dbed1
--- /dev/null
+++ b/test/runner-stage-fixture.js
@@ -0,0 +1,76 @@
+/** Windowed runner capacity and placement API fixture [C7,O5]. */
+import assert from 'node:assert/strict';
+import { DatabaseSync } from 'node:sqlite';
+import { join } from 'node:path';
+import * as runnerApi from '../src/runners.js';
+
+/** Seed the native private Git/SQLite context before exercising the namespace API. */
+export async function runnerStageFixture(t, e2e) {
+ const box = e2e.project('true');
+ const boardFile = join(box.repo, '.git', 'pullboard', 'board.sqlite');
+ const board = new DatabaseSync(boardFile, { readOnly: true });
+ try {
+ assert.ok(board.prepare("SELECT 1 AS present FROM sqlite_master WHERE type = 'table' AND name = 'event'").get()?.present,
+ 'the fixture has its own initialized native board database');
+ } finally { board.close(); }
+
+ // A missing export fails after the private native setup instead of as an import/setup error.
+ assert.equal(typeof runnerApi.runnerStage, 'function', 'runnerStage is available from the runner namespace');
+ assert.equal(typeof runnerApi.placeRunner, 'function', 'placeRunner is available from the runner namespace');
+
+ const from = Date.parse('2026-10-24T00:00:00.000Z');
+ const to = from + 1_000;
+ const runners = [
+ { id: 'runner-a', kind: 'ssh', slots: { current: 2, limit: 2, elastic: false }, capabilities: { os: 'linux', browser: true }, active: 2 },
+ { id: 'runner-b', kind: 'ci', slots: { current: 2, limit: 2, elastic: false }, capabilities: { os: 'darwin', browser: false }, active: 0 },
+ { id: 'runner-c', kind: 'elastic', slots: { current: 0, limit: 4, elastic: true }, capabilities: { os: 'linux', browser: true }, active: 0 },
+ { id: 'runner-d', kind: 'local', slots: { current: 1, limit: 1, elastic: false }, capabilities: { os: 'linux', browser: true }, active: 1 },
+ { id: 'runner-e', kind: 'ssh', slots: { current: 2, limit: 2, elastic: false }, capabilities: { os: 'windows', browser: true }, active: 1 },
+ ];
+ const at = milliseconds => new Date(milliseconds).toISOString();
+ const receiptRunner = (runner, slots = runner.slots) => ({ id: runner.id, kind: runner.kind,
+ slots: structuredClone(slots), capabilities: { ...runner.capabilities } });
+ const receipts = [
+ { runner: receiptRunner(runners[0]), timing: true, admittedAt: at(from - 500), finishedAt: at(from + 500), waitMs: 10 },
+ { runner: receiptRunner(runners[0]), occupiedSlots: 2, admittedAt: at(from + 500), finishedAt: at(to + 500), waitMs: 50 },
+ { runner: receiptRunner(runners[1], { ...runners[1].slots, current: 1 }), admittedAt: at(from - 5_000), finishedAt: at(from - 4_000), waitMs: 5 },
+ { runner: receiptRunner(runners[3]), admittedAt: at(from - 100), finishedAt: at(from + 300), waitMs: 30 },
+ { runner: receiptRunner(runners[4], { ...runners[4].slots, current: 1 }), admittedAt: at(from + 100), finishedAt: at(to + 100), waitMs: 99 },
+ ];
+ const stage = runnerApi.runnerStage(runners, receipts, { from: at(from), to: at(to) });
+ const byId = id => stage.runners.find(runner => runner.id === id);
+
+ assert.deepEqual(byId('runner-a'), { ...runners[0], busyShare: 1, runsServed: 1 },
+ 'the descriptor is preserved with its clipped capacity share and in-window completions');
+ assert.equal(byId('runner-a').busyShare, 1,
+ 'exclusive timing and an explicitly full-capacity receipt saturate the two-slot runner');
+ assert.equal(byId('runner-a').runsServed, 1, 'only completions inside the half-open window count');
+ assert.equal(byId('runner-b').busyShare, 0, 'idle runner with only out-of-window history has measured zero share');
+ assert.equal(byId('runner-b').runsServed, 0, 'out-of-window completion is excluded');
+ assert.equal(byId('runner-c').busyShare, null, 'zero current capacity has unknown share, not a fabricated zero');
+ assert.equal(byId('runner-c').runsServed, 0);
+ assert.equal(byId('runner-d').busyShare, 0.3, 'a run crossing the left boundary is clipped to the window');
+ assert.equal(byId('runner-d').runsServed, 1);
+ assert.equal(byId('runner-e').busyShare, null, 'a receipt with stale capacity leaves the share unknown');
+ assert.deepEqual(stage.slotWait, { fastQuarterMs: 10, medianMs: 20, slowQuarterMs: 30, worstMs: 30, count: 2 },
+ 'only finite waits for runs completed in the half-open window form the quartiles');
+
+ const linuxBrowser = { capabilities: { os: 'linux', browser: true }, slotWaitTargetMs: 15 };
+ assert.deepEqual(runnerApi.placeRunner(stage, { slotWaitTargetMs: 15 }),
+ { action: 'run', runnerId: 'runner-b' }, 'an unconstrained request picks the least-busy compatible runner');
+ assert.deepEqual(runnerApi.placeRunner(stage, { capabilities: { os: 'darwin', browser: false }, slotWaitTargetMs: 15 }),
+ { action: 'run', runnerId: 'runner-b' }, 'an idle compatible fixed runner runs; an incompatible one is not selected');
+ assert.deepEqual(runnerApi.placeRunner(stage, linuxBrowser),
+ { action: 'scale', runnerId: 'runner-c', slots: 1 }, 'scale the elastic runner only after compatible fixed slots are busy past target');
+
+ const waiting = runnerApi.placeRunner(stage, { ...linuxBrowser, slotWaitTargetMs: 25 });
+ assert.equal(waiting.action, 'wait', 'below the measured wait threshold, keep waiting');
+ const full = structuredClone(stage);
+ const elastic = full.runners.find(runner => runner.id === 'runner-c');
+ elastic.slots.current = elastic.slots.limit;
+ elastic.active = elastic.slots.current;
+ assert.deepEqual(runnerApi.placeRunner(full, linuxBrowser), { action: 'request' },
+ 'request elastic capacity only after all matching fixed and elastic capacity is saturated');
+ assert.deepEqual(runnerApi.placeRunner(stage, { capabilities: { os: 'freebsd', browser: true }, slotWaitTargetMs: 15 }),
+ { action: 'incompatible' }, 'do not scale a pool that cannot satisfy the requested capabilities');
+}
diff --git a/test/runner-timing.test.js b/test/runner-timing.test.js
new file mode 100644
index 00000000..a01bdff2
--- /dev/null
+++ b/test/runner-timing.test.js
@@ -0,0 +1,99 @@
+/** Timing checks hold an entire physical machine, including its short slots [C7,Q1]. */
+import assert from 'node:assert/strict';
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { test } from 'node:test';
+import { createE2eHelpers } from './e2e-helpers.js';
+import { runFixtureChildAsync } from './fixture-child.js';
+import { BROWSER_SLOT_HELD_ENV, GATE_SLOT_HELD_ENV, GATE_SLOT_POOL_ENV, GATE_SLOT_TOKEN_ENV } from '../src/resources.js';
+
+/** A real board and Git checkout with a private two-slot machine and no inherited outer lease. */
+function fixture(t) {
+ const helpers = createE2eHelpers();
+ t.after(helpers.cleanup);
+ const box = helpers.project();
+ mkdirSync(box.env.PULLBOARD_MACHINE_HOME, { recursive: true });
+ writeFileSync(join(box.env.PULLBOARD_MACHINE_HOME, 'settings.json'), JSON.stringify({ gateSlots: 2, browserSlots: 1, gate: { shortSlots: 1 } }));
+ for (const key of [GATE_SLOT_HELD_ENV, GATE_SLOT_POOL_ENV, GATE_SLOT_TOKEN_ENV, BROWSER_SLOT_HELD_ENV]) delete box.env[key];
+ return box;
+}
+
+/** Run the public admission APIs in the private machine and preserve all failed output. */
+async function runFixture(box, body) {
+ const source = `import assert from 'node:assert/strict';
+import { takeResource, listResources, gateSlotEnvironment } from ${JSON.stringify(new URL('../src/resources.js', import.meta.url).href)};
+import { withGateSlot } from ${JSON.stringify(new URL('../src/gate.js', import.meta.url).href)};
+const root = process.cwd();
+${body}`;
+ const result = await runFixtureChildAsync(process.execPath, ['--input-type=module', '-e', source], { cwd: box.repo, env: box.env });
+ assert.equal(result.status, 0, `${result.context}\n${result.stdout}\n${result.stderr}`);
+ return result;
+}
+
+test('timing admission drains ordinary work and excludes ordinary and short slots [C7,Q1]', async t => {
+ const box = fixture(t);
+ await runFixture(box, `
+const leases = [];
+const base = { name: 'gate', capacity: 2, scope: 'machine', root, browser: false };
+try {
+ const ordinary = await takeResource({ ...base, agent: 'ordinary-before' }); leases.push(ordinary);
+ let waiting;
+ const observed = new Promise(resolve => { waiting = resolve; });
+ const pending = takeResource({ ...base, timing: true, agent: 'timing', onWait: state => waiting({ state }) });
+ pending.then(lease => { leases.push(lease); waiting({ lease }); });
+ const first = await observed;
+ assert.ok(first.state, 'timing waits even when one of two ordinary slots is free');
+ assert.equal(listResources({ root }).find(row => row.name === 'gate').holders.length, 1);
+ ordinary.release();
+ const timing = await pending;
+ assert.equal(timing.timing, true, 'the real lease carries exclusive timing admission');
+ const holder = listResources({ root }).find(row => row.name === 'gate').holders;
+ assert.equal(holder.length, 1);
+ assert.equal(holder[0].timing, true);
+ let secondWaiting;
+ const secondObserved = new Promise(resolve => { secondWaiting = resolve; });
+ const next = takeResource({ ...base, agent: 'ordinary-after', shortEligible: true,
+ shortCapacityProvider: () => 1, onWait: state => secondWaiting({ state }) });
+ next.then(lease => { leases.push(lease); secondWaiting({ lease }); });
+ const second = await secondObserved;
+ assert.ok(second.state, 'an exclusive timing lease blocks ordinary and spare short capacity');
+ timing.release();
+ const after = await next;
+ assert.equal(after.timing, false);
+ after.release();
+ assert.equal(listResources({ root }).find(row => row.name === 'gate').holders.length, 0);
+} finally { for (const lease of leases.reverse()) lease.release(); }
+`);
+});
+
+test('timing cannot escalate an inherited ordinary lease and its receipt records actual load [C7,Q1]', async t => {
+ const box = fixture(t);
+ await runFixture(box, `
+const ordinary = await takeResource({ name: 'gate', capacity: 2, scope: 'machine', root });
+try {
+ Object.assign(process.env, gateSlotEnvironment(ordinary));
+ await assert.rejects(withGateSlot(root, () => assert.fail('ordinary lease must not run a timing check'),
+ { timing: true, remote: false }), { code: 'TIMING_SLOT_REQUIRED' });
+} finally {
+ ordinary.release();
+ for (const key of ['PULLBOARD_GATE_SLOT_HELD', 'PULLBOARD_GATE_SLOT_POOL', 'PULLBOARD_GATE_SLOT_TOKEN', 'PULLBOARD_BROWSER_SLOT_HELD']) delete process.env[key];
+}
+let receipt;
+await withGateSlot(root, async lease => {
+ assert.equal(lease.timing, true);
+ Object.assign(process.env, gateSlotEnvironment(lease));
+ try { assert.equal(await withGateSlot(root, inherited => inherited.timing, { timing: true, remote: false }), true); }
+ finally { for (const key of ['PULLBOARD_GATE_SLOT_HELD', 'PULLBOARD_GATE_SLOT_POOL', 'PULLBOARD_GATE_SLOT_TOKEN', 'PULLBOARD_BROWSER_SLOT_HELD']) delete process.env[key]; }
+}, { timing: true, remote: false, onReceipt: value => { receipt = value; } });
+assert.equal(receipt.timing, true);
+assert.equal(receipt.runner.kind, 'local');
+for (const sample of [receipt.load.start, receipt.load.end]) {
+ assert.ok(Number.isFinite(Date.parse(sample.at)), 'load has an executor timestamp');
+ assert.equal(sample.average.length, 3);
+ assert.ok(sample.average.every(Number.isFinite));
+ assert.ok(Number.isSafeInteger(sample.cpus) && sample.cpus > 0);
+}
+assert.ok(Date.parse(receipt.load.end.at) >= Date.parse(receipt.load.start.at));
+assert.ok(Date.parse(receipt.finishedAt) >= Date.parse(receipt.admittedAt));
+`);
+});
diff --git a/test/runner-tooling-fixture.js b/test/runner-tooling-fixture.js
new file mode 100644
index 00000000..0e489bd5
--- /dev/null
+++ b/test/runner-tooling-fixture.js
@@ -0,0 +1,190 @@
+/** Shared registered-runner test tooling and bounded round-trip helpers. */
+import assert from 'node:assert/strict';
+import { copyFileSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+import { pathToFileURL } from 'node:url';
+import { runFixtureChild, runFixtureChildAsync, startFixtureChild } from './fixture-child.js';
+import { stopGroup } from './runner-process-group.js';
+
+const BIN = resolve(import.meta.dirname, '../bin/pullboard.js');
+export const TOOLING_SNAPSHOTS = new Map();
+export const ROUND_TRIP_DEADLINES = { dispatch: 20_000, 'remote run': 30_000, 'result return': 10_000 }; // budget: a stuck runner fails by name in about a minute, far under the 660-second per-test limit.
+
+/** Run Git in the private source repository and preserve useful failure context. */
+export function git(repo, env, ...args) {
+ const result = runFixtureChild('git', args, { cwd: repo, env, encoding: 'utf8' });
+ assert.equal(result.status, 0, result.context);
+ return result.stdout.trim();
+}
+
+/** Copy current tooling bytes into a private committed checkout for runner source validation. */
+export function committedToolingSnapshot(sourceRoot, env) {
+ const key = realpathSync(sourceRoot);
+ const cached = TOOLING_SNAPSHOTS.get(key);
+ if (cached) return cached.root;
+ const directory = mkdtempSync(join(tmpdir(), 'pullboard-runner-tooling-'));
+ const root = join(directory, 'tooling');
+ const home = join(directory, 'home');
+ mkdirSync(root, { recursive: true });
+ mkdirSync(home, { recursive: true });
+ try {
+ copyFileSync(join(key, 'package.json'), join(root, 'package.json'));
+ cpSync(join(key, 'src'), join(root, 'src'), { recursive: true });
+ cpSync(join(key, 'bin'), join(root, 'bin'), { recursive: true });
+ const gitEnv = { ...env, HOME: home, PULLBOARD_HOME: join(home, '.pullboard'),
+ PULLBOARD_MACHINE_HOME: join(home, '.pullboard'), GIT_CONFIG_GLOBAL: '/dev/null',
+ GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_COUNT: '0',
+ GIT_AUTHOR_NAME: 'Runner Tooling Fixture', GIT_AUTHOR_EMAIL: '[email protected]',
+ GIT_COMMITTER_NAME: 'Runner Tooling Fixture', GIT_COMMITTER_EMAIL: '[email protected]' };
+ for (const key of ['GIT_CONFIG_PARAMETERS', 'GIT_DIR', 'GIT_WORK_TREE', 'GIT_INDEX_FILE', 'GIT_COMMON_DIR']) delete gitEnv[key];
+ git(root, gitEnv, 'init', '-q', '-b', 'main');
+ git(root, gitEnv, 'add', 'package.json', 'src', 'bin');
+ git(root, gitEnv, 'commit', '-q', '-m', 'test: snapshot current runner tooling');
+ } catch (error) {
+ process.stderr.write(`retained incomplete runner tooling snapshot at ${directory}: ${error.message}\n`);
+ throw error;
+ }
+ TOOLING_SNAPSHOTS.set(key, { root, directory, active: 0, retained: false });
+ return root;
+}
+
+/** Clean retained tooling copies after fixture children have stopped. */
+export function cleanupToolingSnapshots() {
+ for (const record of TOOLING_SNAPSHOTS.values()) {
+ if (record.active || record.retained) {
+ process.stderr.write(`retained runner tooling snapshot at ${record.directory}\n`);
+ continue;
+ }
+ rmSync(record.directory, { recursive: true, force: true });
+ }
+ TOOLING_SNAPSHOTS.clear();
+}
+
+/** Spawn the exact checked-in CLI from a fixture checkout. */
+export function cli(repo, env, ...args) {
+ return runFixtureChild(process.execPath, [BIN, ...args], { cwd: repo, env, encoding: 'utf8' });
+}
+
+
+/** Start the current CLI from the same clean committed tooling snapshot used by API fixtures. */
+export function startToolingCli(repo, env, args, { toolingRoot = resolve(import.meta.dirname, '..'), ...options } = {}) {
+ const sourceRoot = realpathSync(toolingRoot);
+ const bin = join(committedToolingSnapshot(sourceRoot, env), 'bin', 'pullboard.js');
+ const snapshot = TOOLING_SNAPSHOTS.get(sourceRoot);
+ snapshot.active++;
+ try {
+ const child = startFixtureChild(process.execPath, [bin, ...args], {
+ cwd: repo, env, stdio: ['ignore', 'pipe', 'pipe'], ...options,
+ });
+ let settled = false;
+ const finish = (status, signal, failed) => {
+ if (settled) return;
+ settled = true;
+ snapshot.active--;
+ if (failed || status !== 0 || signal) snapshot.retained = true;
+ };
+ child.once('close', (status, signal) => finish(status, signal, false));
+ child.once('error', () => finish(null, null, true));
+ return child;
+ } catch (error) {
+ snapshot.active--;
+ snapshot.retained = true;
+ throw error;
+ }
+}
+
+/** Count the stand-in's phase markers; a missing file means nothing was dispatched yet. */
+export function phaseCounts(file) {
+ const lines = existsSync(file) ? readFileSync(file, 'utf8').split('\n') : [];
+ return { dispatched: lines.filter(line => line === 'dispatched').length, returned: lines.filter(line => line === 'returned').length };
+}
+
+/**
+ * Wait for a runner round trip phase by phase: dispatch (until the stand-in is reached), remote run
+ * (until the worker returns) and result return (until the controller exits). A phase past its
+ * deadline stops the whole controller group and throws an error naming that phase, so a stuck
+ * runner fails the test in about a minute instead of holding it to the per-test limit. The caller
+ * passes the marker counts it read before spawning the child: a fast child can dispatch before this
+ * function first runs, and a baseline read here would swallow that dispatch.
+ */
+export async function awaitRoundTrip(child, closed, phasesFile, before, deadlines = ROUND_TRIP_DEADLINES, firstReceiptAccepted = null) {
+ const limits = { ...ROUND_TRIP_DEADLINES, ...deadlines };
+ let done = null;
+ closed.then(result => { done = result; });
+ const timings = [];
+ // The marker file may already hold earlier round trips of the same fixture; count only this one.
+ assert.ok(!phasesFile || before, 'awaitRoundTrip needs the phase baseline read before the child was spawned');
+ let phase = 'dispatch';
+ let phaseStarted = Date.now();
+ let observed = { dispatched: 0, returned: 0 };
+ while (!done) {
+ const total = phasesFile ? phaseCounts(phasesFile) : { dispatched: 0, returned: 0 };
+ const counts = phasesFile ? { dispatched: total.dispatched - before.dispatched, returned: total.returned - before.returned } : total;
+ // The observer saves the accepted first receipt before preparing its paired dispatch. That real
+ // controller-side witness ends result delivery; a worker's "returned" marker alone cannot do so.
+ const redispatch = counts.dispatched === 1 && counts.returned === 1 && firstReceiptAccepted?.();
+ const next = redispatch ? 'dispatch' : counts.dispatched > counts.returned ? 'remote run' : counts.dispatched > 0 ? 'result return' : 'dispatch';
+ // A returned attempt and its paired dispatch can arrive in one poll without changing the phase's name.
+ // Each real marker advance starts a fresh clock, while a phase with no new marker remains bounded.
+ if (next !== phase || counts.dispatched > observed.dispatched || counts.returned > observed.returned) {
+ timings.push(`${phase} ${Date.now() - phaseStarted}ms (attempt ${observed.dispatched})`);
+ phase = next;
+ phaseStarted = Date.now();
+ observed = counts;
+ }
+ const spent = Date.now() - phaseStarted;
+ if (spent > limits[phase]) {
+ await stopGroup(child, closed);
+ throw new Error(`runner round trip stuck in phase "${phase}": no progress after ${spent} ms (deadline ${limits[phase]} ms); earlier phases: ${timings.join(', ') || 'none'}\n${fixtureResultContext(done ?? {})}`);
+ }
+ await Promise.race([closed, new Promise(resolve => setTimeout(resolve, 50))]); // budget: polling interval for the stand-in's phase markers.
+ }
+ timings.push(`${phase} ${Date.now() - phaseStarted}ms`);
+ process.stderr.write(`runner round trip phases: ${timings.join(', ')}\n`);
+ return done;
+}
+
+/**
+ * Run a gate command through its real lease and return the profile/result emitted by the API child.
+ * `requests` names the stand-in's request log, whose .phases file times the round trip by phase.
+ */
+export async function runThroughGate(repo, env, command, { browser = true, item = '663', kind = 'check', landing = false, toolingRoot = resolve(import.meta.dirname, '..'), requests, deadlines } = {}) {
+ const sourceRoot = realpathSync(toolingRoot);
+ const gateUrl = pathToFileURL(join(committedToolingSnapshot(sourceRoot, env), 'src', 'gate.js')).href;
+ const snapshot = TOOLING_SNAPSHOTS.get(sourceRoot);
+ snapshot.active++;
+ const script = `const { runProfiledShell, withGateSlot } = await import(${JSON.stringify(gateUrl)});
+const result = await withGateSlot(${JSON.stringify(repo)}, lease => runProfiledShell(${JSON.stringify(repo)}, ${JSON.stringify(command)}, { gateLease: lease, landing: ${landing} }),
+ { browser: ${browser}, item: ${JSON.stringify(item)}, kind: ${JSON.stringify(kind)}, landing: ${landing}, agent: 'runner-fixture' });
+process.stdout.write(result.output);
+process.stdout.write('PULLBOARD_RUN_RESULT ' + JSON.stringify({ isGreen: result.isGreen, runner: result.profile.executionRunner, sandboxBlocked: result.sandboxBlocked, profile: result.profile }) + '\\n');
+process.exitCode = result.isGreen ? 0 : 1;
+`;
+ try {
+ // Read the phase baseline before spawning, so a dispatch that beats the first poll still counts.
+ const phases = requests ? `${requests}.phases` : '';
+ const before = phases ? phaseCounts(phases) : null;
+ // Its own process group, so a stuck phase can stop the controller, the stand-in and the worker together.
+ const closed = runFixtureChildAsync(process.execPath, ['--input-type=module', '-e', script], { cwd: repo, env, detached: true });
+ const result = await awaitRoundTrip(closed.child, closed, phases, before, deadlines);
+ if (result.status !== 0) snapshot.retained = true;
+ return result;
+ } catch (error) {
+ snapshot.retained = true;
+ throw error;
+ } finally { snapshot.active--; }
+}
+
+/** Parse the bounded final API line without rewriting the command's raw TAP output. */
+export function gateResult(stdout) {
+ const line = stdout.split(/\r?\n/u).findLast(value => value.startsWith('PULLBOARD_RUN_RESULT '));
+ assert.ok(line, stdout);
+ return JSON.parse(line.slice('PULLBOARD_RUN_RESULT '.length));
+}
+
+/** Include bounded command output when a remote fixture assertion fails. */
+export function fixtureResultContext(result) {
+ return `${result.context}\nstdout tail:\n${String(result.stdout ?? '').slice(-4_000)}\nstderr tail:\n${String(result.stderr ?? '').slice(-4_000)}`;
+}
+
diff --git a/test/runners.test.js b/test/runners.test.js
index c3d584d9..3ae685c0 100644
--- a/test/runners.test.js
+++ b/test/runners.test.js
@@ -27,194 +27,18 @@ import { writeCacheProbe, writeProbe } from './runner-probe-fixture.js';
import { browserFreeRunnerCheck, browserRefusalTextStaysRed } from './runner-browser-free-fixture.js';
import { runnerUsesRegisteredCapacity } from './runner-capacity-fixture.js';
import { removingBusyRunnerDrains, runnerRemoveKeepsSettings } from './runner-drain-fixture.js';
+import { awaitRoundTrip, cleanupToolingSnapshots, cli, committedToolingSnapshot, fixtureResultContext, gateResult, git, phaseCounts, ROUND_TRIP_DEADLINES, runThroughGate, startToolingCli, TOOLING_SNAPSHOTS } from './runner-tooling-fixture.js';
+import { runnerStageFixture } from './runner-stage-fixture.js';
-/** Per-phase deadlines for one runner round trip; together they stay near the 60-second hang ceiling. */
-const ROUND_TRIP_DEADLINES = { dispatch: 20_000, 'remote run': 30_000, 'result return': 10_000 }; // budget: a stuck runner fails by name in about a minute, far under the 660-second per-test limit.
const PAIRED_RERUN_FIXTURE = { createE2eHelpers, cli, observe, startToolingCli, awaitRoundTrip, phaseCounts,
ROUND_TRIP_DEADLINES, HOST, shellQuote, installSshStandIn, processesUnder, stopGroup };
const BIN = resolve(import.meta.dirname, '../bin/pullboard.js');
const TEMP = [];
-const TOOLING_SNAPSHOTS = new Map();
const e2e = createE2eHelpers();
after(e2e.cleanup);
after(cleanupFixtureChildren);
-
-/** Run Git in the private source repository and preserve useful failure context. */
-function git(repo, env, ...args) {
- const result = runFixtureChild('git', args, { cwd: repo, env, encoding: 'utf8' });
- assert.equal(result.status, 0, result.context);
- return result.stdout.trim();
-}
-
-/** Copy current tooling bytes into a private committed checkout for runner source validation. */
-function committedToolingSnapshot(sourceRoot, env) {
- const key = realpathSync(sourceRoot);
- const cached = TOOLING_SNAPSHOTS.get(key);
- if (cached) return cached.root;
- const directory = mkdtempSync(join(tmpdir(), 'pullboard-runner-tooling-'));
- const root = join(directory, 'tooling');
- const home = join(directory, 'home');
- mkdirSync(root, { recursive: true });
- mkdirSync(home, { recursive: true });
- try {
- copyFileSync(join(key, 'package.json'), join(root, 'package.json'));
- cpSync(join(key, 'src'), join(root, 'src'), { recursive: true });
- cpSync(join(key, 'bin'), join(root, 'bin'), { recursive: true });
- const gitEnv = { ...env, HOME: home, PULLBOARD_HOME: join(home, '.pullboard'),
- PULLBOARD_MACHINE_HOME: join(home, '.pullboard'), GIT_CONFIG_GLOBAL: '/dev/null',
- GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_COUNT: '0',
- GIT_AUTHOR_NAME: 'Runner Tooling Fixture', GIT_AUTHOR_EMAIL: '[email protected]',
- GIT_COMMITTER_NAME: 'Runner Tooling Fixture', GIT_COMMITTER_EMAIL: '[email protected]' };
- for (const key of ['GIT_CONFIG_PARAMETERS', 'GIT_DIR', 'GIT_WORK_TREE', 'GIT_INDEX_FILE', 'GIT_COMMON_DIR']) delete gitEnv[key];
- git(root, gitEnv, 'init', '-q', '-b', 'main');
- git(root, gitEnv, 'add', 'package.json', 'src', 'bin');
- git(root, gitEnv, 'commit', '-q', '-m', 'test: snapshot current runner tooling');
- } catch (error) {
- process.stderr.write(`retained incomplete runner tooling snapshot at ${directory}: ${error.message}\n`);
- throw error;
- }
- TOOLING_SNAPSHOTS.set(key, { root, directory, active: 0, retained: false });
- return root;
-}
-
-after(() => {
- for (const record of TOOLING_SNAPSHOTS.values()) {
- if (record.active || record.retained) {
- process.stderr.write(`retained runner tooling snapshot at ${record.directory}\n`);
- continue;
- }
- rmSync(record.directory, { recursive: true, force: true });
- }
- TOOLING_SNAPSHOTS.clear();
-});
-
-/** Spawn the exact checked-in CLI from a fixture checkout. */
-function cli(repo, env, ...args) {
- return runFixtureChild(process.execPath, [BIN, ...args], { cwd: repo, env, encoding: 'utf8' });
-}
-
-
-/** Start the current CLI from the same clean committed tooling snapshot used by API fixtures. */
-function startToolingCli(repo, env, args, { toolingRoot = resolve(import.meta.dirname, '..'), ...options } = {}) {
- const sourceRoot = realpathSync(toolingRoot);
- const bin = join(committedToolingSnapshot(sourceRoot, env), 'bin', 'pullboard.js');
- const snapshot = TOOLING_SNAPSHOTS.get(sourceRoot);
- snapshot.active++;
- try {
- const child = startFixtureChild(process.execPath, [bin, ...args], {
- cwd: repo, env, stdio: ['ignore', 'pipe', 'pipe'], ...options,
- });
- let settled = false;
- const finish = (status, signal, failed) => {
- if (settled) return;
- settled = true;
- snapshot.active--;
- if (failed || status !== 0 || signal) snapshot.retained = true;
- };
- child.once('close', (status, signal) => finish(status, signal, false));
- child.once('error', () => finish(null, null, true));
- return child;
- } catch (error) {
- snapshot.active--;
- snapshot.retained = true;
- throw error;
- }
-}
-
-/** Count the stand-in's phase markers; a missing file means nothing was dispatched yet. */
-function phaseCounts(file) {
- const lines = existsSync(file) ? readFileSync(file, 'utf8').split('\n') : [];
- return { dispatched: lines.filter(line => line === 'dispatched').length, returned: lines.filter(line => line === 'returned').length };
-}
-
-/**
- * Wait for a runner round trip phase by phase: dispatch (until the stand-in is reached), remote run
- * (until the worker returns) and result return (until the controller exits). A phase past its
- * deadline stops the whole controller group and throws an error naming that phase, so a stuck
- * runner fails the test in about a minute instead of holding it to the per-test limit. The caller
- * passes the marker counts it read before spawning the child: a fast child can dispatch before this
- * function first runs, and a baseline read here would swallow that dispatch.
- */
-async function awaitRoundTrip(child, closed, phasesFile, before, deadlines = ROUND_TRIP_DEADLINES, firstReceiptAccepted = null) {
- const limits = { ...ROUND_TRIP_DEADLINES, ...deadlines };
- let done = null;
- closed.then(result => { done = result; });
- const timings = [];
- // The marker file may already hold earlier round trips of the same fixture; count only this one.
- assert.ok(!phasesFile || before, 'awaitRoundTrip needs the phase baseline read before the child was spawned');
- let phase = 'dispatch';
- let phaseStarted = Date.now();
- let observed = { dispatched: 0, returned: 0 };
- while (!done) {
- const total = phasesFile ? phaseCounts(phasesFile) : { dispatched: 0, returned: 0 };
- const counts = phasesFile ? { dispatched: total.dispatched - before.dispatched, returned: total.returned - before.returned } : total;
- // The observer saves the accepted first receipt before preparing its paired dispatch. That real
- // controller-side witness ends result delivery; a worker's "returned" marker alone cannot do so.
- const redispatch = counts.dispatched === 1 && counts.returned === 1 && firstReceiptAccepted?.();
- const next = redispatch ? 'dispatch' : counts.dispatched > counts.returned ? 'remote run' : counts.dispatched > 0 ? 'result return' : 'dispatch';
- // A returned attempt and its paired dispatch can arrive in one poll without changing the phase's name.
- // Each real marker advance starts a fresh clock, while a phase with no new marker remains bounded.
- if (next !== phase || counts.dispatched > observed.dispatched || counts.returned > observed.returned) {
- timings.push(`${phase} ${Date.now() - phaseStarted}ms (attempt ${observed.dispatched})`);
- phase = next;
- phaseStarted = Date.now();
- observed = counts;
- }
- const spent = Date.now() - phaseStarted;
- if (spent > limits[phase]) {
- await stopGroup(child, closed);
- throw new Error(`runner round trip stuck in phase "${phase}": no progress after ${spent} ms (deadline ${limits[phase]} ms); earlier phases: ${timings.join(', ') || 'none'}\n${fixtureResultContext(done ?? {})}`);
- }
- await Promise.race([closed, new Promise(resolve => setTimeout(resolve, 50))]); // budget: polling interval for the stand-in's phase markers.
- }
- timings.push(`${phase} ${Date.now() - phaseStarted}ms`);
- process.stderr.write(`runner round trip phases: ${timings.join(', ')}\n`);
- return done;
-}
-
-/**
- * Run a gate command through its real lease and return the profile/result emitted by the API child.
- * `requests` names the stand-in's request log, whose .phases file times the round trip by phase.
- */
-async function runThroughGate(repo, env, command, { browser = true, item = '663', kind = 'check', landing = false, toolingRoot = resolve(import.meta.dirname, '..'), requests, deadlines } = {}) {
- const sourceRoot = realpathSync(toolingRoot);
- const gateUrl = pathToFileURL(join(committedToolingSnapshot(sourceRoot, env), 'src', 'gate.js')).href;
- const snapshot = TOOLING_SNAPSHOTS.get(sourceRoot);
- snapshot.active++;
- const script = `const { runProfiledShell, withGateSlot } = await import(${JSON.stringify(gateUrl)});
-const result = await withGateSlot(${JSON.stringify(repo)}, lease => runProfiledShell(${JSON.stringify(repo)}, ${JSON.stringify(command)}, { gateLease: lease, landing: ${landing} }),
- { browser: ${browser}, item: ${JSON.stringify(item)}, kind: ${JSON.stringify(kind)}, landing: ${landing}, agent: 'runner-fixture' });
-process.stdout.write(result.output);
-process.stdout.write('PULLBOARD_RUN_RESULT ' + JSON.stringify({ isGreen: result.isGreen, runner: result.profile.executionRunner, sandboxBlocked: result.sandboxBlocked, profile: result.profile }) + '\\n');
-process.exitCode = result.isGreen ? 0 : 1;
-`;
- try {
- // Read the phase baseline before spawning, so a dispatch that beats the first poll still counts.
- const phases = requests ? `${requests}.phases` : '';
- const before = phases ? phaseCounts(phases) : null;
- // Its own process group, so a stuck phase can stop the controller, the stand-in and the worker together.
- const closed = runFixtureChildAsync(process.execPath, ['--input-type=module', '-e', script], { cwd: repo, env, detached: true });
- const result = await awaitRoundTrip(closed.child, closed, phases, before, deadlines);
- if (result.status !== 0) snapshot.retained = true;
- return result;
- } catch (error) {
- snapshot.retained = true;
- throw error;
- } finally { snapshot.active--; }
-}
-
-/** Parse the bounded final API line without rewriting the command's raw TAP output. */
-function gateResult(stdout) {
- const line = stdout.split(/\r?\n/u).findLast(value => value.startsWith('PULLBOARD_RUN_RESULT '));
- assert.ok(line, stdout);
- return JSON.parse(line.slice('PULLBOARD_RUN_RESULT '.length));
-}
-
-/** Include bounded command output when a remote fixture assertion fails. */
-function fixtureResultContext(result) {
- return `${result.context}\nstdout tail:\n${String(result.stdout ?? '').slice(-4_000)}\nstderr tail:\n${String(result.stderr ?? '').slice(-4_000)}`;
-}
+after(cleanupToolingSnapshots);
/** Wait for the marker itself: a missed file notification must not hide observable readiness. */
async function waitForMarker(path) {
@@ -1490,6 +1314,8 @@ test('a runner job leaves no process behind, detached ones included [C7]',
test('a runner job leaves no process behind, and never signals a process it cannot prove is the job\'s [C7]',
{ timeout: 30_000 }, t => unprovenProcessFixture(t, { createDescendantTracker, parseProcessCensus })); // budget: scripted census, no real processes.
+test('machines report busy share and a new run goes to the idle one [C7,O5]', { timeout: 30_000 }, t => runnerStageFixture(t, e2e));
+
test('a runner uses the capacity it was registered with [C7,O5]', { timeout: 180_000 }, t => runnerUsesRegisteredCapacity(t, { cli, git, observe, startQueueController, gateResult })); // budget: two rounds of four concurrent runner round trips.
test('review655 a registered runner honors the same live person waiver at landing [B3,R3,C7]', { timeout: 120_000 }, async (t) => {