Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

fix(runners): drain busy runners before removing them [C7]

Corey Olson420cc9c3fa6e
diff --git a/src/commands/runner.js b/src/commands/runner.js
index 24303721..e4cf32a9 100644
--- a/src/commands/runner.js
+++ b/src/commands/runner.js
@@ -7,15 +7,16 @@ import {
loadMachineSettings,
removeRunner,
} from '../settings.js';
+import { runnerResourceStatus } from '../resources.js';
/** List, add or remove machine-level SSH capacity without requiring a board identity or runner account. */
export function runnerCommand(io, { first, second, rest, values }) {
if (first === 'list') {
if (second || rest.length || Object.keys(values).some(key => key !== 'json')) throw new Refused('USAGE', 'pullboard runner list');
// Registered capacity is what each runner enforces, whatever its own settings say (#1066).
- const runners = loadMachineSettings().runners ?? [];
+ const runners = runnerResourceStatus(loadMachineSettings().runners ?? []);
io.result?.({ runners });
- io.say(runners.length ? runners.map(runner => `${runner.host}: ${runner.slots} slots, ${runner.browser} browser`).join('\n') : 'no runners registered');
+ io.say(runners.length ? runners.map(runner => `${runner.host}: ${runner.slots} slots, ${runner.browser} browser${runner.draining ? ' (draining)' : ''}`).join('\n') : 'no runners registered');
return 0;
}
if (first === 'remove') {
@@ -24,7 +25,7 @@ export function runnerCommand(io, { first, second, rest, values }) {
}
const runner = removeRunner(second);
io.result?.({ runner });
- io.say(`removed runner ${runner.host}`);
+ io.say(`${runner.draining ? 'draining' : 'removed'} runner ${runner.host}`);
return 0;
}
if (first !== 'add' || !second || rest.length || !/^[1-9]\d*$/u.test(values.slots ?? '')
diff --git a/src/resources.js b/src/resources.js
index f866d5ee..bed4c979 100644
--- a/src/resources.js
+++ b/src/resources.js
@@ -125,6 +125,7 @@ function openQueue(file) {
CREATE INDEX IF NOT EXISTS holder_name ON holder(name);
CREATE TABLE IF NOT EXISTS waiter (ticket INTEGER PRIMARY KEY AUTOINCREMENT, token TEXT UNIQUE NOT NULL, name TEXT NOT NULL, pid INTEGER NOT NULL, started TEXT NOT NULL DEFAULT '', agent TEXT NOT NULL, repo TEXT NOT NULL, since TEXT NOT NULL, heartbeat INTEGER NOT NULL, landing INTEGER NOT NULL DEFAULT 0, item_check INTEGER NOT NULL DEFAULT 0, browser INTEGER NOT NULL DEFAULT 0, item TEXT NOT NULL DEFAULT '', kind TEXT NOT NULL DEFAULT '', remote INTEGER NOT NULL DEFAULT 0, short_eligible INTEGER NOT NULL DEFAULT 0);
CREATE TABLE IF NOT EXISTS runner_unavailable (host TEXT PRIMARY KEY, since TEXT NOT NULL, detail TEXT NOT NULL DEFAULT '');
+ CREATE TABLE IF NOT EXISTS runner_draining (host TEXT PRIMARY KEY, runner TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS waiter_name_ticket ON waiter(name, ticket);`);
for (const table of ['holder', 'waiter']) {
const columns = db.prepare(`PRAGMA table_info(${table})`).all().map(({ name }) => name);
@@ -436,7 +437,7 @@ function gateAdmission(db, ticket, capacity, shortCapacity, runners, order = ARR
browsers: remoteHolders.filter(holder => holder.browser === 1).length,
}];
}));
- const unavailable = new Set(db.prepare('SELECT host FROM runner_unavailable').all().map(row => row.host));
+ const unavailable = new Set(db.prepare('SELECT host FROM runner_unavailable UNION SELECT host FROM runner_draining').all().map(row => row.host));
/** Keep an already holding process from taking another slot ahead of a different landing. */
const blockedByLanding = waiter => holders.some(holder => holder.pid === waiter.pid)
&& waiters.some(other => other.landing === 1 && other.pid !== waiter.pid);
@@ -483,18 +484,25 @@ export function runnerResourceStatus(runners) {
const configured = validatedRunners(runners);
const db = open(machineResourceFile());
try {
- return update(db, () => configured.map(runner => {
- const holders = db.prepare('SELECT agent, repo, since, browser, item, kind, runner FROM holder WHERE name = ? ORDER BY since, token')
- .all(runnerResourceName(runner.host));
- const unavailable = db.prepare('SELECT since, detail FROM runner_unavailable WHERE host = ?').get(runner.host);
- return { ...runner, inUse: holders.length, browserInUse: holders.filter(holder => holder.browser === 1).length,
- unavailable: Boolean(unavailable), ...(unavailable ? { unavailableSince: unavailable.since, unavailableReason: unavailable.detail } : {}), holders };
- }));
+ return update(db, () => {
+ const visible = new Map(configured.map(runner => [runner.host, runner]));
+ for (const row of db.prepare('SELECT host, runner FROM runner_draining').all()) {
+ visible.set(row.host, { ...JSON.parse(row.runner), draining: true });
+ }
+ return [...visible.values()].flatMap(runner => {
+ const holders = db.prepare('SELECT agent, repo, since, browser, item, kind, runner FROM holder WHERE name = ? ORDER BY since, token')
+ .all(runnerResourceName(runner.host));
+ if (runner.draining && !holders.length) return [];
+ const unavailable = db.prepare('SELECT since, detail FROM runner_unavailable WHERE host = ?').get(runner.host);
+ return [{ ...runner, inUse: holders.length, browserInUse: holders.filter(holder => holder.browser === 1).length,
+ unavailable: Boolean(unavailable), ...(unavailable ? { unavailableSince: unavailable.since, unavailableReason: unavailable.detail } : {}), holders }];
+ });
+ });
} finally { db.close(); }
}
-/** Serialize runner-setting changes against active remote leases and clear that host's quarantine. */
-export function updateRunnerSettings(host, persist) {
+/** Serialize runner changes with admission; removal may drain active leases instead of refusing. */
+export function updateRunnerSettings(host, persist, { drain = false } = {}) {
if (typeof host !== 'string' || !host.trim() || host.trim() !== host || typeof persist !== 'function') {
throw new Refused('BAD_RUNNER_SETTINGS', 'a runner host and settings persistence callback are required');
}
@@ -503,13 +511,23 @@ export function updateRunnerSettings(host, persist) {
return update(db, () => {
const resourceName = runnerResourceName(host);
const held = db.prepare('SELECT COUNT(*) AS count FROM holder WHERE name = ?').get(resourceName).count;
- if (held) throw new Refused('RUNNER_BUSY', `runner ${host} has ${held} active slot(s); wait for them to finish before changing its settings`);
+ if (held && !drain) throw new Refused('RUNNER_BUSY', `runner ${host} has ${held} active slot(s); wait for them to finish before changing its settings`);
+ const draining = drain && held && db.prepare('SELECT runner FROM runner_draining WHERE host = ?').get(host);
+ if (draining) return { ...JSON.parse(draining.runner), draining: true };
const result = persist();
+ if (drain && held) {
+ // Dispatch settings no longer name this host. Keep its tombstone even after the last
+ // release, so a stale provider cannot offer it again; explicit registration clears it.
+ db.prepare('INSERT INTO runner_draining(host, runner) VALUES (?, ?) ON CONFLICT(host) DO UPDATE SET runner = excluded.runner')
+ .run(host, JSON.stringify(result));
+ return { ...result, draining: true };
+ }
if (Number.isSafeInteger(result?.slots) && result.slots > 0) {
db.prepare('INSERT INTO resource(name, capacity) VALUES (?, ?) ON CONFLICT(name) DO UPDATE SET capacity = excluded.capacity')
.run(resourceName, result.slots);
}
db.prepare('DELETE FROM runner_unavailable WHERE host = ?').run(host);
+ db.prepare('DELETE FROM runner_draining WHERE host = ?').run(host);
return result;
});
} finally { db.close(); }
diff --git a/src/settings.js b/src/settings.js
index 8e72d632..ec0c26f7 100644
--- a/src/settings.js
+++ b/src/settings.js
@@ -201,7 +201,7 @@ export function addRunner(value) {
});
}
-/** Remove one registered runner while serialized against its active remote leases. */
+/** Stop dispatching to a runner immediately; admitted jobs keep their leases until they finish. */
export function removeRunner(host) {
return updateRunnerSettings(host, () => {
const current = readMachineSettingsFile(machineSettingsFile()) ?? {};
@@ -215,5 +215,5 @@ export function removeRunner(host) {
const [runner] = runners.splice(index, 1);
writeMachineSettings({ ...current, runners });
return runner;
- });
+ }, { drain: true });
}
diff --git a/test/runner-drain-fixture.js b/test/runner-drain-fixture.js
new file mode 100644
index 00000000..4bf4b4ad
--- /dev/null
+++ b/test/runner-drain-fixture.js
@@ -0,0 +1,143 @@
+/** Busy runner removal stops new dispatches while admitted private jobs finish [C7]. */
+import assert from 'node:assert/strict';
+import { existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { takeResource } from '../src/resources.js';
+import { runFixtureChild } from './fixture-child.js';
+import { HOST, installSshStandIn, shellQuote } from './runner-ssh-fixture.js';
+
+/** Preserve unrelated settings and the established immediate-removal contract for idle runners. */
+export function runnerRemoveKeepsSettings(t, e2e) {
+ const box = e2e.project();
+ const settingsFile = join(box.env.PULLBOARD_MACHINE_HOME, 'settings.json');
+ const removed = { host: 'runner-one.example.invalid', slots: 2, browser: 1, node: 'node', root: '/srv/runner-one' };
+ const remaining = { host: 'runner-two.example.invalid', slots: 1, browser: 0 };
+ const original = { gateSlots: 3, browserSlots: 2, gate: { shortSlots: 1, queue: { reviewsFirst: true } },
+ relay: { offerAtInit: true }, custom: { keep: 'unchanged' } };
+ mkdirSync(box.env.PULLBOARD_MACHINE_HOME, { recursive: true });
+ writeFileSync(settingsFile, JSON.stringify(original, null, 2));
+ const firstAdded = box.run(box.repo, 'runner', 'add', removed.host, '--slots', String(removed.slots), '--browser', String(removed.browser),
+ '--node', removed.node, '--root', removed.root);
+ assert.equal(firstAdded.code, 0, firstAdded.err);
+ const secondAdded = box.run(box.repo, 'runner', 'add', remaining.host, '--slots', String(remaining.slots), '--browser', String(remaining.browser));
+ assert.equal(secondAdded.code, 0, secondAdded.err);
+ const registered = { ...original, runners: [removed, remaining] };
+ assert.deepEqual(JSON.parse(readFileSync(settingsFile, 'utf8')), registered);
+
+ const result = box.run(box.repo, 'runner', 'remove', removed.host);
+ assert.equal(result.code, 0, result.err);
+ assert.ok(result.out.includes(`removed runner ${removed.host}`), result.out);
+ assert.deepEqual(JSON.parse(readFileSync(settingsFile, 'utf8')), { ...original, runners: [remaining] });
+
+ const missing = box.run(box.repo, 'runner', 'remove', removed.host, '--json');
+ assert.equal(missing.code, 1, missing.err);
+ const refusal = JSON.parse(missing.out).error;
+ assert.equal(refusal.code, 'RUNNER_NOT_REGISTERED');
+ assert.ok(refusal.next.includes(remaining.host), refusal.next);
+ assert.ok(!refusal.next.includes(removed.host), refusal.next);
+ assert.deepEqual(JSON.parse(readFileSync(settingsFile, 'utf8')), { ...original, runners: [remaining] });
+}
+
+/** Wait for a visible private-job state, reporting an early controller failure rather than timing out. */
+async function waitFor(runs, check, label) {
+ const deadline = Date.now() + 60_000; // budget: source setup and two private SSH stand-in jobs.
+ while (!check()) {
+ for (const run of runs) if (run.result && run.result.status !== 0) {
+ throw new Error(`${label}: ${run.item} failed\n${run.result.stdout}\n${run.result.stderr}`);
+ }
+ assert.ok(Date.now() < deadline, label);
+ await new Promise(done => setTimeout(done, 20)); // budget: private state observation only.
+ }
+}
+
+/** Removing a runner with two real admitted jobs drains them and sends later work to the local executor. */
+export async function removingBusyRunnerDrains(t, { cli, git, observe, startQueueController, gateResult }) {
+ const directory = realpathSync(mkdtempSync(join(tmpdir(), 'pullboard-runner-drain-')));
+ const repo = join(directory, 'repo'), home = join(directory, 'home');
+ const machineHome = join(home, '.pullboard'), remoteHome = join(directory, 'worker-home');
+ const remoteMachineHome = join(remoteHome, '.pullboard'), worker = join(directory, 'worker');
+ const bin = join(directory, 'bin'), markers = join(directory, 'markers');
+ for (const path of [repo, home, machineHome, remoteHome, remoteMachineHome, worker, bin, markers]) mkdirSync(path, { recursive: true });
+ const env = { ...process.env, HOME: home, PULLBOARD_HOME: home, PULLBOARD_MACHINE_HOME: machineHome,
+ PATH: `${bin}:${process.env.PATH}`, GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_NOSYSTEM: '1',
+ GIT_AUTHOR_NAME: 'Runner Drain Fixture', GIT_AUTHOR_EMAIL: '[email protected]',
+ GIT_COMMITTER_NAME: 'Runner Drain Fixture', GIT_COMMITTER_EMAIL: '[email protected]' };
+ const initialized = runFixtureChild('git', ['init', '-q', '-b', 'main'], { cwd: repo, env, encoding: 'utf8' });
+ assert.equal(initialized.status, 0, initialized.context);
+ writeFileSync(join(repo, 'pullboard.json'), JSON.stringify({ gate: 'true' }));
+ writeFileSync(join(repo, 'tracked.txt'), 'private runner drain source\n');
+ git(repo, env, 'add', 'pullboard.json', 'tracked.txt');
+ git(repo, env, 'commit', '-q', '-m', 'test: prepare runner drain fixture');
+ writeFileSync(join(machineHome, 'settings.json'), JSON.stringify({ gateSlots: 1, browserSlots: 1, custom: { preserve: true } }));
+ installSshStandIn({ bin, worker, home: remoteHome, machineHome: remoteMachineHome, requests: join(directory, 'ssh-requests.jsonl') });
+ const added = cli(repo, env, 'runner', 'add', HOST, '--slots', '2', '--browser', '0', '--json');
+ assert.equal(added.status, 0, added.context);
+ const previousHome = process.env.PULLBOARD_HOME, previousMachineHome = process.env.PULLBOARD_MACHINE_HOME;
+ process.env.PULLBOARD_HOME = home;
+ process.env.PULLBOARD_MACHINE_HOME = machineHome;
+ const runs = [];
+ let local;
+ /** Launch one ordinary production runner round trip, holding its child command on a private release file. */
+ function start(item, queueMarker = null) {
+ const started = join(markers, `${item}.started`), release = join(markers, `${item}.release`);
+ const code = `const fs=require('node:fs');fs.writeFileSync(${JSON.stringify(started)},'started');const timer=setInterval(()=>{if(fs.existsSync(${JSON.stringify(release)}))clearInterval(timer)},10);`;
+ const run = { item, started, release, ...observe(startQueueController(repo, env, `node -e ${shellQuote(code)}`, { item, browser: false, queueMarker, firstItem: null })), result: null };
+ run.closed.then(result => { run.result = result; });
+ runs.push(run);
+ return run;
+ }
+ /** Read the public runner list from this isolated machine, through the real CLI. */
+ function listed() {
+ const result = cli(repo, env, 'runner', 'list', '--json');
+ assert.equal(result.status, 0, result.context);
+ return JSON.parse(result.stdout).runners;
+ }
+ try {
+ local = await takeResource({ name: 'gate', capacity: 1, scope: 'machine', root: repo, agent: 'drain-local-holder', kind: 'fixture' });
+ const first = start('first'), second = start('second');
+ await waitFor(runs, () => [first, second].every(run => existsSync(run.started)), 'both jobs start on the two-slot runner');
+ const removed = cli(repo, env, 'runner', 'remove', HOST, '--json');
+ assert.equal(removed.status, 0, removed.context);
+ assert.equal(JSON.parse(removed.stdout).runner.draining, true, 'busy remove succeeds and declares draining');
+ assert.equal(listed().find(runner => runner.host === HOST)?.draining, true, 'runner list keeps the active draining host visible');
+ assert.equal(JSON.parse(readFileSync(join(machineHome, 'settings.json'), 'utf8')).custom.preserve, true);
+ const thirdQueued = join(markers, 'third.queued');
+ const third = start('third', thirdQueued);
+ await waitFor(runs, () => existsSync(thirdQueued), 'the post-remove job joins the queue while the local and remote slots are occupied');
+ writeFileSync(first.release, 'release');
+ const firstResult = await first.closed;
+ assert.equal(firstResult.status, 0, `${firstResult.stdout}${firstResult.stderr}`);
+ assert.equal(gateResult(firstResult.stdout).runner, HOST, 'the first admitted job finishes on the removed runner');
+ await waitFor(runs, () => listed().find(runner => runner.host === HOST)?.inUse === 1, 'the second job keeps the host draining after the first releases');
+ assert.equal(existsSync(third.started), false, 'later work has not taken the now-free draining runner slot');
+ local.release(); local = null;
+ await waitFor(runs, () => existsSync(third.started), 'later work starts locally while the second remote job remains held');
+ writeFileSync(third.release, 'release');
+ const thirdResult = await third.closed;
+ assert.equal(thirdResult.status, 0, `${thirdResult.stdout}${thirdResult.stderr}`);
+ assert.equal(gateResult(thirdResult.stdout).runner, undefined, 'the post-remove job runs locally without a remote execution receipt');
+ writeFileSync(second.release, 'release');
+ const secondResult = await second.closed;
+ assert.equal(secondResult.status, 0, `${secondResult.stdout}${secondResult.stderr}`);
+ assert.equal(gateResult(secondResult.stdout).runner, HOST, 'the second admitted job also finishes normally');
+ assert.ok(!listed().some(runner => runner.host === HOST), 'the runner leaves the list after its final job ends');
+ const registeredAgain = cli(repo, env, 'runner', 'add', HOST, '--slots', '2', '--browser', '0', '--json');
+ assert.equal(registeredAgain.status, 0, registeredAgain.context);
+ assert.ok(listed().some(runner => runner.host === HOST && !runner.draining), 'explicit re-registration clears the drain state');
+ const idleRemove = cli(repo, env, 'runner', 'remove', HOST, '--json');
+ assert.equal(idleRemove.status, 0, idleRemove.context);
+ assert.ok(!listed().some(runner => runner.host === HOST), 'an idle runner still disappears immediately');
+ } finally {
+ local?.release();
+ for (const run of runs) writeFileSync(run.release, 'cleanup-release');
+ const settled = Promise.all(runs.map(run => run.closed));
+ const ceiling = new Promise(done => { const timer = setTimeout(done, 20_000); timer.unref(); }); // budget: already-released private workers finish in this window.
+ await Promise.race([settled, ceiling]);
+ for (const run of runs) if (!run.result) run.child.kill('SIGTERM');
+ await settled;
+ if (previousHome === undefined) delete process.env.PULLBOARD_HOME; else process.env.PULLBOARD_HOME = previousHome;
+ if (previousMachineHome === undefined) delete process.env.PULLBOARD_MACHINE_HOME; else process.env.PULLBOARD_MACHINE_HOME = previousMachineHome;
+ rmSync(directory, { recursive: true, force: true });
+ }
+}
diff --git a/test/runners.test.js b/test/runners.test.js
index 38f35782..682caca8 100644
--- a/test/runners.test.js
+++ b/test/runners.test.js
@@ -24,6 +24,7 @@ import { runnerSourceRefusalCases } from './runner-source-refusal-fixture.js';
import { writeCacheProbe, writeProbe } from './runner-probe-fixture.js';
import { browserFreeRunnerCheck, browserRefusalTextStaysRed } from './runner-browser-free-fixture.js';
import { runnerUsesRegisteredCapacity } from './runner-capacity-fixture.js';
+import { removingBusyRunnerDrains, runnerRemoveKeepsSettings } from './runner-drain-fixture.js';
/** Per-phase deadlines for one runner round trip; together they stay near the 60-second hang ceiling. */
const ROUND_TRIP_DEADLINES = { dispatch: 20_000, 'remote run': 30_000, 'result return': 10_000 }; // budget: a stuck runner fails by name in about a minute, far under the 660-second per-test limit.
@@ -311,36 +312,9 @@ test("the runner's trusted runtime list is closed under its relative imports [C7
}
});
-test('runner remove deletes one runner and keeps the rest [C7]', (t) => {
- const box = e2e.project();
- const settingsFile = join(box.env.PULLBOARD_MACHINE_HOME, 'settings.json');
- const removed = { host: 'runner-one.example.invalid', slots: 2, browser: 1, node: 'node', root: '/srv/runner-one' };
- const remaining = { host: 'runner-two.example.invalid', slots: 1, browser: 0 };
- const original = { gateSlots: 3, browserSlots: 2, gate: { shortSlots: 1, queue: { reviewsFirst: true } },
- relay: { offerAtInit: true }, custom: { keep: 'unchanged' } };
- mkdirSync(box.env.PULLBOARD_MACHINE_HOME, { recursive: true });
- writeFileSync(settingsFile, JSON.stringify(original, null, 2));
- const firstAdded = box.run(box.repo, 'runner', 'add', removed.host, '--slots', String(removed.slots), '--browser', String(removed.browser),
- '--node', removed.node, '--root', removed.root);
- assert.equal(firstAdded.code, 0, firstAdded.err);
- const secondAdded = box.run(box.repo, 'runner', 'add', remaining.host, '--slots', String(remaining.slots), '--browser', String(remaining.browser));
- assert.equal(secondAdded.code, 0, secondAdded.err);
- const registered = { ...original, runners: [removed, remaining] };
- assert.deepEqual(JSON.parse(readFileSync(settingsFile, 'utf8')), registered);
-
- const result = box.run(box.repo, 'runner', 'remove', removed.host);
- assert.equal(result.code, 0, result.err);
- assert.ok(result.out.includes(`removed runner ${removed.host}`), result.out);
- assert.deepEqual(JSON.parse(readFileSync(settingsFile, 'utf8')), { ...original, runners: [remaining] });
-
- const missing = box.run(box.repo, 'runner', 'remove', removed.host, '--json');
- assert.equal(missing.code, 1, missing.err);
- const refusal = JSON.parse(missing.out).error;
- assert.equal(refusal.code, 'RUNNER_NOT_REGISTERED');
- assert.ok(refusal.next.includes(remaining.host), refusal.next);
- assert.ok(!refusal.next.includes(removed.host), refusal.next);
- assert.deepEqual(JSON.parse(readFileSync(settingsFile, 'utf8')), { ...original, runners: [remaining] });
-});
+test('runner remove deletes one runner and keeps the rest [C7]', (t) => runnerRemoveKeepsSettings(t, e2e));
+
+test('removing a busy runner drains it [C7]', (t) => removingBusyRunnerDrains(t, { cli, git, observe, startQueueController, gateResult }));
test('a queued run executes on a registered runner and its result comes back (host checkout isolation) [C7]', async (t) => {
const dir = realpathSync(mkdtempSync(join(tmpdir(), 'pullboard-runner-tooling-isolation-')));