Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.
JavaScript agentic-codingai-agentsai-governanceautonomous-agentsclaude-codeclicode-reviewcodexcoding-agentsdeveloper-toolsgit-hooksllmlocal-firstmulti-agentsoftware-factoryspec-driven-developmentverificationvibe-coding
fix(runner): an EPERM group signal never aborts lease release [C7]
Corey Olson2920bbf8d412
diff --git a/src/runner-worker.js b/src/runner-worker.jsindex 4d0786ee..0eaf63a6 100644--- a/src/runner-worker.js+++ b/src/runner-worker.js@@ -641,7 +641,13 @@ function createDescendantTracker(directory) {});if (!ownedMember) return false;try { process.kill(-pgid, signal); }- catch (error) { if (error.code === 'ESRCH') return false; throw error; }+ catch (error) {+ if (error.code === 'ESRCH') return false;+ // macOS answers EPERM for a group whose members exited but are not yet reaped. Throwing here escaped cleanup and+ // skipped the lease release (#888); retry on a fresh census instead, as for any other ambiguity.+ if (error.code === 'EPERM') return null;+ throw error;+ }return true;}diff --git a/test/runner-parent-death-fixture.js b/test/runner-parent-death-fixture.jsindex c58f1fe2..31f7d9f2 100644--- a/test/runner-parent-death-fixture.js+++ b/test/runner-parent-death-fixture.js@@ -2,6 +2,7 @@import assert from 'node:assert/strict';import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, writeFileSync } from 'node:fs';import { join } from 'node:path';+import { pathToFileURL } from 'node:url';import { tmpdir } from 'node:os';import { DatabaseSync } from 'node:sqlite';import { takeResource } from '../src/resources.js';@@ -10,10 +11,12 @@ import { readRemoteGateHolders, waitForCondition } from './runner-fixture-state./*** Keep the original custody assertions identical in ordinary and forced ordering cases: `holdRelease` forces a real- * release write conflict, and `chatty` makes the command keep writing output, so some always arrives after the- * transport drops (#888: a forwarded write then waited forever for a drain from a destroyed stdout).+ * release write conflict, `chatty` makes the command keep writing output, so some always arrives after the+ * transport drops (#888: a forwarded write then waited forever for a drain from a destroyed stdout), and `epermOnce`+ * makes the worker's repeat signal to one process group answer EPERM, as macOS did when the command group finished+ * exiting during the TERM grace loop (#888); the launcher lingers 300 ms after TERM so that repeat signal is sent.*/-export async function parentDeathDrainFixture(t, tools, { holdRelease = false, chatty = false } = {}) {+export async function parentDeathDrainFixture(t, tools, { holdRelease = false, chatty = false, epermOnce = false } = {}) {const { cli, git, installSshStandIn, observe, startQueueController, waitForMarker, processIsAlive, shellQuote, HOST } = tools;const dir = realpathSync(mkdtempSync(join(tmpdir(), 'pullboard-runner-parent-death-')));t.after(() => rmSync(dir, { recursive: true, force: true }));@@ -32,6 +35,7 @@ export async function parentDeathDrainFixture(t, tools, { holdRelease = false, cconst releaseChild = join(dir, 'release-detached-child');const childGone = join(dir, 'detached-child-gone');const workerLog = join(dir, 'worker-stderr.log');+ const epermAnswered = join(dir, 'eperm-answered');let releaseLock;/** The worker's own account of a stalled release: its runner status lines and any cleanup files in its private job (#888). */function workerCleanupReport() {@@ -74,6 +78,7 @@ const child = spawn(process.execPath, [require.resolve('./remote-detached-child.});child.unref();${chatty ? "setInterval(() => process.stdout.write('launcher output\\n'), 20);" : 'setInterval(() => {}, 1_000);'}+${epermOnce ? "process.on('SIGTERM', () => setTimeout(() => process.exit(0), 300));" : ''}`);writeFileSync(join(repo, 'remote-detached-child.js'), `const { existsSync, writeFileSync } = require('node:fs');const [ready, pidFile, term, release, gone] = process.argv.slice(2);@@ -95,7 +100,26 @@ setInterval(() => {}, 1_000);`);git(repo, env, 'add', 'pullboard.json', 'tracked.txt', 'remote-parent-death-launcher.js', 'remote-detached-child.js');git(repo, env, 'commit', '-q', '-m', 'test: prepare committed detached-child launchers');- installSshStandIn({ bin, worker, home: remoteHome, machineHome: remoteMachineHome, requests, stderrLog: workerLog });+ let nodeOptions = '';+ if (epermOnce) {+ const preload = join(dir, 'eperm-once.mjs');+ writeFileSync(preload, `import { writeFileSync } from 'node:fs';+const kill = process.kill.bind(process);+const groups = new Map();+let answered = false;+process.kill = (pid, signal) => {+ if (pid < 0) groups.set(pid, (groups.get(pid) ?? 0) + 1);+ if (!answered && pid < 0 && groups.get(pid) === 2) {+ answered = true;+ writeFileSync(${JSON.stringify(epermAnswered)}, 'eperm\\n');+ throw Object.assign(new Error('kill EPERM'), { code: 'EPERM', errno: -1, syscall: 'kill' });+ }+ return kill(pid, signal);+};+`);+ nodeOptions = '--import=' + pathToFileURL(preload).href;+ }+ installSshStandIn({ bin, worker, home: remoteHome, machineHome: remoteMachineHome, requests, stderrLog: workerLog, nodeOptions });const added = cli(repo, env, 'runner', 'add', HOST, '--slots', '1', '--browser', '1', '--json');assert.equal(added.status, 0, added.context);@@ -175,6 +199,7 @@ setInterval(() => {}, 1_000);t.diagnostic(message);t.diagnostic('worker cleanup report ' + JSON.stringify(workerCleanupReport()));} });+ if (epermOnce) assert.equal(existsSync(epermAnswered), true, 'a repeat group signal really answered EPERM before the release');const controller = await remoteRun.closed;remoteRun = null;assert.equal(controller.signal, 'SIGTERM', 'the owned API controller alone received SIGTERM');diff --git a/test/runners.test.js b/test/runners.test.jsindex 5ff99806..9082997e 100644--- a/test/runners.test.js+++ b/test/runners.test.js@@ -88,9 +88,10 @@ function cli(repo, env, ...args) {* Make a local ssh stand-in that executes the actual worker command in a second private machine.* It appends "dispatched" and "returned" to <requests>.phases so a round trip can be timed by phase,* and `hold` ('dispatch', 'run' or 'return') keeps it alive at that phase, as a stuck runner would.- * `stderrLog` also copies the worker's stderr, so a test can read its result receipt.+ * `stderrLog` also copies the worker's stderr, so a test can read its result receipt, and `nodeOptions` reaches the+ * worker's own Node process, so a test can force an ordering inside it.*/-function installSshStandIn({ bin, worker, home, machineHome, requests, starts, nodeCommand = 'node', dropFirst = false, dropAfterMarker, hold = '', stderrLog }) {+function installSshStandIn({ bin, worker, home, machineHome, requests, starts, nodeCommand = 'node', dropFirst = false, dropAfterMarker, hold = '', stderrLog, nodeOptions = '' }) {const ssh = join(bin, 'ssh');const source = `#!/usr/bin/env nodeimport { appendFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs';@@ -157,7 +158,8 @@ ${dropFirst ? `if (host.endsWith('-drop') && call === 1) {process.stdin.resume();} else` : ''} {const env = { PATH: process.env.PATH, HOME: home, TMPDIR: process.env.TMPDIR,- PULLBOARD_HOME: home, PULLBOARD_MACHINE_HOME: machineHome, LANG: 'C', LC_ALL: 'C', TZ: 'UTC' };+ PULLBOARD_HOME: home, PULLBOARD_MACHINE_HOME: machineHome, LANG: 'C', LC_ALL: 'C', TZ: 'UTC',+ ...(${JSON.stringify(nodeOptions)} ? { NODE_OPTIONS: ${JSON.stringify(nodeOptions)} } : {}) };// Replace the marker-driven shell so the injected HUP reaches the actual worker process.const child = spawn('/bin/sh', ['-c', remoteCommand], { cwd: worker, env,stdio: ['pipe', 'pipe', 'pipe'] });@@ -1675,6 +1677,9 @@ test('parent death retries a failed remote holder deletion after the real databatest('a queued run executes on a registered runner and its result comes back [C7] (parent death drains its worker while the command keeps writing output)',{ timeout: 90_000 }, t => parentDeathDrainFixture(t, parentDeathTools, { chatty: true })); // budget: same fixture ceiling as the ordinary parent-death case.+test('a queued run executes on a registered runner and its result comes back [C7] (parent death drains its worker when a group signal answers EPERM)',+ { timeout: 90_000 }, t => parentDeathDrainFixture(t, parentDeathTools, { epermOnce: true })); // budget: same fixture ceiling as the ordinary parent-death case.+test('a queued run executes on a registered runner and its result comes back [C7] (unverified cleanup times out as a named red result)', { timeout: 90_000 }, async (t) => { // budget: bounded 30-second remote cleanup timeout plus startup, transport and fixture cleanup ceiling.const dir = realpathSync(mkdtempSync(join(tmpdir(), 'pullboard-runner-cleanup-timeout-')));const home = join(dir, 'home');