Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.
JavaScript agentic-codingai-agentsai-governanceautonomous-agentsclaude-codeclicode-reviewcodexcoding-agentsdeveloper-toolsgit-hooksllmlocal-firstmulti-agentsoftware-factoryspec-driven-developmentverificationvibe-coding
fix(runner): retry holder deletion after parent death [C7]
Corey Olson4def1315381e
diff --git a/src/resources.js b/src/resources.jsindex 9f4c376b..58440f62 100644--- a/src/resources.js+++ b/src/resources.js@@ -766,15 +766,13 @@ export async function takeResource(options) {heartbeat.unref();};startHeartbeat();- /** Release this holder and close its SQLite handle unless ownership was transferred. */+ /** Commit this holder's deletion before closing; a failed update remains live and retryable. */const release = () => {if (released || transferred) return;+ update(db, () => db.prepare('DELETE FROM holder WHERE token = ?').run(token));released = true;clearInterval(heartbeat);- try {- update(db, () => db.prepare('DELETE FROM holder WHERE token = ?').run(token));- }- finally { db.close(); }+ db.close();};/** Refresh this lease, or refuse if another process already evicted it. */const renew = () => {diff --git a/src/runner-worker.js b/src/runner-worker.jsindex 9cecf8d9..e609b008 100644--- a/src/runner-worker.js+++ b/src/runner-worker.js@@ -88,6 +88,29 @@ function reportCleanupTimeout() {process.stderr.write(`PULLBOARD_RUNNER_CLEANUP_TIMEOUT ${workerId}\n`);}+/** Retry a drained holder's real deletion instead of abandoning its row after a transient write failure. */+async function releaseDrainedLease(lease) {+ let reported = false;+ for (;;) {+ try { lease.release(); return; }+ catch {+ reportCleanupTimeout();+ if (!reported) {+ // Parent death can close the transport before stderr arrives; retain this real failed attempt in the private job.+ if (cleanupDirectory) {+ try { writeFileSync(join(cleanupDirectory, 'release-pending.json'),+ JSON.stringify({ id: workerId, stage: 'lease release', pending: true, observedAt: new Date().toISOString() }) + '\n', { mode: 0o600 }); }+ catch { /* The live retry and named cleanup timeout remain authoritative if diagnostic storage fails. */ }+ }+ process.stderr.write(`PULLBOARD_RUNNER_RELEASE_PENDING ${workerId}\n`);+ reported = true;+ }+ // The lease and its database stay open until deletion commits; the normal cleanup budget reports a persistent failure.+ await new Promise(resolve => setTimeout(resolve, 250));+ }+ }+}+/** Parse and strictly validate the controller's one-shot work specification. */async function readSpec() {let buffered = Buffer.alloc(0);@@ -1242,7 +1265,7 @@ async function main() {}if (lease) {cleanupStage = 'lease release';- try { lease.release(); } catch { /* Retaining a lease on cleanup error is safer than speculative deletion. */ }+ await releaseDrainedLease(lease);}cleanupStage = 'job files';if (runDirectory && !cleanupTimedOut) { try { rmSync(runDirectory, { recursive: true, force: true }); } catch {} }diff --git a/test/runner-fixture-state.js b/test/runner-fixture-state.jsnew file mode 100644index 00000000..e3dcc0d5--- /dev/null+++ b/test/runner-fixture-state.js@@ -0,0 +1,62 @@+/** Shared observation and wait helpers for private runner fixtures [C7]. */+import { DatabaseSync } from 'node:sqlite';++/** Read the remote machine's live gate holders without changing its shared admission state. */+export function readRemoteGateHolders(file) {+ const db = new DatabaseSync(file, { readOnly: true });+ try {+ db.exec('PRAGMA busy_timeout = 10000');+ return db.prepare("SELECT token, pid, agent, browser, item, kind FROM holder WHERE name = 'gate'").all();+ } finally { db.close(); }+}++/**+ * Poll an observable state until the fixture's own signal expires; callers without a signal+ * retain their existing 20-second fallback. Timeout diagnostics describe the last observation.+ * @param {() => boolean} check+ * @param {string} label+ * @param {{signal?: AbortSignal, state?: () => unknown, onTimeout?: (message: string) => void}} options+ */+export async function waitForCondition(check, label, { signal, state, onTimeout } = {}) {+ const started = performance.now();+ return new Promise((resolve, reject) => {+ let timer;+ let finished = false;+ /** Remove the poll and abort listener on every terminal path. */+ function finish(error) {+ if (finished) return;+ finished = true;+ clearTimeout(timer);+ signal?.removeEventListener('abort', expired);+ if (error) reject(error);+ else resolve();+ }+ /** Preserve an unavailable diagnostic as such; it never becomes an empty observed state. */+ function lastState() {+ try { return JSON.stringify(state?.() ?? 'not observed').slice(0, 2_000); }+ catch (error) { return `unavailable (${error.message})`; }+ }+ /** Explain the actual fixture deadline before rejecting the pending state wait. */+ function expired() {+ if (finished) return;+ const error = new Error(`fixture cleanup state did not arrive: ${label}; elapsed ${Math.round(performance.now() - started)} ms; last state: ${lastState()}`);+ try { onTimeout?.(error.message); }+ catch (diagnosticError) { error.message += `; timeout diagnostic failed: ${diagnosticError.message}`; }+ finish(error);+ }+ /** Test the actual condition; only the existing legacy fallback owns a separate clock. */+ function poll() {+ if (finished) return;+ if (signal?.aborted) { expired(); return; }+ let ready;+ try { ready = check(); }+ catch (error) { finish(error); return; }+ if (ready) { finish(); return; }+ // budget: preserve the existing fallback only for callers without a test-owned signal.+ if (!signal && performance.now() - started >= 20_000) { expired(); return; }+ timer = setTimeout(poll, 20); // budget: existing polling interval for a real observable state.+ }+ signal?.addEventListener('abort', expired, { once: true });+ poll();+ });+}diff --git a/test/runner-fixture-state.test.js b/test/runner-fixture-state.test.jsnew file mode 100644index 00000000..547c0687--- /dev/null+++ b/test/runner-fixture-state.test.js@@ -0,0 +1,50 @@+/** Runner fixture waits follow their actual deadline and retain useful state [C7]. */+import assert from 'node:assert/strict';+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { pathToFileURL } from 'node:url';+import { DatabaseSync } from 'node:sqlite';+import { test } from 'node:test';+import { readRemoteGateHolders, waitForCondition } from './runner-fixture-state.js';+import { runFixtureChildAsync } from './fixture-child.js';++test('a runner fixture wait observes a real holder release within its fixture deadline [C7]', async t => {+ const dir = mkdtempSync(join(tmpdir(), 'pullboard-fixture-wait-'));+ const file = join(dir, 'resources.sqlite');+ const db = new DatabaseSync(file);+ t.after(() => { db.close(); rmSync(dir, { recursive: true, force: true }); });+ db.exec("CREATE TABLE holder (name TEXT, token TEXT, pid INTEGER, agent TEXT, browser INTEGER, item TEXT, kind TEXT); INSERT INTO holder VALUES ('gate','fixture-only',42,'owned-fixture',0,'wait','fixture')");+ const controller = new AbortController();+ t.after(() => controller.abort());+ const release = setTimeout(() => db.exec('DELETE FROM holder'), 40); // budget: a controlled real SQLite release handshake, not a throughput limit.+ t.after(() => clearTimeout(release));+ await waitForCondition(() => readRemoteGateHolders(file).length === 0, 'owned holder release', { signal: controller.signal });+ assert.deepEqual(readRemoteGateHolders(file), []);+});++test('a runner fixture timeout names elapsed time and last state at its real test deadline [C7]', async t => {+ const dir = mkdtempSync(join(tmpdir(), 'pullboard-fixture-deadline-'));+ t.after(() => rmSync(dir, { recursive: true, force: true }));+ const file = join(dir, 'held.test.mjs');+ const module = pathToFileURL(join(import.meta.dirname, 'runner-fixture-state.js')).href;+ writeFileSync(file, `import { test } from 'node:test';+import { DatabaseSync } from 'node:sqlite';+import { waitForCondition } from ${JSON.stringify(module)};+test('held state reaches its actual fixture deadline [C7]', { timeout: 80 }, async t => { // budget: deliberate test-owned abort; the wait must report it without a private deadline.+ const db = new DatabaseSync(':memory:');+ db.exec("CREATE TABLE holder (pid INTEGER, kind TEXT); INSERT INTO holder VALUES (42,'held-fixture')");+ t.after(() => db.close());+ let last;+ await waitForCondition(() => { last = db.prepare('SELECT pid,kind FROM holder').all(); return last.length === 0; },+ 'held resource release', { signal: t.signal, state: () => last, onTimeout: message => t.diagnostic(message) });+});+`);+ const env = { ...process.env };+ delete env.NODE_TEST_CONTEXT;+ const result = await runFixtureChildAsync(process.execPath, ['--test', '--test-reporter=tap', file], { env });+ assert.equal(result.status, 1, result.context);+ assert.match(result.stdout, /held resource release; elapsed \d+ ms; last state:/u);+ assert.match(result.stdout, /"pid":42/u);+ assert.match(result.stdout, /"kind":"held-fixture"/u);+});diff --git a/test/runner-parent-death-fixture.js b/test/runner-parent-death-fixture.jsnew file mode 100644index 00000000..6133bf81--- /dev/null+++ b/test/runner-parent-death-fixture.js@@ -0,0 +1,177 @@+/** Exercise actual remote parent-death custody, optionally forcing a real release write conflict [C7]. */+import assert from 'node:assert/strict';+import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, writeFileSync } from 'node:fs';+import { join } from 'node:path';+import { tmpdir } from 'node:os';+import { DatabaseSync } from 'node:sqlite';+import { takeResource } from '../src/resources.js';+import { runFixtureChild } from './fixture-child.js';+import { readRemoteGateHolders, waitForCondition } from './runner-fixture-state.js';++/** Keep the original custody assertions identical in ordinary and forced database ordering cases. */+export async function parentDeathDrainFixture(t, tools, { holdRelease = false } = {}) {+ const { cli, git, installSshStandIn, observe, startQueueController, waitForMarker, processIsAlive, shellQuote, HOST } = tools;+ const dir = realpathSync(mkdtempSync(join(tmpdir(), 'pullboard-runner-parent-death-')));+ t.after(() => rmSync(dir, { recursive: true, force: true }));+ const home = join(dir, 'home');+ const machineHome = join(home, '.pullboard');+ const remoteHome = join(dir, 'worker-home');+ const remoteMachineHome = join(remoteHome, '.pullboard');+ const remoteDatabase = join(remoteMachineHome, 'resources.sqlite');+ const repo = join(dir, 'repo');+ const worker = join(dir, 'second-machine');+ const bin = join(dir, 'bin');+ const requests = join(dir, 'ssh-requests.jsonl');+ const childReady = join(dir, 'detached-child-ready');+ const childPidFile = join(dir, 'detached-child-pid');+ const childTerm = join(dir, 'detached-child-term');+ const releaseChild = join(dir, 'release-detached-child');+ const childGone = join(dir, 'detached-child-gone');+ const workerLog = join(dir, 'worker-stderr.log');+ let releaseLock;+ /** Release only this fixture's explicit write reservation. */+ function unlockRelease() {+ if (!releaseLock) return;+ releaseLock.exec('ROLLBACK');+ releaseLock.close();+ releaseLock = null;+ }+ for (const folder of [repo, worker, bin, home, machineHome, remoteHome, remoteMachineHome]) {+ mkdirSync(folder, { recursive: true });+ }+ writeFileSync(join(machineHome, 'settings.json'), JSON.stringify({ gateSlots: 1 }));+ const env = {+ ...process.env, HOME: home, PULLBOARD_HOME: home, PULLBOARD_MACHINE_HOME: machineHome,+ PATH: `${bin}:${process.env.PATH}`, GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_NOSYSTEM: '1',+ GIT_AUTHOR_NAME: 'Parent Death Fixture', GIT_AUTHOR_EMAIL: '[email protected]',+ GIT_COMMITTER_NAME: 'Parent Death Fixture', GIT_COMMITTER_EMAIL: '[email protected]',+ };+ const initialized = runFixtureChild('git', ['init', '-q', '-b', 'main'], { cwd: repo, env, encoding: 'utf8' });+ assert.equal(initialized.status, 0, initialized.context);+ writeFileSync(join(repo, 'pullboard.json'), JSON.stringify({ gate: 'true' }));+ writeFileSync(join(repo, 'tracked.txt'), 'parent-death runner fixture\n');+ writeFileSync(join(repo, 'remote-parent-death-launcher.js'), `const { spawn } = require('node:child_process');+const child = spawn(process.execPath, [require.resolve('./remote-detached-child.js'), ...process.argv.slice(2)], {+ detached: true, stdio: 'ignore', env: process.env,+});+child.unref();+setInterval(() => {}, 1_000);+`);+ writeFileSync(join(repo, 'remote-detached-child.js'), `const { existsSync, writeFileSync } = require('node:fs');+const [ready, pidFile, term, release, gone] = process.argv.slice(2);+let handlingTerm = false;+process.on('SIGTERM', () => {+ if (handlingTerm) return;+ handlingTerm = true;+ writeFileSync(term, 'term\\n');+ const wait = setInterval(() => {+ if (!existsSync(release)) return;+ clearInterval(wait);+ writeFileSync(gone, 'gone\\n');+ process.exit(0);+ }, 5);+});+writeFileSync(pidFile, String(process.pid));+writeFileSync(ready, 'ready\\n');+setInterval(() => {}, 1_000);+`);+ git(repo, env, 'add', 'pullboard.json', 'tracked.txt', 'remote-parent-death-launcher.js', 'remote-detached-child.js');+ git(repo, env, 'commit', '-q', '-m', 'test: prepare committed detached-child launchers');+ installSshStandIn({ bin, worker, home: remoteHome, machineHome: remoteMachineHome, requests, stderrLog: workerLog });+ const added = cli(repo, env, 'runner', 'add', HOST, '--slots', '1', '--browser', '1', '--json');+ assert.equal(added.status, 0, added.context);++ const previousHome = process.env.PULLBOARD_HOME;+ const previousMachineHome = process.env.PULLBOARD_MACHINE_HOME;+ process.env.PULLBOARD_HOME = home;+ process.env.PULLBOARD_MACHINE_HOME = machineHome;+ let localHold;+ let remoteRun;+ let detachedPid;+ let remoteAgent;+ try {+ localHold = await takeResource({ name: 'gate', capacity: 1, scope: 'machine', root: repo,+ agent: 'parent-death-local-holder', item: 'parent-death-local-holder', kind: 'fixture-hold', browser: true });+ const command = ['node remote-parent-death-launcher.js', ...[childReady, childPidFile, childTerm, releaseChild, childGone].map(shellQuote)].join(' ');+ remoteRun = observe(startQueueController(repo, env, command,+ { item: 'parent-death-remote-holder', queueMarker: null, firstItem: null }));+ await waitForMarker(childReady);+ detachedPid = Number(readFileSync(childPidFile, 'utf8').trim());+ assert.ok(Number.isSafeInteger(detachedPid) && detachedPid > 0);+ assert.notEqual(detachedPid, remoteRun.child.pid, 'the committed launcher owns a separate detached child');+ assert.equal(processIsAlive(detachedPid), true);+ const held = readRemoteGateHolders(remoteDatabase);+ assert.equal(held.length, 1, 'the remote worker keeps its physical gate holder while the launcher is live');+ remoteAgent = held[0].agent;+ assert.match(remoteAgent, /^pullboard-ssh-/u);++ remoteRun.child.kill('SIGTERM');+ await waitForMarker(childTerm);+ assert.equal(processIsAlive(detachedPid), true, 'the detached child remains alive in its controlled TERM handler');+ const duringDrain = readRemoteGateHolders(remoteDatabase);+ assert.equal(duringDrain.length, 1);+ assert.equal(duringDrain[0].agent, remoteAgent, 'the physical runner lease remains held during descendant cleanup');++ if (holdRelease) {+ releaseLock = new DatabaseSync(remoteDatabase);+ releaseLock.exec('PRAGMA busy_timeout = 10000');+ releaseLock.exec('BEGIN IMMEDIATE');+ }+ writeFileSync(releaseChild, 'release\n');+ await waitForMarker(childGone);+ await waitForCondition(() => !processIsAlive(detachedPid), 'the detached child PID exits');+ if (holdRelease) {+ await waitForCondition(() => {+ const jobs = join(remoteHome, '.pullboard-runner', 'jobs');+ const id = remoteAgent.slice('pullboard-ssh-'.length);+ for (const name of existsSync(jobs) ? readdirSync(jobs) : []) {+ if (!name.startsWith(id + '-')) continue;+ const pending = join(jobs, name, 'release-pending.json');+ if (!existsSync(pending)) continue;+ const diagnostic = JSON.parse(readFileSync(pending, 'utf8'));+ assert.equal(diagnostic.id, id);+ assert.equal(diagnostic.stage, 'lease release');+ assert.equal(diagnostic.pending, true);+ t.diagnostic('observed real failed holder deletion in private job: ' + JSON.stringify(diagnostic));+ return true;+ }+ assert.equal(processIsAlive(held[0].pid), true,+ 'a failed raw release keeps its worker alive to retry instead of stranding a dead holder');+ return false;+ }, 'the actual remote release reaches the held database transaction',+ { signal: t.signal, state: () => ({ workerAlive: processIsAlive(held[0].pid),+ holders: readRemoteGateHolders(remoteDatabase).map(({ pid, agent, kind }) => ({ pid, agent, kind })) }),+ onTimeout: message => t.diagnostic(message) });+ assert.equal(readRemoteGateHolders(remoteDatabase).some(holder => holder.agent === remoteAgent), true,+ 'the holder remains until its real deletion can commit');+ unlockRelease();+ }+ let lastReleaseState = 'not observed';+ await waitForCondition(() => {+ const holders = readRemoteGateHolders(remoteDatabase).filter(holder => holder.agent === remoteAgent);+ lastReleaseState = { observedAt: new Date().toISOString(), holders: holders.map(({ pid, agent, browser, item, kind }) =>+ ({ pid, agent, browser, item, kind, alive: processIsAlive(pid) })) };+ return holders.length === 0;+ }, 'the remote physical gate holder is released after child exit',+ { signal: t.signal, state: () => lastReleaseState, onTimeout: message => t.diagnostic(message) });+ const controller = await remoteRun.closed;+ remoteRun = null;+ assert.equal(controller.signal, 'SIGTERM', 'the owned API controller alone received SIGTERM');+ } finally {+ unlockRelease();+ if (localHold) localHold.release();+ writeFileSync(releaseChild, 'cleanup-release\n');+ if (remoteRun && remoteRun.child.exitCode === null && remoteRun.child.signalCode === null) {+ remoteRun.child.kill('SIGTERM');+ }+ if (detachedPid && processIsAlive(detachedPid)) {+ try { process.kill(detachedPid, 'SIGTERM'); } catch (error) { if (error.code !== 'ESRCH') throw error; }+ }+ if (remoteRun) await remoteRun.closed;+ if (detachedPid) await waitForCondition(() => !processIsAlive(detachedPid), 'fixture detached child cleanup');+ if (previousHome === undefined) delete process.env.PULLBOARD_HOME;+ else process.env.PULLBOARD_HOME = previousHome;+ if (previousMachineHome === undefined) delete process.env.PULLBOARD_MACHINE_HOME;+ else process.env.PULLBOARD_MACHINE_HOME = previousMachineHome;+ }+}diff --git a/test/runners.test.js b/test/runners.test.jsindex 939b1a5a..710f28a7 100644--- a/test/runners.test.js+++ b/test/runners.test.js@@ -12,6 +12,8 @@ import { gateSlotEnvironment, listResources, setResourceCapacity, takeResource }import { parseProcessCensus } from '../src/process-census.js';import { listResources as listPreRunnerResources, takeResource as takePreRunnerResource } from './fixtures/resources-pre-runners.js';import { runFixtureChild, runFixtureChildAsync, startFixtureChild } from './fixture-child.js';+import { readRemoteGateHolders, waitForCondition } from './runner-fixture-state.js';+import { parentDeathDrainFixture } from './runner-parent-death-fixture.js';import { createE2eHelpers } from './e2e-helpers.js';import { refuseSandboxBlocked } from '../src/gate.js';@@ -422,15 +424,6 @@ function fixtureResultContext(result) {return `${result.context}\nstdout tail:\n${String(result.stdout ?? '').slice(-4_000)}\nstderr tail:\n${String(result.stderr ?? '').slice(-4_000)}`;}-/** Read the remote machine's live gate holders without changing its shared admission state. */-function readRemoteGateHolders(file) {- const db = new DatabaseSync(file, { readOnly: true });- try {- db.exec('PRAGMA busy_timeout = 10000');- return db.prepare("SELECT token, pid, agent, browser, item, kind FROM holder WHERE name = 'gate'").all();- } finally { db.close(); }-}-/** Wait for the marker itself: a missed file notification must not hide observable readiness. */async function waitForMarker(path) {const deadline = Date.now() + 20_000; // budget: unchanged hang ceiling for the queue-to-parent handoff.@@ -440,15 +433,6 @@ async function waitForMarker(path) {}}-/** Poll a concrete process/resource state; the deadline only bounds a broken cleanup handshake. */-async function waitForCondition(check, label) {- const deadline = Date.now() + 20_000;- while (!check()) {- if (Date.now() >= deadline) throw new Error(`fixture cleanup state did not arrive: ${label}`);- await new Promise(resolve => setTimeout(resolve, 20)); // budget: polling interval for an observable cleanup condition.- }-}-/** Check exact PID presence without signalling or treating permission denial as absence. */function processIsAlive(pid) {try { process.kill(pid, 0); return true; }@@ -1681,124 +1665,13 @@ syncBuiltinESMExports();\n`);}});-test('a queued run executes on a registered runner and its result comes back [C7] (parent death drains a detached worker descendant before releasing its lease)', async (t) => {- const dir = realpathSync(mkdtempSync(join(tmpdir(), 'pullboard-runner-parent-death-')));- t.after(() => rmSync(dir, { recursive: true, force: true }));- const home = join(dir, 'home');- const machineHome = join(home, '.pullboard');- const remoteHome = join(dir, 'worker-home');- const remoteMachineHome = join(remoteHome, '.pullboard');- const remoteDatabase = join(remoteMachineHome, 'resources.sqlite');- const repo = join(dir, 'repo');- const worker = join(dir, 'second-machine');- const bin = join(dir, 'bin');- const requests = join(dir, 'ssh-requests.jsonl');- const childReady = join(dir, 'detached-child-ready');- const childPidFile = join(dir, 'detached-child-pid');- const childTerm = join(dir, 'detached-child-term');- const releaseChild = join(dir, 'release-detached-child');- const childGone = join(dir, 'detached-child-gone');- for (const folder of [repo, worker, bin, home, machineHome, remoteHome, remoteMachineHome]) {- mkdirSync(folder, { recursive: true });- }- writeFileSync(join(machineHome, 'settings.json'), JSON.stringify({ gateSlots: 1 }));- const env = {- ...process.env, HOME: home, PULLBOARD_HOME: home, PULLBOARD_MACHINE_HOME: machineHome,- PATH: `${bin}:${process.env.PATH}`, GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_NOSYSTEM: '1',- GIT_AUTHOR_NAME: 'Parent Death Fixture', GIT_AUTHOR_EMAIL: '[email protected]',- GIT_COMMITTER_NAME: 'Parent Death Fixture', GIT_COMMITTER_EMAIL: '[email protected]',- };- const initialized = runFixtureChild('git', ['init', '-q', '-b', 'main'], { cwd: repo, env, encoding: 'utf8' });- assert.equal(initialized.status, 0, initialized.context);- writeFileSync(join(repo, 'pullboard.json'), JSON.stringify({ gate: 'true' }));- writeFileSync(join(repo, 'tracked.txt'), 'parent-death runner fixture\n');- writeFileSync(join(repo, 'remote-parent-death-launcher.js'), `const { spawn } = require('node:child_process');-const child = spawn(process.execPath, [require.resolve('./remote-detached-child.js'), ...process.argv.slice(2)], {- detached: true, stdio: 'ignore', env: process.env,-});-child.unref();-setInterval(() => {}, 1_000);-`);- writeFileSync(join(repo, 'remote-detached-child.js'), `const { existsSync, writeFileSync } = require('node:fs');-const [ready, pidFile, term, release, gone] = process.argv.slice(2);-let handlingTerm = false;-process.on('SIGTERM', () => {- if (handlingTerm) return;- handlingTerm = true;- writeFileSync(term, 'term\\n');- const wait = setInterval(() => {- if (!existsSync(release)) return;- clearInterval(wait);- writeFileSync(gone, 'gone\\n');- process.exit(0);- }, 5);-});-writeFileSync(pidFile, String(process.pid));-writeFileSync(ready, 'ready\\n');-setInterval(() => {}, 1_000);-`);- git(repo, env, 'add', 'pullboard.json', 'tracked.txt', 'remote-parent-death-launcher.js', 'remote-detached-child.js');- git(repo, env, 'commit', '-q', '-m', 'test: prepare committed detached-child launchers');- installSshStandIn({ bin, worker, home: remoteHome, machineHome: remoteMachineHome, requests });- const added = cli(repo, env, 'runner', 'add', HOST, '--slots', '1', '--browser', '1', '--json');- assert.equal(added.status, 0, added.context);+const parentDeathTools = { cli, git, installSshStandIn, observe, startQueueController, waitForMarker, processIsAlive, shellQuote, HOST };+test('a queued run executes on a registered runner and its result comes back [C7] (parent death drains a detached worker descendant before releasing its lease)',+ { timeout: 90_000 }, t => parentDeathDrainFixture(t, parentDeathTools)); // budget: fixture ceiling covers declared30-second remote cleanup plus startup, transport and final cleanup.++test('parent death retries a failed remote holder deletion after the real database writer releases [C7]',+ { timeout: 90_000 }, t => parentDeathDrainFixture(t, parentDeathTools, { holdRelease: true })); // budget: same fixture deadline includes the real10-second SQLite busy attempt and release retry.- const previousHome = process.env.PULLBOARD_HOME;- const previousMachineHome = process.env.PULLBOARD_MACHINE_HOME;- process.env.PULLBOARD_HOME = home;- process.env.PULLBOARD_MACHINE_HOME = machineHome;- let localHold;- let remoteRun;- let detachedPid;- let remoteAgent;- try {- localHold = await takeResource({ name: 'gate', capacity: 1, scope: 'machine', root: repo,- agent: 'parent-death-local-holder', item: 'parent-death-local-holder', kind: 'fixture-hold', browser: true });- const command = ['node remote-parent-death-launcher.js', ...[childReady, childPidFile, childTerm, releaseChild, childGone].map(shellQuote)].join(' ');- remoteRun = observe(startQueueController(repo, env, command,- { item: 'parent-death-remote-holder', queueMarker: null, firstItem: null }));- await waitForMarker(childReady);- detachedPid = Number(readFileSync(childPidFile, 'utf8').trim());- assert.ok(Number.isSafeInteger(detachedPid) && detachedPid > 0);- assert.notEqual(detachedPid, remoteRun.child.pid, 'the committed launcher owns a separate detached child');- assert.equal(processIsAlive(detachedPid), true);- const held = readRemoteGateHolders(remoteDatabase);- assert.equal(held.length, 1, 'the remote worker keeps its physical gate holder while the launcher is live');- remoteAgent = held[0].agent;- assert.match(remoteAgent, /^pullboard-ssh-/u);-- remoteRun.child.kill('SIGTERM');- await waitForMarker(childTerm);- assert.equal(processIsAlive(detachedPid), true, 'the detached child remains alive in its controlled TERM handler');- const duringDrain = readRemoteGateHolders(remoteDatabase);- assert.equal(duringDrain.length, 1);- assert.equal(duringDrain[0].agent, remoteAgent, 'the physical runner lease remains held during descendant cleanup');-- writeFileSync(releaseChild, 'release\n');- await waitForMarker(childGone);- await waitForCondition(() => !processIsAlive(detachedPid), 'the detached child PID exits');- await waitForCondition(() => readRemoteGateHolders(remoteDatabase).every(holder => holder.agent !== remoteAgent),- 'the remote physical gate holder is released after child exit');- const controller = await remoteRun.closed;- remoteRun = null;- assert.equal(controller.signal, 'SIGTERM', 'the owned API controller alone received SIGTERM');- } finally {- if (localHold) localHold.release();- writeFileSync(releaseChild, 'cleanup-release\n');- if (remoteRun && remoteRun.child.exitCode === null && remoteRun.child.signalCode === null) {- remoteRun.child.kill('SIGTERM');- }- if (detachedPid && processIsAlive(detachedPid)) {- try { process.kill(detachedPid, 'SIGTERM'); } catch (error) { if (error.code !== 'ESRCH') throw error; }- }- if (remoteRun) await remoteRun.closed;- if (detachedPid) await waitForCondition(() => !processIsAlive(detachedPid), 'fixture detached child cleanup');- if (previousHome === undefined) delete process.env.PULLBOARD_HOME;- else process.env.PULLBOARD_HOME = previousHome;- if (previousMachineHome === undefined) delete process.env.PULLBOARD_MACHINE_HOME;- else process.env.PULLBOARD_MACHINE_HOME = previousMachineHome;- }-});test('a queued run executes on a registered runner and its result comes back [C7] (unverified cleanup times out as a named red result)', { timeout: 90_000 }, async (t) => { // budget: bounded 30-second remote cleanup timeout plus startup, transport and fixture cleanup ceiling.const dir = realpathSync(mkdtempSync(join(tmpdir(), 'pullboard-runner-cleanup-timeout-')));