Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: settle the running stage before a cancelled pregate exits (#12641)

[._.]/ Adam Eivy9ea03389947f
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
index 6edaa839a4..fb890f7487 100644
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@@ -120,6 +120,12 @@ that was validated — untracked and uncommitted files included, so a later
commit of that state yields the same `HEAD^{tree}`. A green summary never
claims a suite it skipped; required CI checks still apply.
+Interrupting pregate (Ctrl-C or SIGTERM) does not leave tests running behind
+it: the running stage — including the test runner's npm and Vitest processes,
+which share that stage's process group — gets the signal and a 5-second grace,
+then the group is killed until no member remains. Only then does pregate write
+an incomplete receipt (`passed: false`) and exit by the same signal.
+
After the push (and, when you open the pull request yourself, before
`gh pr create`), `node scripts/pregate-attest.js` posts that receipt as a
`pregate` commit status on the pushed head, bound to the validated tree and
diff --git a/scripts/pregate.js b/scripts/pregate.js
index f6b55a87d9..9083f428ab 100644
--- a/scripts/pregate.js
+++ b/scripts/pregate.js
@@ -143,9 +143,13 @@ export function pregateWorkerEnv(env) {
* The child environment for one stage. A stage value of `undefined` REMOVES
* the variable (the DB stages drop `TEST_DB_OK` this way), which is the only
* way an inherited escape hatch can be kept away from a runner.
+ *
+ * `ownsGroup` (POSIX, where each stage gets its own process group) tells
+ * run-ci-tests.js to keep npm and Vitest in that group instead of detaching
+ * them, so cancelling the stage group reaches every test process (#12641).
*/
-export function stageSpawnEnv(baseEnv, stageEnv) {
- const env = { ...pregateWorkerEnv(baseEnv) };
+export function stageSpawnEnv(baseEnv, stageEnv, { ownsGroup = !IS_WIN32 } = {}) {
+ const env = { ...pregateWorkerEnv(baseEnv), ...(ownsGroup ? { PORTOS_CI_TEST_SUPERVISED_GROUP: '1' } : {}) };
for (const [name, value] of Object.entries(stageEnv)) {
if (value === undefined) delete env[name];
else env[name] = value;
@@ -400,47 +404,108 @@ function stageSpawn(stage, env) {
return prepareWindowsSafeSpawn(resolveWindowsExecutable('npm', IS_WIN32, env) || 'npm', stage.npm, IS_WIN32);
}
-/** Wait asynchronously so owner signals can be attributed during a stage. */
-function runStage(stage) {
- return new Promise((resolve) => {
- const env = stageSpawnEnv(process.env, stage.env);
- const { command, args } = stageSpawn(stage, env);
- const child = spawn(command, args, {
- cwd: REPO_ROOT,
- stdio: 'inherit',
- env,
- // A separate group lets cancellation reach only this stage's descendants.
- detached: !IS_WIN32,
- });
- const handlers = new Map();
- const cleanup = () => {
- for (const [signal, handler] of handlers) process.removeListener(signal, handler);
+/** How long a cancelled stage may wind down before its whole group is killed. Exceeds run-ci-tests.js's own 2s npm escalation. */
+const STAGE_CANCEL_GRACE_MS = 5000;
+/** Bound on proving a killed stage group empty; past it the cancellation is reported unverified. */
+const STAGE_SETTLE_MS = 5000;
+
+/**
+ * Owner signal handling for the whole stage loop. A Ctrl-C reaches pregate
+ * twice — from the terminal's process group and again forwarded by `npm run`.
+ * A handler that removed itself on the first restored the default action, so
+ * the duplicate killed pregate before it signalled the stage group and the
+ * detached runner kept testing (#12641). These handlers stay installed,
+ * absorb duplicates, and are released only after every owned group settled.
+ */
+function installOwnerCancellation() {
+ const owner = { signal: null, onSignal: null };
+ const handlers = ['SIGINT', 'SIGTERM'].map((signal) => {
+ const handler = () => {
+ if (owner.signal) return;
+ owner.signal = signal;
+ owner.onSignal?.(signal);
};
- for (const signal of ['SIGINT', 'SIGTERM']) {
- const handler = () => {
- // Flush before restoring the signal's default termination behavior.
- writeSync(2, `❌ Pregate incomplete: stage="${stage.name}" owner_pid=${process.pid} child_pid=${child.pid ?? 'unavailable'} received_signal=${signal}.\n`);
- cleanup();
- try {
- if (IS_WIN32) child.kill(signal);
- else if (child.pid) process.kill(-child.pid, signal);
- } catch (error) {
- if (error.code !== 'ESRCH') writeSync(2, `❌ Stage cancellation failed: ${error.code ?? 'unknown'}.\n`);
- }
- process.kill(process.pid, signal);
- };
- handlers.set(signal, handler);
- process.once(signal, handler);
+ process.on(signal, handler);
+ return [signal, handler];
+ });
+ owner.release = () => {
+ for (const [signal, handler] of handlers) process.removeListener(signal, handler);
+ };
+ return owner;
+}
+
+function signalStage(child, signal) {
+ try {
+ if (IS_WIN32) child.kill(signal);
+ else process.kill(-child.pid, signal);
+ } catch (error) {
+ if (error.code !== 'ESRCH') writeSync(2, `❌ Stage cancellation failed: ${error.code ?? 'unknown'}.\n`);
+ }
+}
+
+const within = (promise, ms) => {
+ let timer;
+ return Promise.race([
+ promise.then(() => true),
+ new Promise((resolve) => { timer = setTimeout(() => resolve(false), ms); }),
+ ]).finally(() => clearTimeout(timer));
+};
+
+/**
+ * Stop a stage and everything in its process group before pregate finishes:
+ * forward the owner's signal, give the stage its grace to report, kill the
+ * group if it is still running, then kill any member that outlived the
+ * leader until the group no longer exists. Resolves `{ settled, reason? }`.
+ */
+async function cancelStage(stage, child, closed, signal) {
+ writeSync(2, `❌ Pregate incomplete: stage="${stage.name}" owner_pid=${process.pid} child_pid=${child.pid ?? 'unavailable'} received_signal=${signal}.\n`);
+ if (!child.pid) return { settled: true };
+ signalStage(child, signal);
+ if (!await within(closed, STAGE_CANCEL_GRACE_MS)) {
+ writeSync(2, `⚠️ Stage "${stage.name}" outlived its ${STAGE_CANCEL_GRACE_MS / 1000}s cancellation grace; killing its process group.\n`);
+ signalStage(child, 'SIGKILL');
+ if (!await within(closed, STAGE_SETTLE_MS)) return { settled: false, reason: 'stage process did not exit after SIGKILL' };
+ }
+ if (IS_WIN32) return { settled: true };
+ // The leader is reaped, but a descendant that ignored the signal keeps the
+ // group (and its open log descriptors) alive. ESRCH is the proof of empty.
+ const deadline = Date.now() + STAGE_SETTLE_MS;
+ for (;;) {
+ try {
+ process.kill(-child.pid, 'SIGKILL');
+ } catch (error) {
+ return error.code === 'ESRCH' ? { settled: true } : { settled: false, reason: `group kill failed (${error.code ?? 'unknown'})` };
}
- child.once('error', (error) => {
- cleanup();
- resolve({ error });
- });
- child.once('close', (status, signal) => {
- cleanup();
- resolve({ status, signal, pid: child.pid });
- });
+ if (Date.now() >= deadline) return { settled: false, reason: 'group members remained after SIGKILL' };
+ await new Promise((resolve) => setTimeout(resolve, 25));
+ }
+}
+
+/** Run one stage; an owner signal during it resolves only after the stage group settled. */
+async function runStage(stage, owner) {
+ const env = stageSpawnEnv(process.env, stage.env);
+ const { command, args } = stageSpawn(stage, env);
+ const child = spawn(command, args, {
+ cwd: REPO_ROOT,
+ stdio: 'inherit',
+ env,
+ // A separate group lets cancellation reach only this stage's descendants.
+ detached: !IS_WIN32,
});
+ const closed = new Promise((resolve) => {
+ child.once('error', (error) => resolve({ error }));
+ child.once('close', (status, signal) => resolve({ status, signal }));
+ });
+ let cancellation = null;
+ let interrupted = null;
+ owner.onSignal = (signal) => {
+ interrupted = signal;
+ cancellation = cancelStage(stage, child, closed, signal);
+ };
+ const outcome = await closed;
+ owner.onSignal = null;
+ if (!cancellation) return { ...outcome, pid: child.pid };
+ return { ...outcome, pid: child.pid, interrupted, ...await cancellation };
}
const formatDuration = (ms) => (ms >= 60_000 ? `${(ms / 60_000).toFixed(1)}m` : `${(ms / 1000).toFixed(1)}s`);
@@ -612,8 +677,9 @@ async function main() {
const results = stages.map((stage) => ({ ...stage, command: describeStageCommand(stage), status: 'not-run', durationMs: 0 }));
let exitCode = 0;
let failure = null;
+ const owner = installOwnerCancellation();
for (const result of results) {
- if (failure) break;
+ if (failure || owner.signal) break;
if (result.suite !== 'scan') {
try {
hashTree();
@@ -625,9 +691,16 @@ async function main() {
}
console.log(`▶️ ${result.name}`);
const started = Date.now();
- const outcome = await runStage(result);
+ const outcome = await runStage(result, owner);
result.durationMs = Date.now() - started;
- if (outcome.error) {
+ if (outcome.interrupted) {
+ result.status = 'failed';
+ result.signal = outcome.interrupted;
+ failure = outcome.settled
+ ? `❌ Pregate incomplete: stage="${result.name}" cancelled by ${outcome.interrupted}; its process group has exited.`
+ : `❌ Pregate incomplete: stage="${result.name}" cancelled by ${outcome.interrupted}; owned process group settlement unverified (${outcome.reason}).`;
+ exitCode = 128 + constants.signals[outcome.interrupted];
+ } else if (outcome.error) {
result.status = 'failed';
failure = `❌ ${result.name} could not start: ${outcome.error.message}`;
exitCode = 1;
@@ -648,6 +721,12 @@ async function main() {
}
}
+ if (owner.signal && !failure) {
+ writeSync(2, `❌ Pregate incomplete: owner_pid=${process.pid} received_signal=${owner.signal} between stages.\n`);
+ failure = `❌ Pregate incomplete: cancelled by ${owner.signal} before every stage ran.`;
+ exitCode = 128 + constants.signals[owner.signal];
+ }
+
let treeAfterRun = null;
try {
if (!failure) hashTree();
@@ -675,11 +754,15 @@ async function main() {
console.warn(`⚠️ The working tree changed while pregate ran (${tree.slice(0, 12)} → ${treeAfterRun?.slice(0, 12) ?? 'unhashable'}); the receipt covers the tree it started with.`);
}
+ owner.release();
if (failure) {
console.error(failure);
console.error(`📄 Receipt: ${PREGATE_REPORT_PATH}`);
- process.exit(exitCode);
}
+ // Every owned group has settled and the receipt is written; end the way the
+ // caller's signal asked. A signal after the last stage leaves its truthful receipt.
+ if (owner.signal) process.kill(process.pid, owner.signal);
+ if (failure) process.exit(exitCode);
const ran = results.map((stage) => stage.name).join(', ');
console.log(`✅ Pregate passed: ${ran}.`);
diff --git a/scripts/pregate.test.js b/scripts/pregate.test.js
index 6cd5d1fca9..9a461aa603 100644
--- a/scripts/pregate.test.js
+++ b/scripts/pregate.test.js
@@ -161,7 +161,7 @@ describe('pregate hidden-content invocation', () => {
mkdirSync(dirname(join(root, path)), { recursive: true });
copyFileSync(join(REPO_ROOT, path), join(root, path));
}
- write('package.json', '{"type":"module"}\n');
+ write('package.json', '{"type":"module","scripts":{"pregate":"node scripts/pregate.js"}}\n');
// The receipt lands under /.cache/, which the real repository ignores too.
write('.gitignore', 'node_modules\n/.cache/\n');
for (const workspace of ['', 'client', 'server', 'autofixer']) {
@@ -335,6 +335,74 @@ describe('pregate hidden-content invocation', () => {
}
});
+ // #12641: a Ctrl-C reaches pregate from the terminal group AND from npm's
+ // forwarding; the duplicate used to kill pregate before it signalled the
+ // stage, orphaning the runner. The fixture stage mirrors run-ci-tests.js: a
+ // launcher that settles its own separately grouped child on a signal, plus
+ // an in-group descendant that ignores signals and keeps writing.
+ it.skipIf(process.platform === 'win32').each(['SIGINT', 'SIGTERM'])('settles every owned stage process before a %s through npm finishes', async (signal) => {
+ const heartbeat = join(scratch, 'heartbeat');
+ write('scripts/scan-diff-hidden-content.js', `
+ import { spawn } from 'child_process';
+ const straggler = spawn(process.execPath, ['-e', "process.on('SIGINT', () => {}); process.on('SIGTERM', () => {});"
+ + " setInterval(() => require('fs').appendFileSync(process.env.FIXTURE_HEARTBEAT, '.'), 10);"], { stdio: 'ignore' });
+ const launched = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore', detached: true });
+ const settle = () => {
+ process.kill(-launched.pid, 'SIGKILL');
+ launched.once('exit', () => setTimeout(() => process.exit(1), 100));
+ };
+ process.on('SIGINT', settle);
+ process.on('SIGTERM', settle);
+ setTimeout(() => console.log('fixture-stage-ready ' + JSON.stringify([process.pid, straggler.pid, launched.pid])), 200);
+ setInterval(() => {}, 1000);
+ `);
+ const unrelated = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore', detached: true });
+ const npm = spawn('npm', ['run', 'pregate', '--', '--base', base, '--skip-lint'], {
+ cwd: root, stdio: ['ignore', 'pipe', 'pipe'], detached: true, env: { ...process.env, FIXTURE_HEARTBEAT: heartbeat },
+ });
+ const alive = (pid) => {
+ try { process.kill(pid, 0); return true; } catch (error) { return error.code !== 'ESRCH'; }
+ };
+ let owned = [];
+ let stdout = '';
+ let stderr = '';
+ npm.stderr.on('data', (data) => { stderr += data; });
+ npm.stdout.on('data', (data) => {
+ stdout += data;
+ const ready = /fixture-stage-ready (\[[\d,]+\])/.exec(stdout);
+ if (ready && owned.length === 0) {
+ owned = JSON.parse(ready[1]);
+ // Ctrl-C reaches the whole foreground group: npm and pregate alike.
+ process.kill(-npm.pid, signal);
+ }
+ });
+ try {
+ // `exit`, not `close`: a surviving descendant would hold the pipes open.
+ const result = await new Promise((resolve, reject) => {
+ npm.once('error', reject);
+ npm.once('exit', (status, exitSignal) => resolve({ status, signal: exitSignal }));
+ });
+ expect(owned).toHaveLength(3);
+ expect(owned.filter(alive)).toEqual([]);
+ expect(result.status === null ? result.signal : result.status).not.toBe(0);
+ expect(stderr).toMatch(new RegExp(`stage="hidden-content scan" owner_pid=\\d+ child_pid=${owned[0]} received_signal=${signal}`));
+ expect(stderr).toContain(`stage="hidden-content scan" cancelled by ${signal}; its process group has exited.`);
+ expect(stdout).not.toContain('Pregate passed');
+ const report = JSON.parse(readFileSync(join(root, PREGATE_REPORT_PATH), 'utf8'));
+ expect(report.passed).toBe(false);
+ expect(report.stages[0]).toMatchObject({ name: 'hidden-content scan', status: 'failed', signal });
+ const written = readFileSync(heartbeat, 'utf8').length;
+ await new Promise((resolve) => setTimeout(resolve, 100));
+ expect(readFileSync(heartbeat, 'utf8').length).toBe(written);
+ expect(alive(unrelated.pid)).toBe(true);
+ } finally {
+ for (const pid of [npm.pid, unrelated.pid, ...owned]) {
+ try { process.kill(-pid, 'SIGKILL'); } catch { /* not a live group */ }
+ try { process.kill(pid, 'SIGKILL'); } catch { /* already gone */ }
+ }
+ }
+ });
+
it.skipIf(process.platform === 'win32')('reports a stage child signal as incomplete rather than an ordinary assertion failure', () => {
write('scripts/scan-diff-hidden-content.js', "process.kill(process.pid, 'SIGTERM');\n");
const result = run('--skip-lint');
@@ -868,10 +936,14 @@ describe('uncoveredSuites', () => {
describe('stageSpawnEnv', () => {
it('applies stage values and removes the variables a stage sets to undefined', () => {
- const env = stageSpawnEnv({ TEST_DB_OK: '1', PGDATABASE: 'portos', KEEP: 'x' }, { TEST_DB_OK: undefined, PGDATABASE: 'portos_test' });
+ const env = stageSpawnEnv({ TEST_DB_OK: '1', PGDATABASE: 'portos', KEEP: 'x' }, { TEST_DB_OK: undefined, PGDATABASE: 'portos_test' }, { ownsGroup: false });
expect(env).toEqual({ KEEP: 'x', PGDATABASE: 'portos_test', PORTOS_PREGATE_MAX_WORKERS: '4' });
expect('TEST_DB_OK' in env).toBe(false);
});
+
+ it('keeps the test runner\'s npm and Vitest inside the stage group pregate owns', () => {
+ expect(stageSpawnEnv({}, {}, { ownsGroup: true }).PORTOS_CI_TEST_SUPERVISED_GROUP).toBe('1');
+ });
});
describe('testDatabaseStageEnv', () => {
diff --git a/scripts/run-ci-tests.js b/scripts/run-ci-tests.js
index 7f22f0a0ee..cfca7e83c9 100644
--- a/scripts/run-ci-tests.js
+++ b/scripts/run-ci-tests.js
@@ -271,9 +271,10 @@ function runNpmWithReport(scope, script, extraArgs, reportPath, env, abortSignal
persistTestDiagnostic(invocationPath, invocation);
return new Promise((resolve) => {
// npm is a launcher, not the entire test tree. Give this invocation its
- // own POSIX group; Windows killProcessTree uses taskkill /T instead. The
- // browser supervisor instead owns the inherited group, so its independent
- // watchdog can retire npm even if this launcher's event loop is stopped.
+ // own POSIX group; Windows killProcessTree uses taskkill /T instead. A
+ // supervisor (the browser watchdog, or pregate's stage group) instead owns
+ // the inherited group, so it can retire npm and Vitest even after this
+ // launcher's event loop stopped or the launcher itself was killed.
const processGroup = process.platform !== 'win32' && env.PORTOS_CI_TEST_SUPERVISED_GROUP !== '1';
const childEnv = { ...env, PORTOS_CI_DIAGNOSTICS_PATH: runReportPath };
// Only the workflow launcher owns durable attempts. Nested launcher