Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: settle the running stage before a cancelled pregate exits (#12641)
[._.]/ Adam Eivy9ea03389947f
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.mdindex 6edaa839a4..fb890f7487 100644--- a/docs/CONTRIBUTING.md+++ b/docs/CONTRIBUTING.md@@ -120,6 +120,12 @@ that was validated — untracked and uncommitted files included, so a latercommit of that state yields the same `HEAD^{tree}`. A green summary neverclaims a suite it skipped; required CI checks still apply.+Interrupting pregate (Ctrl-C or SIGTERM) does not leave tests running behind+it: the running stage — including the test runner's npm and Vitest processes,+which share that stage's process group — gets the signal and a 5-second grace,+then the group is killed until no member remains. Only then does pregate write+an incomplete receipt (`passed: false`) and exit by the same signal.+After the push (and, when you open the pull request yourself, before`gh pr create`), `node scripts/pregate-attest.js` posts that receipt as a`pregate` commit status on the pushed head, bound to the validated tree anddiff --git a/scripts/pregate.js b/scripts/pregate.jsindex f6b55a87d9..9083f428ab 100644--- a/scripts/pregate.js+++ b/scripts/pregate.js@@ -143,9 +143,13 @@ export function pregateWorkerEnv(env) {* The child environment for one stage. A stage value of `undefined` REMOVES* the variable (the DB stages drop `TEST_DB_OK` this way), which is the only* way an inherited escape hatch can be kept away from a runner.+ *+ * `ownsGroup` (POSIX, where each stage gets its own process group) tells+ * run-ci-tests.js to keep npm and Vitest in that group instead of detaching+ * them, so cancelling the stage group reaches every test process (#12641).*/-export function stageSpawnEnv(baseEnv, stageEnv) {- const env = { ...pregateWorkerEnv(baseEnv) };+export function stageSpawnEnv(baseEnv, stageEnv, { ownsGroup = !IS_WIN32 } = {}) {+ const env = { ...pregateWorkerEnv(baseEnv), ...(ownsGroup ? { PORTOS_CI_TEST_SUPERVISED_GROUP: '1' } : {}) };for (const [name, value] of Object.entries(stageEnv)) {if (value === undefined) delete env[name];else env[name] = value;@@ -400,47 +404,108 @@ function stageSpawn(stage, env) {return prepareWindowsSafeSpawn(resolveWindowsExecutable('npm', IS_WIN32, env) || 'npm', stage.npm, IS_WIN32);}-/** Wait asynchronously so owner signals can be attributed during a stage. */-function runStage(stage) {- return new Promise((resolve) => {- const env = stageSpawnEnv(process.env, stage.env);- const { command, args } = stageSpawn(stage, env);- const child = spawn(command, args, {- cwd: REPO_ROOT,- stdio: 'inherit',- env,- // A separate group lets cancellation reach only this stage's descendants.- detached: !IS_WIN32,- });- const handlers = new Map();- const cleanup = () => {- for (const [signal, handler] of handlers) process.removeListener(signal, handler);+/** How long a cancelled stage may wind down before its whole group is killed. Exceeds run-ci-tests.js's own 2s npm escalation. */+const STAGE_CANCEL_GRACE_MS = 5000;+/** Bound on proving a killed stage group empty; past it the cancellation is reported unverified. */+const STAGE_SETTLE_MS = 5000;++/**+ * Owner signal handling for the whole stage loop. A Ctrl-C reaches pregate+ * twice — from the terminal's process group and again forwarded by `npm run`.+ * A handler that removed itself on the first restored the default action, so+ * the duplicate killed pregate before it signalled the stage group and the+ * detached runner kept testing (#12641). These handlers stay installed,+ * absorb duplicates, and are released only after every owned group settled.+ */+function installOwnerCancellation() {+ const owner = { signal: null, onSignal: null };+ const handlers = ['SIGINT', 'SIGTERM'].map((signal) => {+ const handler = () => {+ if (owner.signal) return;+ owner.signal = signal;+ owner.onSignal?.(signal);};- for (const signal of ['SIGINT', 'SIGTERM']) {- const handler = () => {- // Flush before restoring the signal's default termination behavior.- writeSync(2, `❌ Pregate incomplete: stage="${stage.name}" owner_pid=${process.pid} child_pid=${child.pid ?? 'unavailable'} received_signal=${signal}.\n`);- cleanup();- try {- if (IS_WIN32) child.kill(signal);- else if (child.pid) process.kill(-child.pid, signal);- } catch (error) {- if (error.code !== 'ESRCH') writeSync(2, `❌ Stage cancellation failed: ${error.code ?? 'unknown'}.\n`);- }- process.kill(process.pid, signal);- };- handlers.set(signal, handler);- process.once(signal, handler);+ process.on(signal, handler);+ return [signal, handler];+ });+ owner.release = () => {+ for (const [signal, handler] of handlers) process.removeListener(signal, handler);+ };+ return owner;+}++function signalStage(child, signal) {+ try {+ if (IS_WIN32) child.kill(signal);+ else process.kill(-child.pid, signal);+ } catch (error) {+ if (error.code !== 'ESRCH') writeSync(2, `❌ Stage cancellation failed: ${error.code ?? 'unknown'}.\n`);+ }+}++const within = (promise, ms) => {+ let timer;+ return Promise.race([+ promise.then(() => true),+ new Promise((resolve) => { timer = setTimeout(() => resolve(false), ms); }),+ ]).finally(() => clearTimeout(timer));+};++/**+ * Stop a stage and everything in its process group before pregate finishes:+ * forward the owner's signal, give the stage its grace to report, kill the+ * group if it is still running, then kill any member that outlived the+ * leader until the group no longer exists. Resolves `{ settled, reason? }`.+ */+async function cancelStage(stage, child, closed, signal) {+ writeSync(2, `❌ Pregate incomplete: stage="${stage.name}" owner_pid=${process.pid} child_pid=${child.pid ?? 'unavailable'} received_signal=${signal}.\n`);+ if (!child.pid) return { settled: true };+ signalStage(child, signal);+ if (!await within(closed, STAGE_CANCEL_GRACE_MS)) {+ writeSync(2, `⚠️ Stage "${stage.name}" outlived its ${STAGE_CANCEL_GRACE_MS / 1000}s cancellation grace; killing its process group.\n`);+ signalStage(child, 'SIGKILL');+ if (!await within(closed, STAGE_SETTLE_MS)) return { settled: false, reason: 'stage process did not exit after SIGKILL' };+ }+ if (IS_WIN32) return { settled: true };+ // The leader is reaped, but a descendant that ignored the signal keeps the+ // group (and its open log descriptors) alive. ESRCH is the proof of empty.+ const deadline = Date.now() + STAGE_SETTLE_MS;+ for (;;) {+ try {+ process.kill(-child.pid, 'SIGKILL');+ } catch (error) {+ return error.code === 'ESRCH' ? { settled: true } : { settled: false, reason: `group kill failed (${error.code ?? 'unknown'})` };}- child.once('error', (error) => {- cleanup();- resolve({ error });- });- child.once('close', (status, signal) => {- cleanup();- resolve({ status, signal, pid: child.pid });- });+ if (Date.now() >= deadline) return { settled: false, reason: 'group members remained after SIGKILL' };+ await new Promise((resolve) => setTimeout(resolve, 25));+ }+}++/** Run one stage; an owner signal during it resolves only after the stage group settled. */+async function runStage(stage, owner) {+ const env = stageSpawnEnv(process.env, stage.env);+ const { command, args } = stageSpawn(stage, env);+ const child = spawn(command, args, {+ cwd: REPO_ROOT,+ stdio: 'inherit',+ env,+ // A separate group lets cancellation reach only this stage's descendants.+ detached: !IS_WIN32,});+ const closed = new Promise((resolve) => {+ child.once('error', (error) => resolve({ error }));+ child.once('close', (status, signal) => resolve({ status, signal }));+ });+ let cancellation = null;+ let interrupted = null;+ owner.onSignal = (signal) => {+ interrupted = signal;+ cancellation = cancelStage(stage, child, closed, signal);+ };+ const outcome = await closed;+ owner.onSignal = null;+ if (!cancellation) return { ...outcome, pid: child.pid };+ return { ...outcome, pid: child.pid, interrupted, ...await cancellation };}const formatDuration = (ms) => (ms >= 60_000 ? `${(ms / 60_000).toFixed(1)}m` : `${(ms / 1000).toFixed(1)}s`);@@ -612,8 +677,9 @@ async function main() {const results = stages.map((stage) => ({ ...stage, command: describeStageCommand(stage), status: 'not-run', durationMs: 0 }));let exitCode = 0;let failure = null;+ const owner = installOwnerCancellation();for (const result of results) {- if (failure) break;+ if (failure || owner.signal) break;if (result.suite !== 'scan') {try {hashTree();@@ -625,9 +691,16 @@ async function main() {}console.log(`▶️ ${result.name}`);const started = Date.now();- const outcome = await runStage(result);+ const outcome = await runStage(result, owner);result.durationMs = Date.now() - started;- if (outcome.error) {+ if (outcome.interrupted) {+ result.status = 'failed';+ result.signal = outcome.interrupted;+ failure = outcome.settled+ ? `❌ Pregate incomplete: stage="${result.name}" cancelled by ${outcome.interrupted}; its process group has exited.`+ : `❌ Pregate incomplete: stage="${result.name}" cancelled by ${outcome.interrupted}; owned process group settlement unverified (${outcome.reason}).`;+ exitCode = 128 + constants.signals[outcome.interrupted];+ } else if (outcome.error) {result.status = 'failed';failure = `❌ ${result.name} could not start: ${outcome.error.message}`;exitCode = 1;@@ -648,6 +721,12 @@ async function main() {}}+ if (owner.signal && !failure) {+ writeSync(2, `❌ Pregate incomplete: owner_pid=${process.pid} received_signal=${owner.signal} between stages.\n`);+ failure = `❌ Pregate incomplete: cancelled by ${owner.signal} before every stage ran.`;+ exitCode = 128 + constants.signals[owner.signal];+ }+let treeAfterRun = null;try {if (!failure) hashTree();@@ -675,11 +754,15 @@ async function main() {console.warn(`⚠️ The working tree changed while pregate ran (${tree.slice(0, 12)} → ${treeAfterRun?.slice(0, 12) ?? 'unhashable'}); the receipt covers the tree it started with.`);}+ owner.release();if (failure) {console.error(failure);console.error(`📄 Receipt: ${PREGATE_REPORT_PATH}`);- process.exit(exitCode);}+ // Every owned group has settled and the receipt is written; end the way the+ // caller's signal asked. A signal after the last stage leaves its truthful receipt.+ if (owner.signal) process.kill(process.pid, owner.signal);+ if (failure) process.exit(exitCode);const ran = results.map((stage) => stage.name).join(', ');console.log(`✅ Pregate passed: ${ran}.`);diff --git a/scripts/pregate.test.js b/scripts/pregate.test.jsindex 6cd5d1fca9..9a461aa603 100644--- a/scripts/pregate.test.js+++ b/scripts/pregate.test.js@@ -161,7 +161,7 @@ describe('pregate hidden-content invocation', () => {mkdirSync(dirname(join(root, path)), { recursive: true });copyFileSync(join(REPO_ROOT, path), join(root, path));}- write('package.json', '{"type":"module"}\n');+ write('package.json', '{"type":"module","scripts":{"pregate":"node scripts/pregate.js"}}\n');// The receipt lands under /.cache/, which the real repository ignores too.write('.gitignore', 'node_modules\n/.cache/\n');for (const workspace of ['', 'client', 'server', 'autofixer']) {@@ -335,6 +335,74 @@ describe('pregate hidden-content invocation', () => {}});+ // #12641: a Ctrl-C reaches pregate from the terminal group AND from npm's+ // forwarding; the duplicate used to kill pregate before it signalled the+ // stage, orphaning the runner. The fixture stage mirrors run-ci-tests.js: a+ // launcher that settles its own separately grouped child on a signal, plus+ // an in-group descendant that ignores signals and keeps writing.+ it.skipIf(process.platform === 'win32').each(['SIGINT', 'SIGTERM'])('settles every owned stage process before a %s through npm finishes', async (signal) => {+ const heartbeat = join(scratch, 'heartbeat');+ write('scripts/scan-diff-hidden-content.js', `+ import { spawn } from 'child_process';+ const straggler = spawn(process.execPath, ['-e', "process.on('SIGINT', () => {}); process.on('SIGTERM', () => {});"+ + " setInterval(() => require('fs').appendFileSync(process.env.FIXTURE_HEARTBEAT, '.'), 10);"], { stdio: 'ignore' });+ const launched = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore', detached: true });+ const settle = () => {+ process.kill(-launched.pid, 'SIGKILL');+ launched.once('exit', () => setTimeout(() => process.exit(1), 100));+ };+ process.on('SIGINT', settle);+ process.on('SIGTERM', settle);+ setTimeout(() => console.log('fixture-stage-ready ' + JSON.stringify([process.pid, straggler.pid, launched.pid])), 200);+ setInterval(() => {}, 1000);+ `);+ const unrelated = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore', detached: true });+ const npm = spawn('npm', ['run', 'pregate', '--', '--base', base, '--skip-lint'], {+ cwd: root, stdio: ['ignore', 'pipe', 'pipe'], detached: true, env: { ...process.env, FIXTURE_HEARTBEAT: heartbeat },+ });+ const alive = (pid) => {+ try { process.kill(pid, 0); return true; } catch (error) { return error.code !== 'ESRCH'; }+ };+ let owned = [];+ let stdout = '';+ let stderr = '';+ npm.stderr.on('data', (data) => { stderr += data; });+ npm.stdout.on('data', (data) => {+ stdout += data;+ const ready = /fixture-stage-ready (\[[\d,]+\])/.exec(stdout);+ if (ready && owned.length === 0) {+ owned = JSON.parse(ready[1]);+ // Ctrl-C reaches the whole foreground group: npm and pregate alike.+ process.kill(-npm.pid, signal);+ }+ });+ try {+ // `exit`, not `close`: a surviving descendant would hold the pipes open.+ const result = await new Promise((resolve, reject) => {+ npm.once('error', reject);+ npm.once('exit', (status, exitSignal) => resolve({ status, signal: exitSignal }));+ });+ expect(owned).toHaveLength(3);+ expect(owned.filter(alive)).toEqual([]);+ expect(result.status === null ? result.signal : result.status).not.toBe(0);+ expect(stderr).toMatch(new RegExp(`stage="hidden-content scan" owner_pid=\\d+ child_pid=${owned[0]} received_signal=${signal}`));+ expect(stderr).toContain(`stage="hidden-content scan" cancelled by ${signal}; its process group has exited.`);+ expect(stdout).not.toContain('Pregate passed');+ const report = JSON.parse(readFileSync(join(root, PREGATE_REPORT_PATH), 'utf8'));+ expect(report.passed).toBe(false);+ expect(report.stages[0]).toMatchObject({ name: 'hidden-content scan', status: 'failed', signal });+ const written = readFileSync(heartbeat, 'utf8').length;+ await new Promise((resolve) => setTimeout(resolve, 100));+ expect(readFileSync(heartbeat, 'utf8').length).toBe(written);+ expect(alive(unrelated.pid)).toBe(true);+ } finally {+ for (const pid of [npm.pid, unrelated.pid, ...owned]) {+ try { process.kill(-pid, 'SIGKILL'); } catch { /* not a live group */ }+ try { process.kill(pid, 'SIGKILL'); } catch { /* already gone */ }+ }+ }+ });+it.skipIf(process.platform === 'win32')('reports a stage child signal as incomplete rather than an ordinary assertion failure', () => {write('scripts/scan-diff-hidden-content.js', "process.kill(process.pid, 'SIGTERM');\n");const result = run('--skip-lint');@@ -868,10 +936,14 @@ describe('uncoveredSuites', () => {describe('stageSpawnEnv', () => {it('applies stage values and removes the variables a stage sets to undefined', () => {- const env = stageSpawnEnv({ TEST_DB_OK: '1', PGDATABASE: 'portos', KEEP: 'x' }, { TEST_DB_OK: undefined, PGDATABASE: 'portos_test' });+ const env = stageSpawnEnv({ TEST_DB_OK: '1', PGDATABASE: 'portos', KEEP: 'x' }, { TEST_DB_OK: undefined, PGDATABASE: 'portos_test' }, { ownsGroup: false });expect(env).toEqual({ KEEP: 'x', PGDATABASE: 'portos_test', PORTOS_PREGATE_MAX_WORKERS: '4' });expect('TEST_DB_OK' in env).toBe(false);});++ it('keeps the test runner\'s npm and Vitest inside the stage group pregate owns', () => {+ expect(stageSpawnEnv({}, {}, { ownsGroup: true }).PORTOS_CI_TEST_SUPERVISED_GROUP).toBe('1');+ });});describe('testDatabaseStageEnv', () => {diff --git a/scripts/run-ci-tests.js b/scripts/run-ci-tests.jsindex 7f22f0a0ee..cfca7e83c9 100644--- a/scripts/run-ci-tests.js+++ b/scripts/run-ci-tests.js@@ -271,9 +271,10 @@ function runNpmWithReport(scope, script, extraArgs, reportPath, env, abortSignalpersistTestDiagnostic(invocationPath, invocation);return new Promise((resolve) => {// npm is a launcher, not the entire test tree. Give this invocation its- // own POSIX group; Windows killProcessTree uses taskkill /T instead. The- // browser supervisor instead owns the inherited group, so its independent- // watchdog can retire npm even if this launcher's event loop is stopped.+ // own POSIX group; Windows killProcessTree uses taskkill /T instead. A+ // supervisor (the browser watchdog, or pregate's stage group) instead owns+ // the inherited group, so it can retire npm and Vitest even after this+ // launcher's event loop stopped or the launcher itself was killed.const processGroup = process.platform !== 'win32' && env.PORTOS_CI_TEST_SUPERVISED_GROUP !== '1';const childEnv = { ...env, PORTOS_CI_DIAGNOSTICS_PATH: runReportPath };// Only the workflow launcher owns durable attempts. Nested launcher