Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix(tests): retain process ownership and verify containment settlement (#13301)

[._.]/ Adam Eivyc3c74cd1d803
diff --git a/client/vitest.config.js b/client/vitest.config.js
index 67eb880ff9..89e6dd8be1 100644
--- a/client/vitest.config.js
+++ b/client/vitest.config.js
@@ -51,7 +51,7 @@ const config = defineConfig({
// largest CI phase, and happy-dom cuts it by roughly two thirds for the same
// 10k assertions. Files that need no DOM at all still opt out entirely with a
// `// @vitest-environment node` pragma — see docs/GITHUB_ACTIONS.md.
- environment: 'happy-dom',
+ environment: fileURLToPath(new URL('../scripts/vitestProcessEnvironment.js', import.meta.url)),
// Vitest 5 clears mock call history before every test by default
// (`clearMocks`); see the longer note in server/vitest.config.js. Accepted
// rather than pinned back to the vitest 4 default — implementations set by
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
index 24e4ac3dd0..41713bb47a 100644
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@@ -89,6 +89,18 @@ See [VERSIONING.md](./VERSIONING.md) for full details.
Pregate first checks installed package versions against all four workspace lockfiles without modifying dependency trees. Stale, missing, or unreadable evidence stops lint/tests with a reconciliation diagnostic. Pregate requires development dependencies (`npm ci --include=dev` in an owning workspace); a production-only installation cannot run its checks. Ordinary worktree links must be repaired only in their owning source checkout; a separate clone with its own locked installs is also suitable for validation. `--plan-only` prints the plan without checking installed dependencies.
+Test process containment keeps signal delivery separate from verified termination.
+After worker-pool closure, any discovered owned child gets a bounded report in
+`.cache/test-process-diagnostics/`; the CI launcher also retains `processes.json`
+in its attempt diagnostics. Reports contain repository-relative suite names,
+bounded worker IDs and fixed lifecycle/settlement categories, never raw process
+arguments, environments, PIDs or private paths. Suite attribution is captured at
+child construction and matched by a hash of the PID and birth identity. Children
+that bypass the observer (including grandchildren whose parent already exited)
+remain explicitly unattributed. Unknown/unconfirmed settlement retains the run
+root and fails the run; a successful signal alone does not authorize removal.
+Interrupted-run sweeps retain roots while their deferred escalation is pending.
+
Pregate proves only the stages it runs, and it says which it did not. A scoped
plan runs every suite CI would run for it, in CI's dependency order and with
CI's environment: the hidden-content scan, early structural guards, lint,
diff --git a/scripts/lib/darwinTestProcesses.py b/scripts/lib/darwinTestProcesses.py
index 00ce84ed57..5fa74a4b38 100644
--- a/scripts/lib/darwinTestProcesses.py
+++ b/scripts/lib/darwinTestProcesses.py
@@ -18,7 +18,7 @@ def snapshot():
values = layout.unpack(buf.raw)
if values[3] != pid or values[5] != os.getuid():
return None
- return {'pid': pid, 'ppid': values[4], 'start': f'{values[-2]}:{values[-1]}'}
+ return {'pid': pid, 'ppid': values[4], 'start': f'{values[-2]}:{values[-1]}', 'terminal': values[1] == 5}
requested = json.load(sys.stdin)
if requested is None:
diff --git a/scripts/lib/testOwnedProcesses.js b/scripts/lib/testOwnedProcesses.js
index 11fcb360d4..3acd680480 100644
--- a/scripts/lib/testOwnedProcesses.js
+++ b/scripts/lib/testOwnedProcesses.js
@@ -7,17 +7,17 @@ const ownershipKeys = new Set(['PORTOS_TEST_TEMP_ROOT', 'TMPDIR', 'VITEST', 'NOD
function legacyBrowserProcesses(pids, exec) {
try {
- const output = exec('ps', [...(pids ? ['-p', pids.join(',')] : ['-A']), '-ww', '-o', 'pid=,ppid=,uid=,lstart=,args='], {
+ const output = exec('ps', [...(pids ? ['-p', pids.join(',')] : ['-A']), '-ww', '-o', 'pid=,ppid=,uid=,stat=,lstart=,args='], {
encoding: 'utf8', timeout: 5000, maxBuffer: 16 * 1024 * 1024,
stdio: ['ignore', 'pipe', 'ignore'], env: { ...process.env, LC_ALL: 'C' },
});
return output.split('\n').flatMap(line => {
- const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\w{3}\s+\w{3}\s+\d+\s+\d{2}:\d{2}:\d{2}\s+\d{4})\s+(.*)$/.exec(line);
+ const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\w{3}\s+\w{3}\s+\d+\s+\d{2}:\d{2}:\d{2}\s+\d{4})\s+(.*)$/.exec(line);
if (!match || Number(match[3]) !== process.getuid()) return [];
// Compatibility only: this is the pre-existing browser flag rule. Flat
// ps text is never used to infer generic argv or environment ownership.
- const profile = /(?:^|\s)--user-data-dir=(\S+)/.exec(match[5])?.[1];
- return [{ pid: Number(match[1]), ppid: Number(match[2]), start: `legacy:${match[4]}`,
+ const profile = /(?:^|\s)--user-data-dir=(\S+)/.exec(match[6])?.[1];
+ return [{ pid: Number(match[1]), ppid: Number(match[2]), start: `legacy:${match[5]}`, terminal: match[4].startsWith('Z'),
args: profile ? [`--user-data-dir=${profile}`] : [], env: {} }];
});
} catch { return null; }
@@ -28,8 +28,7 @@ function linuxIdentity(pid) {
if (statSync(path).uid !== process.getuid()) return null;
const stat = readFileSync(join(path, 'stat'), 'utf8');
const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
- if (fields[0] === 'Z') return null;
- return { pid, ppid: Number(fields[1]), start: fields[19] };
+ return { pid, ppid: Number(fields[1]), start: fields[19], terminal: fields[0] === 'Z' };
}
function linuxProcess(pid) {
@@ -117,3 +116,20 @@ export function processAncestors(records, pid = process.pid) {
}
return ancestors;
}
+
+/** A missing inventory row is not proof of exit: readers can lose permission. */
+export function probeTestPid(pid) {
+ try { process.kill(pid, 0); return true; }
+ catch (error) { return error.code === 'ESRCH' ? false : null; }
+}
+
+export function testProcessSettlement(record, records, probe = probeTestPid) {
+ const current = Array.isArray(records) ? records.find(item => item.pid === record.pid) : null;
+ if (!current) return probe(record.pid) === false ? 'terminated' : 'unknown';
+ if (typeof current.start !== 'string' || !current.start) return 'unknown';
+ if (current.terminal === true) return 'terminated';
+ // A fallback reader uses a different identity encoding; that is not PID reuse.
+ if (current.start.startsWith('legacy:') !== record.start.startsWith('legacy:')) return 'unknown';
+ if (current.start !== record.start) return 'terminated';
+ return 'running';
+}
diff --git a/scripts/lib/testProcessAttribution.js b/scripts/lib/testProcessAttribution.js
new file mode 100644
index 0000000000..958c0b0062
--- /dev/null
+++ b/scripts/lib/testProcessAttribution.js
@@ -0,0 +1,100 @@
+import { channel } from 'node:diagnostics_channel';
+import { createHash, randomUUID } from 'node:crypto';
+import { mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import { join, relative, sep } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { readTestProcesses } from './testOwnedProcesses.js';
+
+export const PROCESS_OWNERS_DIR = '.process-owners';
+const repo = fileURLToPath(new URL('../../', import.meta.url));
+const spawned = channel('child_process');
+const limit = 64;
+const safeSuite = value => typeof value === 'string' && value.length <= 240
+ && /^(?:server|client|scripts|lib|autofixer)\/(?:[\w.-]+\/)*[\w.-]+\.(?:test|spec)\.[cm]?[jt]sx?$/.test(value)
+ && !value.split('/').includes('..');
+const suiteIdentity = value => {
+ if (typeof value !== 'string') return 'unavailable';
+ const path = relative(repo, value).split(sep).join('/');
+ return safeSuite(path) ? path : 'unavailable';
+};
+const workerIdentity = value => /^\d{1,6}$/.test(value ?? '') ? String(value) : 'unavailable';
+export const processIdentity = ({ pid, start }) => createHash('sha256').update(`${pid}:${start}`).digest('hex');
+
+/** Capture the suite at child construction, never the suite active at cleanup. */
+export function observeTestProcesses(getSuite, { root = process.env.PORTOS_TEST_TEMP_ROOT, read = readTestProcesses } = {}) {
+ if (!root) return () => {};
+ const directory = join(root, PROCESS_OWNERS_DIR);
+ const path = join(directory, `${randomUUID()}.json`);
+ const children = new Map();
+ const worker = workerIdentity(process.env.VITEST_WORKER_ID);
+ let phase = 'suite-active';
+ let capped = false;
+ const persist = () => {
+ try {
+ if (!children.size && !capped) { rmSync(path, { force: true }); return; }
+ mkdirSync(directory, { recursive: true });
+ writeFileSync(`${path}.tmp`, JSON.stringify({ capped, children: [...children.values()].map(item => item.row) }));
+ renameSync(`${path}.tmp`, path);
+ } catch { /* missing evidence is explicitly unavailable at containment */ }
+ };
+ const listener = ({ process: child }) => {
+ if (children.size >= limit) { capped = true; persist(); return; }
+ let suite = 'unavailable';
+ try { suite = suiteIdentity(getSuite()); } catch { /* diagnostic context is optional */ }
+ const row = { identity: null, suite, worker, phase, lifecycle: 'constructed' };
+ const onSpawn = () => {
+ try {
+ const record = read([child.pid])?.find(item => item.pid === child.pid);
+ if (record?.start && !record.terminal) row.identity = processIdentity(record);
+ } catch { /* attribution remains unavailable, never guess by PID */ }
+ row.lifecycle = 'spawned'; persist();
+ };
+ const onExit = () => { row.lifecycle = 'exit-observed'; persist(); };
+ const onClose = () => { children.delete(child); persist(); };
+ children.set(child, { row, onSpawn, onExit, onClose });
+ child.once('spawn', onSpawn);
+ child.once('exit', onExit);
+ child.once('close', onClose);
+ };
+ spawned.subscribe(listener);
+ return () => {
+ spawned.unsubscribe(listener);
+ phase = 'suite-ended';
+ for (const [child, item] of children) {
+ item.row.phase = phase;
+ child.removeListener('spawn', item.onSpawn);
+ child.removeListener('exit', item.onExit);
+ child.removeListener('close', item.onClose);
+ }
+ persist();
+ };
+}
+
+/** All on-disk input is reprojected; no raw process data enters the report. */
+export function processAttribution(root, identity) {
+ const unavailable = state => ({ suite: 'unavailable', worker: 'unavailable', phase: 'unavailable', lifecycle: 'unavailable', attribution: state });
+ try {
+ const directory = join(root, PROCESS_OWNERS_DIR);
+ const names = readdirSync(directory);
+ if (names.length > 256) return unavailable('capped');
+ let capped = false;
+ let match;
+ for (const name of names) {
+ if (!/^[\da-f-]{36}\.json$/.test(name)) continue;
+ const path = join(directory, name);
+ if (statSync(path).size > 48_000) return unavailable('capped');
+ const data = JSON.parse(readFileSync(path, 'utf8'));
+ if (!Array.isArray(data.children) || data.children.length > limit) return unavailable('unavailable');
+ capped ||= data.capped === true;
+ for (const item of data.children) {
+ if (item.identity !== identity) continue;
+ if (match) return unavailable('ambiguous');
+ match = { suite: safeSuite(item.suite) ? item.suite : 'unavailable', worker: workerIdentity(item.worker),
+ phase: ['suite-active', 'suite-ended'].includes(item.phase) ? item.phase : 'unavailable',
+ lifecycle: ['constructed', 'spawned', 'exit-observed'].includes(item.lifecycle) ? item.lifecycle : 'unavailable',
+ attribution: 'observed' };
+ }
+ }
+ return match ?? unavailable(capped ? 'capped' : 'unavailable');
+ } catch { return unavailable('unavailable'); }
+}
diff --git a/scripts/lib/vitestStaleRunRoots.js b/scripts/lib/vitestStaleRunRoots.js
index 77254398e0..ed20220c83 100644
--- a/scripts/lib/vitestStaleRunRoots.js
+++ b/scripts/lib/vitestStaleRunRoots.js
@@ -10,12 +10,18 @@
* to the conservative 6h age rule.
*/
import { execFileSync } from 'node:child_process';
-import { lstatSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { existsSync, lstatSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { basename, dirname, join } from 'node:path';
-import { processAncestors, readTestProcesses, testProcessRoot } from './testOwnedProcesses.js';
+import { processAncestors, probeTestPid, readTestProcesses, testProcessRoot, testProcessSettlement } from './testOwnedProcesses.js';
+import { processAttribution, processIdentity } from './testProcessAttribution.js';
import { killWithEscalation } from '../../server/lib/killWithEscalation.js';
export const OWNER_FILE = '.owner.pid';
+export const PENDING_FILE = '.process-settlement-pending';
+
+function markPending(root) {
+ try { writeFileSync(join(root, PENDING_FILE), 'unverified\n'); } catch { /* preserve in this snapshot even if the stamp fails */ }
+}
export const STALE_ROOT_AGE_MS = 6 * 60 * 60 * 1000;
// `ps -o lstart=` has 1s resolution and the recorded start is derived from
// process.uptime(), so allow slack before calling a pid "reused".
@@ -123,9 +129,9 @@ function ownedSnapshot(parent, prefix, read) {
const ancestors = processAncestors(records);
if (!ancestors) return null;
const owned = records.flatMap(record => {
- if (!Number.isSafeInteger(record.pid) || record.pid <= 1 || typeof record.start !== 'string' || !record.start) return [];
+ if (record.terminal || !Number.isSafeInteger(record.pid) || record.pid <= 1 || typeof record.start !== 'string' || !record.start) return [];
const root = testProcessRoot(record, parent, prefix);
- return root && !ancestors.has(record.pid) ? [{ pid: record.pid, start: record.start, root, kind: processKind(record), parent: ancestors.has(record.ppid) ? 'runner' : 'descendant' }] : [];
+ return root && !ancestors.has(record.pid) ? [{ pid: record.pid, start: record.start, root, kind: processKind(record), parent: ancestors.has(record.ppid) ? 'runner' : 'descendant', identity: processIdentity(record) }] : [];
});
return { ancestors, owned };
}
@@ -148,17 +154,22 @@ export function terminatePid(record, { valid, kill = (pid, signal) => process.ki
{ label: 'Orphaned test process', stillRunning: valid, delayMs: 2000 });
}
-/** Only individually proven processes are signalled; an unowned group member is never collateral cleanup. */
+/** Only individually proven processes are signalled; an unowned group member is never collateral cleanup.
+ * Synchronous settlement also serves Node's exit hook, where promises cannot be awaited.
+ */
export function killTestProcessesUnder(root, { read = readTestProcesses, probe = isOwnerAlive,
- kill = (pid, signal) => process.kill(pid, signal), onKilled = () => {} } = {}) {
+ kill = (pid, signal) => process.kill(pid, signal), onKilled = () => {}, onOutcome = () => {}, onInspection = () => {},
+ probePid = probeTestPid, now = () => performance.now(), wait = ms => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms),
+ settlementMs = 2000 } = {}) {
let rootExists = true;
- try { if (!lstatSync(root).isDirectory()) return 0; } catch (error) {
+ try { const stat = lstatSync(root); if (!stat.isDirectory() || stat.isSymbolicLink()) return 0; } catch (error) {
if (error.code !== 'ENOENT') return 0;
rootExists = false;
}
const parent = dirname(root);
const prefix = basename(root);
const snapshot = ownedSnapshot(parent, prefix, read);
+ onInspection(snapshot !== null);
if (!snapshot) return 0;
const owner = readOwner(root);
if (rootExists && !owner) return 0;
@@ -168,11 +179,39 @@ export function killTestProcessesUnder(root, { read = readTestProcesses, probe =
&& Math.abs(startMs - owner.startMs) <= START_TOLERANCE_MS;
if (!verifiedOwner && probe(owner) !== false) return 0;
}
- let killed = 0;
+ const pending = [];
for (const record of snapshot.owned) {
- if (record.root !== root || !stillOwned(record, snapshot, parent, prefix, read)) continue;
- try { kill(record.pid, 'SIGKILL'); killed++; } catch { continue; /* already gone */ }
- onKilled({ kind: record.kind, parent: record.parent });
+ if (record.root !== root) continue;
+ const verified = stillOwned(record, snapshot, parent, prefix, read);
+ if (!verified && testProcessSettlement(record, read([record.pid]), probePid) === 'terminated') continue;
+ // Read the bounded attribution before signaling; it cannot be reconstructed
+ // after the suite worker or run root disappears.
+ const attribution = processAttribution(root, record.identity);
+ let signal = 'not-attempted';
+ if (verified) {
+ try { kill(record.pid, 'SIGKILL'); signal = 'delivered'; } catch { signal = 'failed'; }
+ }
+ pending.push({ record, attribution, signal, settlement: 'unknown' });
+ }
+ const deadline = now() + settlementMs;
+ while (pending.some(item => item.settlement !== 'terminated')) {
+ const current = read(pending.map(item => item.record.pid));
+ for (const item of pending) {
+ if (item.settlement !== 'terminated') item.settlement = testProcessSettlement(item.record, current, probePid);
+ }
+ if (pending.every(item => item.settlement === 'terminated') || now() >= deadline) break;
+ wait(Math.min(25, deadline - now()));
+ }
+ let killed = 0;
+ for (const { record, attribution, signal, settlement } of pending) {
+ const outcome = { kind: record.kind, parent: record.parent, ...attribution, signal,
+ settlement: settlement === 'running' ? 'unconfirmed' : settlement };
+ onOutcome(outcome);
+ if (settlement !== 'terminated') markPending(root);
+ if (settlement === 'terminated') {
+ killed++;
+ onKilled({ kind: record.kind, parent: record.parent });
+ }
}
return killed;
}
@@ -180,17 +219,21 @@ export function killTestProcessesUnder(root, { read = readTestProcesses, probe =
export function sweepOrphanedTestProcesses(tmpRoot, { prefix = 'pvt-', now = Date.now(), probe = isOwnerAlive,
read = readTestProcesses, terminate = terminatePid } = {}) {
const snapshot = ownedSnapshot(tmpRoot, prefix, read);
- if (!snapshot) return { killed: 0, inspected: false, preserved: new Set() };
+ if (!snapshot) return { signaled: 0, inspected: false, preserved: new Set() };
const preserved = new Set();
- let killed = 0;
+ let signaled = 0;
for (const record of snapshot.owned) {
const valid = () => stillOwned(record, snapshot, tmpRoot, prefix, read)
&& rootIsOrphaned(record.root, probe, Date.now());
if (!rootIsOrphaned(record.root, probe, now) || !valid()) { preserved.add(record.root); continue; }
- try { terminate(record, { valid }); killed++; } catch { /* already gone */ }
+ // Escalation is deferred; retain this root until a later snapshot proves
+ // its children gone. Signal scheduling is not verified termination.
+ preserved.add(record.root);
+ markPending(record.root);
+ try { terminate(record, { valid }); signaled++; } catch { /* settlement unknown */ }
}
- if (killed) console.warn(`🧹 Stopped ${killed} test process${killed === 1 ? '' : 'es'} left running by an interrupted test run`);
- return { killed, inspected: true, preserved };
+ if (signaled) console.warn(`🧹 Requested termination of ${signaled} interrupted-run test processes; settlement pending`);
+ return { signaled, inspected: true, preserved };
}
// Retain the established import boundary while callers adopt the broader name.
@@ -213,7 +256,7 @@ export function sweepStaleRunRoots(tmpRoot, { prefix = 'pvt-', now = Date.now(),
for (const name of names) {
if (!name.startsWith(prefix)) continue;
const path = join(tmpRoot, name);
- if (preserved.has(path)) continue;
+ if (preserved.has(path) || (!inspected && existsSync(join(path, PENDING_FILE)))) continue;
try {
const stat = lstatSync(path);
if (!stat.isDirectory() || stat.isSymbolicLink()) continue;
diff --git a/scripts/run-ci-tests.js b/scripts/run-ci-tests.js
index 35d0aaf2b4..3c85bda154 100644
--- a/scripts/run-ci-tests.js
+++ b/scripts/run-ci-tests.js
@@ -1,7 +1,7 @@
#!/usr/bin/env node
import { spawn } from 'child_process';
-import { mkdirSync, mkdtempSync, readFileSync, statSync, rmSync } from 'node:fs';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, statSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { stripVTControlCharacters } from 'node:util';
import { dirname, join } from 'path';
@@ -224,9 +224,11 @@ export async function runNpm(scope, script, extraArgs, { env = process.env, sign
const reportDir = mkdtempSync(join(hostRoot, 'portos-vitest-'));
const reportPath = join(reportDir, 'failures.json');
let cleanupUnverified = false;
+ let processEvidence = false;
try {
writeOwnerFile(reportDir);
const launched = await runNpmWithReport(scope, script, extraArgs, reportPath, env, signal, cancellationTimeoutMs);
+ processEvidence = existsSync(join(reportDir, 'processes.json'));
cleanupUnverified = launched.cleanupUnverified === true;
if (cleanupUnverified) {
console.error(`Test launcher cleanup unverified; diagnostics retained at ${reportDir}`);
@@ -235,6 +237,7 @@ export async function runNpm(scope, script, extraArgs, { env = process.env, sign
// Per-file outcomes and the run-level verdict feed the test memo; an interrupted run proves nothing.
const result = {
...launched,
+ ...(processEvidence ? { diagnosticsPath: reportDir } : {}),
fileResults: launched.interrupted ? [] : fileResultsFromReport(reportPath),
runReport: launched.interrupted ? null : readRunReport(join(reportDir, 'run.json')),
};
@@ -249,7 +252,7 @@ export async function runNpm(scope, script, extraArgs, { env = process.env, sign
}
return result;
} finally {
- if (!durable && !cleanupUnverified) rmSync(reportDir, { recursive: true, force: true });
+ if (!durable && !cleanupUnverified && !processEvidence) rmSync(reportDir, { recursive: true, force: true });
}
}
@@ -277,7 +280,7 @@ function runNpmWithReport(scope, script, extraArgs, reportPath, env, abortSignal
// the inherited group, so it can retire npm and Vitest even after this
// launcher's event loop stopped or the launcher itself was killed.
const processGroup = process.platform !== 'win32' && env.PORTOS_CI_TEST_SUPERVISED_GROUP !== '1';
- const childEnv = { ...env, PORTOS_CI_DIAGNOSTICS_PATH: runReportPath };
+ const childEnv = { ...env, PORTOS_CI_DIAGNOSTICS_PATH: runReportPath, PORTOS_TEST_PROCESS_DIAGNOSTICS_PATH: join(dirname(reportPath), 'processes.json') };
// Only the workflow launcher owns durable attempts. Nested launcher
// fixtures must not overwrite or add to the job's failure provenance.
delete childEnv.PORTOS_CI_DIAGNOSTICS_DIR;
diff --git a/scripts/run-ci-tests.test.js b/scripts/run-ci-tests.test.js
index 5c3ced1de4..c70264998f 100644
--- a/scripts/run-ci-tests.test.js
+++ b/scripts/run-ci-tests.test.js
@@ -800,6 +800,28 @@ describe('structured failure diagnostics', () => {
}, 35_000);
});
+// A containment report written after the ordinary reporter snapshot must survive
+// the launcher's nondurable-attempt cleanup, including otherwise green runs.
+it('retains process settlement evidence from a completed launcher attempt', async () => {
+ const fixture = mkdtempSync(join(tmpdir(), 'process-report-launcher-'));
+ let diagnosticsPath;
+ try {
+ installFakeNpm(fixture, `require('node:fs').writeFileSync(process.env.PORTOS_TEST_PROCESS_DIAGNOSTICS_PATH,
+ JSON.stringify({ version: 1, total: 1, terminated: 1, unsettled: 0, processes: [] }));`);
+ const { runNpm } = await import('./run-ci-tests.js');
+ const result = await runNpm('server', 'test:ci', [], {
+ env: { ...process.env, PORTOS_CI_DIAGNOSTICS_DIR: '', PATH: `${fixture}${delimiter}${process.env.PATH}` },
+ });
+ diagnosticsPath = result.diagnosticsPath;
+ expect(result.status).toBe(0);
+ expect(diagnosticsPath).toBeTruthy();
+ expect(JSON.parse(readFileSync(join(diagnosticsPath, 'processes.json'), 'utf8'))).toMatchObject({ total: 1, terminated: 1 });
+ } finally {
+ if (diagnosticsPath) rmSync(diagnosticsPath, { recursive: true, force: true });
+ rmSync(fixture, { recursive: true, force: true });
+ }
+});
+
describe('report launcher subprocess lifecycle (#9804)', () => {
const runnerUrl = new URL('./run-ci-tests.js', import.meta.url).href;
diff --git a/scripts/testOwnedProcessRecovery.test.js b/scripts/testOwnedProcessRecovery.test.js
index 6feffa64e6..8304ee91d0 100644
--- a/scripts/testOwnedProcessRecovery.test.js
+++ b/scripts/testOwnedProcessRecovery.test.js
@@ -5,6 +5,7 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { STALE_ROOT_AGE_MS, killTestProcessesUnder, sweepStaleRunRoots, writeOwnerFile } from './lib/vitestStaleRunRoots.js';
+import { observeTestProcesses } from './lib/testProcessAttribution.js';
import { processAncestors, readTestProcesses, testProcessRoot } from './lib/testOwnedProcesses.js';
import { isProcessAlive } from '../server/test/processAlive.js';
@@ -43,8 +44,8 @@ describe('test process ownership boundaries', () => {
it.skipIf(process.platform === 'win32')('retains only legacy browser ownership when the optional Darwin reader is unavailable', () => {
const exec = vi.fn((command) => {
if (command !== 'ps') throw Error('reader unavailable');
- return `900001 1 ${process.getuid()} Fri Oct 16 12:00:00 2026 chrome --user-data-dir=/tmp/example/pvt-browser/profile\n`
- + `900002 1 ${process.getuid()} Fri Oct 16 12:00:00 2026 node /tmp/example/pvt-script/file.js PORTOS_TEST_TEMP_ROOT=/tmp/example/pvt-env NODE_ENV=test\n`;
+ return `900001 1 ${process.getuid()} S Fri Oct 16 12:00:00 2026 chrome --user-data-dir=/tmp/example/pvt-browser/profile\n`
+ + `900002 1 ${process.getuid()} S Fri Oct 16 12:00:00 2026 node /tmp/example/pvt-script/file.js PORTOS_TEST_TEMP_ROOT=/tmp/example/pvt-env NODE_ENV=test\n`;
});
const records = readTestProcesses(null, { platform: 'darwin', exec });
expect(records.map(record => testProcessRoot(record, '/tmp/example', 'pvt-'))).toEqual(['/tmp/example/pvt-browser', null]);
@@ -66,7 +67,7 @@ describe('test process ownership boundaries', () => {
expect(testProcessRoot({ args: ['chrome', `--user-data-dir=${root}/profile`] }, parent, 'pvt-')).toBe(root);
});
- it('reports only fixed process categories for successfully stopped owned children', () => {
+ it('waits for terminal birth identity after signaling and distinguishes unconfirmed or unknown settlement', () => {
const host = mkdtempSync(join(tmpdir(), 'recovery-diagnostics-'));
const root = join(host, 'pvt-owned');
mkdirSync(root);
@@ -75,16 +76,26 @@ describe('test process ownership boundaries', () => {
const child = { pid: 900001, ppid: process.pid, start: 'original',
args: ['/example/node', '/example/vitest/dist/workers/forks.js', '--secret=fixture-secret'],
env: { NODE_ENV: 'test', PORTOS_TEST_TEMP_ROOT: root } };
- const onKilled = vi.fn();
- const kill = vi.fn();
try {
- expect(killTestProcessesUnder(root, { read: () => [...ancestors, child], probe: () => false, kill, onKilled })).toBe(1);
- expect(kill).toHaveBeenCalledExactlyOnceWith(child.pid, 'SIGKILL');
- expect(onKilled).toHaveBeenCalledExactlyOnceWith({ kind: 'vitest', parent: 'runner' });
- kill.mockImplementation(() => { throw Error('already gone'); });
- onKilled.mockClear();
- expect(killTestProcessesUnder(root, { read: () => [...ancestors, child], probe: () => false, kill, onKilled })).toBe(0);
- expect(onKilled).not.toHaveBeenCalled();
+ for (const mode of ['terminal', 'reused', 'unconfirmed', 'unknown', 'signal-failed']) {
+ let time = 0;
+ let signaled = false;
+ const onKilled = vi.fn();
+ const onOutcome = vi.fn();
+ const read = () => !signaled || time === 0 ? [...ancestors, child]
+ : mode === 'unknown' ? null
+ : [...ancestors, { ...child, terminal: mode === 'terminal', start: mode === 'reused' ? 'replacement' : child.start }];
+ const kill = vi.fn(() => { signaled = true; if (mode === 'signal-failed') throw Error('denied'); });
+ const result = killTestProcessesUnder(root, { read, probe: () => false, probePid: () => true, kill, onKilled, onOutcome,
+ now: () => time, wait: ms => { expect(onKilled).not.toHaveBeenCalled(); time += ms; }, settlementMs: 50 });
+ const terminated = ['terminal', 'reused'].includes(mode);
+ expect(result).toBe(terminated ? 1 : 0);
+ expect(kill).toHaveBeenCalledExactlyOnceWith(child.pid, 'SIGKILL');
+ expect(onOutcome).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ kind: 'vitest', parent: 'runner',
+ signal: mode === 'signal-failed' ? 'failed' : 'delivered',
+ settlement: terminated ? 'terminated' : mode === 'unknown' ? 'unknown' : 'unconfirmed' }));
+ expect(JSON.stringify(onOutcome.mock.calls)).not.toMatch(/fixture-secret|900001|original|example/);
+ }
} finally { rmSync(host, { recursive: true, force: true }); }
});
@@ -102,14 +113,16 @@ describe('test process ownership boundaries', () => {
try {
sweepStaleRunRoots(host, { read, probe: () => ownerAlive });
expect(kill).toHaveBeenCalledExactlyOnceWith(target.pid, 'SIGTERM');
+ sweepStaleRunRoots(host, { read: () => null, probe: () => false });
+ expect(existsSync(root)).toBe(true); // unknown settlement cannot erase a prior pending stamp
target.start = 'reused';
vi.advanceTimersByTime(2000);
expect(kill).toHaveBeenCalledTimes(1);
- mkdirSync(root); writeFileSync(join(root, '.owner.pid'), '999999 0\n');
+ expect(existsSync(root)).toBe(true); writeFileSync(join(root, '.owner.pid'), '999999 0\n');
target.start = 'next';
sweepStaleRunRoots(host, { read, probe: () => ownerAlive });
expect(kill).toHaveBeenCalledTimes(2);
- mkdirSync(root); writeOwnerFile(root); ownerAlive = true;
+ expect(existsSync(root)).toBe(true); writeOwnerFile(root); ownerAlive = true;
vi.advanceTimersByTime(2000);
expect(kill).toHaveBeenCalledTimes(2);
expect(killTestProcessesUnder(root, { read: () => ancestors, probe: () => true, kill })).toBe(0);
@@ -155,7 +168,10 @@ describe.skipIf(process.platform === 'win32')('real interrupted-run recovery', (
const script = join(dead, 'fixture.sh'); writeFileSync(script, 'echo ready\nread answer\n');
const argumentOnly = await launch(['/bin/sh', script]);
const unrelated = await launch([process.execPath, '-e', code], { OTHER: `PORTOS_TEST_TEMP_ROOT=${dead} NODE_ENV=test` });
- const ownedLive = await launch([process.execPath, '-e', code], { NODE_ENV: 'test', PORTOS_TEST_TEMP_ROOT: live });
+ const dispose = observeTestProcesses(() => fileURLToPath(import.meta.url), { root: live });
+ let ownedLive;
+ try { ownedLive = await launch([process.execPath, '-e', code], { NODE_ENV: 'test', PORTOS_TEST_TEMP_ROOT: live }); }
+ finally { dispose(); }
const inspected = readTestProcesses([environmentOnly.pid, argumentOnly.pid, unrelated.pid]);
expect(inspected?.map(record => testProcessRoot(record, host, 'pvt-'))).toEqual([dead, dead, null]);
sweepStaleRunRoots(host);
@@ -165,7 +181,9 @@ describe.skipIf(process.platform === 'win32')('real interrupted-run recovery', (
}, { timeout: 5000 });
expect(isProcessAlive(unrelated.pid)).toBe(true);
expect(isProcessAlive(ownedLive.pid)).toBe(true);
- expect(killTestProcessesUnder(live)).toBe(1);
+ const outcomes = [];
+ expect(killTestProcessesUnder(live, { onOutcome: outcome => outcomes.push(outcome) })).toBe(1);
+ expect(outcomes).toEqual([expect.objectContaining({ suite: 'scripts/testOwnedProcessRecovery.test.js', phase: 'suite-ended', lifecycle: 'spawned', settlement: 'terminated' })]);
await vi.waitFor(() => expect(isProcessAlive(ownedLive.pid)).toBe(false));
} finally {
await Promise.all(children.map(child => new Promise(resolve => {
diff --git a/scripts/testProcessAttribution.test.js b/scripts/testProcessAttribution.test.js
new file mode 100644
index 0000000000..62e34e458d
--- /dev/null
+++ b/scripts/testProcessAttribution.test.js
@@ -0,0 +1,63 @@
+import { channel } from 'node:diagnostics_channel';
+import { EventEmitter } from 'node:events';
+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { afterEach, expect, it, vi } from 'vitest';
+import { observeTestProcesses, processAttribution, processIdentity } from './lib/testProcessAttribution.js';
+import { killTestProcessesUnder } from './lib/vitestStaleRunRoots.js';
+import { cleanupRunTempRoot } from './vitestTempRootSetup.js';
+
+const originalExit = process.exitCode;
+afterEach(() => { process.exitCode = originalExit; vi.restoreAllMocks(); });
+
+// Regression: a suite-ended child must be attributed before the root disappears,
+// while a successful signal cannot remove that root until terminal identity exists.
+it.each(['terminated', 'unconfirmed', 'unknown'])('retains sanitized spawn ownership and %s settlement through run cleanup', settlement => {
+ const host = mkdtempSync(join(tmpdir(), 'process-evidence-'));
+ const root = mkdtempSync(join(host, 'pvt-'));
+ const reportPath = join(host, 'processes.json');
+ const records = [{ pid: process.pid, ppid: process.ppid }, { pid: process.ppid, ppid: 1 }];
+ const record = { pid: 900001, ppid: process.pid, start: 'birth-private-sentinel', args: ['/example/node', 'private-argument'],
+ env: { NODE_ENV: 'test', PORTOS_TEST_TEMP_ROOT: root } };
+ writeFileSync(join(root, '.owner.pid'), '999999 0\n');
+ let suite = fileURLToPath(import.meta.url);
+ const dispose = observeTestProcesses(() => suite, { root, read: () => [record] });
+ const child = Object.assign(new EventEmitter(), { pid: record.pid });
+ vi.spyOn(console, 'warn').mockImplementation(() => {});
+ vi.spyOn(console, 'error').mockImplementation(() => {});
+ try {
+ channel('child_process').publish({ process: child });
+ child.emit('spawn');
+ suite = '/private/not-the-spawning-suite.test.js';
+ dispose();
+ expect(processAttribution(root, processIdentity(record))).toMatchObject({ suite: 'scripts/testProcessAttribution.test.js', phase: 'suite-ended', lifecycle: 'spawned' });
+ expect(processAttribution(root, processIdentity({ ...record, start: 'reused' })).attribution).toBe('unavailable');
+ let time = 0;
+ const contain = (path, callbacks) => killTestProcessesUnder(path, { ...callbacks,
+ read: () => time > 0 && settlement === 'unknown' ? null : [...records, { ...record, terminal: time > 0 && settlement === 'terminated' }],
+ probe: () => false, probePid: () => true, kill: () => {}, now: () => time, wait: ms => { time += ms; }, settlementMs: 50 });
+ cleanupRunTempRoot(root, { contain, reportPath, platform: 'linux' });
+ const report = JSON.parse(readFileSync(reportPath, 'utf8'));
+ expect(report.processes).toEqual([expect.objectContaining({ suite: 'scripts/testProcessAttribution.test.js', phase: 'suite-ended', lifecycle: 'spawned', signal: 'delivered', settlement })]);
+ expect(existsSync(root)).toBe(settlement !== 'terminated');
+ if (settlement !== 'terminated') expect(process.exitCode).toBe(1);
+ expect(JSON.stringify(report)).not.toMatch(/private|900001|999999|birth-|argument|\/tmp\/|\/Users\//);
+ } finally { dispose(); rmSync(host, { recursive: true, force: true }); }
+});
+
+it('retains the root and an explicit diagnostic gap when process inspection fails', () => {
+ const host = mkdtempSync(join(tmpdir(), 'process-inspection-'));
+ const root = mkdtempSync(join(host, 'pvt-'));
+ const reportPath = join(host, 'processes.json');
+ vi.spyOn(console, 'warn').mockImplementation(() => {});
+ vi.spyOn(console, 'error').mockImplementation(() => {});
+ try {
+ cleanupRunTempRoot(root, { reportPath, platform: 'linux',
+ contain: (path, callbacks) => killTestProcessesUnder(path, { ...callbacks, read: () => null }) });
+ expect(JSON.parse(readFileSync(reportPath, 'utf8'))).toMatchObject({ inspected: false, total: 0, terminated: 0, unsettled: 1 });
+ expect(process.exitCode).toBe(1);
+ expect(existsSync(root)).toBe(true);
+ } finally { rmSync(host, { recursive: true, force: true }); }
+});
diff --git a/scripts/vitestProcessEnvironment.js b/scripts/vitestProcessEnvironment.js
new file mode 100644
index 0000000000..cd5031b0d9
--- /dev/null
+++ b/scripts/vitestProcessEnvironment.js
@@ -0,0 +1,20 @@
+import { builtinEnvironments } from 'vitest/runtime';
+import { observeTestProcesses } from './lib/testProcessAttribution.js';
+
+// Keep the observer in Node's environment realm: client transforms replace
+// import.meta.url with HTTP URLs. Environment teardown also runs for skipped
+// suites and failed collection, unlike an afterAll observer cleanup.
+export default {
+ ...builtinEnvironments['happy-dom'],
+ name: 'portos-observed-happy-dom',
+ async setup(global, options) {
+ const environment = await builtinEnvironments['happy-dom'].setup(global, options);
+ const dispose = observeTestProcesses(() => global.__vitest_worker__?.filepath);
+ return {
+ async teardown() {
+ try { dispose(); }
+ finally { await environment.teardown(global); }
+ },
+ };
+ },
+};
diff --git a/scripts/vitestTempRoot.test.js b/scripts/vitestTempRoot.test.js
index 9b7aa4e9b9..d121461180 100644
--- a/scripts/vitestTempRoot.test.js
+++ b/scripts/vitestTempRoot.test.js
@@ -168,6 +168,9 @@ describe.skipIf(process.platform === 'win32')('orphaned test browser sweep', ()
sweepStaleRunRoots(host);
expect(await gone(orphan.pid)).toBe(true);
+ // The first sweep retains ownership while its asynchronous escalation settles.
+ expect(existsSync(dead)).toBe(true);
+ sweepStaleRunRoots(host);
expect(existsSync(dead)).toBe(false);
expect(isProcessAlive(owned.pid)).toBe(true);
diff --git a/scripts/vitestTempRootSetup.js b/scripts/vitestTempRootSetup.js
index 24d2165667..4c067664ec 100644
--- a/scripts/vitestTempRootSetup.js
+++ b/scripts/vitestTempRootSetup.js
@@ -3,16 +3,19 @@
* redirects temp APIs before Vitest creates its module-transform cache.
* The server's exclusive capture projects share this single global owner.
*/
-import { lstatSync, readdirSync, rmSync } from 'node:fs';
-import { join } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { fileURLToPath } from 'node:url';
+import { PROCESS_OWNERS_DIR } from './lib/testProcessAttribution.js';
+import { lstatSync, mkdirSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
import { describeOwnedBrowserObservation, describeStageFixtureRoot, readOwnedBrowserObservations } from '../server/test/ownedBrowserDiagnostics.js';
-import { OWNER_FILE, killTestProcessesUnder } from './lib/vitestStaleRunRoots.js';
+import { OWNER_FILE, PENDING_FILE, killTestProcessesUnder } from './lib/vitestStaleRunRoots.js';
export function setup({ vitest }) {
const root = process.env.PORTOS_TEST_TEMP_ROOT;
// Vitest runs global teardown before closing its worker pool. Its close hook
// runs after pool.close(), so live runner workers are not mistaken for leaks.
- vitest.onClose(() => cleanupRunTempRoot(root));
+ vitest.onClose(() => cleanupRunTempRoot(root, { reportPath: process.env.PORTOS_TEST_PROCESS_DIAGNOSTICS_PATH, durable: true }));
}
// Vitest's nanoid transform scratch still exists when global teardown runs.
@@ -91,18 +94,44 @@ export function isEffectivelyEmpty(path) {
return children.every((child) => isEffectivelyEmpty(join(path, child)));
}
-export function cleanupRunTempRoot(root = process.env.PORTOS_TEST_TEMP_ROOT) {
+export function cleanupRunTempRoot(root = process.env.PORTOS_TEST_TEMP_ROOT, { contain = killTestProcessesUnder, reportPath, durable = false, platform = process.platform } = {}) {
if (!root) return;
// A process a suite failed to tear down would outlive the run (#10840).
- const kinds = new Map();
- const processes = process.platform === 'win32' ? 0 : killTestProcessesUnder(root, {
- onKilled: ({ kind, parent }) => {
- const key = `kind=${kind} parent=${parent}`;
- kinds.set(key, (kinds.get(key) ?? 0) + 1);
+ const outcomes = [];
+ let total = 0;
+ let unsettled = 0;
+ let inspected = true;
+ const processes = platform === 'win32' ? 0 : contain(root, {
+ onInspection: value => { inspected = value; if (!value) unsettled++; },
+ onOutcome: outcome => {
+ total++;
+ if (outcome.settlement !== 'terminated') unsettled++;
+ if (outcomes.length < 64) outcomes.push(outcome);
},
});
- if (processes) console.warn(`⚠️ test process leak: stopped ${processes} process${processes === 1 ? '' : 'es'} still running under the run temp root`);
- for (const [kind, count] of kinds) console.warn(`⚠️ stopped owned test processes: ${kind} count=${count}`);
+ if (total || !inspected) {
+ const report = { version: 1, inspected, boundary: 'pool-closed', total, terminated: processes, unsettled, capped: total > outcomes.length, processes: outcomes };
+ const retainedPath = join(fileURLToPath(new URL('../.cache/test-process-diagnostics/', import.meta.url)), `${randomUUID()}.json`);
+ const destinations = reportPath ? [reportPath, ...(durable ? [retainedPath] : [])] : [retainedPath];
+ let retained = true;
+ try {
+ for (const destination of destinations) {
+ mkdirSync(dirname(destination), { recursive: true });
+ writeFileSync(destination, JSON.stringify(report));
+ }
+ } catch {
+ retained = false;
+ console.error('❌ owned test process evidence could not be persisted');
+ unsettled++;
+ }
+ console.warn(`⚠️ test process leak: signal/settlement evidence=${retained ? 'retained' : 'unavailable'} inspected=${inspected}; found=${total} terminated=${processes} unsettled=${unsettled}`);
+ for (const outcome of outcomes) console.warn(`⚠️ owned test process: ${Object.entries(outcome).map(([key, value]) => `${key}=${value}`).join(' ')}`);
+ }
+ if (unsettled) {
+ process.exitCode = 1;
+ console.error('❌ owned test process settlement unverified; retaining run temp root');
+ return;
+ }
let entries;
try {
entries = readdirSync(root);
@@ -110,7 +139,7 @@ export function cleanupRunTempRoot(root = process.env.PORTOS_TEST_TEMP_ROOT) {
return;
}
entries = entries
- .filter((name) => name !== OWNER_FILE && name !== OWNERS_FILE)
+ .filter((name) => name !== OWNER_FILE && name !== PENDING_FILE && name !== OWNERS_FILE && name !== PROCESS_OWNERS_DIR)
.filter((name) => !VITEST_INTERNAL_SCRATCH_DIR.test(name))
.filter((name) => !TOOLCHAIN_CACHE_PREFIXES.has(groupLeakPrefix(name)))
.filter((name) => !isDotnetRegistrationCache(root, name))
diff --git a/server/test/mediaRegistryEnvironment.js b/server/test/mediaRegistryEnvironment.js
index a87bb8ee1c..cfa4d3c3f8 100644
--- a/server/test/mediaRegistryEnvironment.js
+++ b/server/test/mediaRegistryEnvironment.js
@@ -1,3 +1,4 @@
+import { observeTestProcesses } from '../../scripts/lib/testProcessAttribution.js';
import { observeOwnedBrowsers } from './ownedBrowserDiagnostics.js';
import { builtinEnvironments } from 'vitest/runtime';
import { mkdtempSync, rmSync } from 'node:fs';
@@ -13,6 +14,7 @@ export default {
viteEnvironment: 'ssr',
async setup(global, options) {
const node = await builtinEnvironments.node.setup(global, options);
+ const disposeProcesses = observeTestProcesses(() => global.__vitest_worker__?.filepath);
const disposeObservations = observeOwnedBrowsers(() => global.__vitest_worker__?.filepath);
try {
const prior = process.env.PORTOS_MEDIA_MODELS_FILE;
@@ -22,6 +24,7 @@ export default {
async teardown() {
try {
disposeObservations();
+ disposeProcesses();
if (root) {
if (prior === undefined) delete process.env.PORTOS_MEDIA_MODELS_FILE;
else process.env.PORTOS_MEDIA_MODELS_FILE = prior;
@@ -36,6 +39,7 @@ export default {
// setup can fail before Vitest receives our teardown owner. Preserve that
// original error while ending any partial diagnostic subscription.
disposeObservations();
+ disposeProcesses();
throw error;
}
},