Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix(tests): retain process ownership and verify containment settlement (#13301)
[._.]/ Adam Eivyc3c74cd1d803
diff --git a/client/vitest.config.js b/client/vitest.config.jsindex 67eb880ff9..89e6dd8be1 100644--- a/client/vitest.config.js+++ b/client/vitest.config.js@@ -51,7 +51,7 @@ const config = defineConfig({// largest CI phase, and happy-dom cuts it by roughly two thirds for the same// 10k assertions. Files that need no DOM at all still opt out entirely with a// `// @vitest-environment node` pragma — see docs/GITHUB_ACTIONS.md.- environment: 'happy-dom',+ environment: fileURLToPath(new URL('../scripts/vitestProcessEnvironment.js', import.meta.url)),// Vitest 5 clears mock call history before every test by default// (`clearMocks`); see the longer note in server/vitest.config.js. Accepted// rather than pinned back to the vitest 4 default — implementations set bydiff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.mdindex 24e4ac3dd0..41713bb47a 100644--- a/docs/CONTRIBUTING.md+++ b/docs/CONTRIBUTING.md@@ -89,6 +89,18 @@ See [VERSIONING.md](./VERSIONING.md) for full details.Pregate first checks installed package versions against all four workspace lockfiles without modifying dependency trees. Stale, missing, or unreadable evidence stops lint/tests with a reconciliation diagnostic. Pregate requires development dependencies (`npm ci --include=dev` in an owning workspace); a production-only installation cannot run its checks. Ordinary worktree links must be repaired only in their owning source checkout; a separate clone with its own locked installs is also suitable for validation. `--plan-only` prints the plan without checking installed dependencies.+Test process containment keeps signal delivery separate from verified termination.+After worker-pool closure, any discovered owned child gets a bounded report in+`.cache/test-process-diagnostics/`; the CI launcher also retains `processes.json`+in its attempt diagnostics. Reports contain repository-relative suite names,+bounded worker IDs and fixed lifecycle/settlement categories, never raw process+arguments, environments, PIDs or private paths. Suite attribution is captured at+child construction and matched by a hash of the PID and birth identity. Children+that bypass the observer (including grandchildren whose parent already exited)+remain explicitly unattributed. Unknown/unconfirmed settlement retains the run+root and fails the run; a successful signal alone does not authorize removal.+Interrupted-run sweeps retain roots while their deferred escalation is pending.+Pregate proves only the stages it runs, and it says which it did not. A scopedplan runs every suite CI would run for it, in CI's dependency order and withCI's environment: the hidden-content scan, early structural guards, lint,diff --git a/scripts/lib/darwinTestProcesses.py b/scripts/lib/darwinTestProcesses.pyindex 00ce84ed57..5fa74a4b38 100644--- a/scripts/lib/darwinTestProcesses.py+++ b/scripts/lib/darwinTestProcesses.py@@ -18,7 +18,7 @@ def snapshot():values = layout.unpack(buf.raw)if values[3] != pid or values[5] != os.getuid():return None- return {'pid': pid, 'ppid': values[4], 'start': f'{values[-2]}:{values[-1]}'}+ return {'pid': pid, 'ppid': values[4], 'start': f'{values[-2]}:{values[-1]}', 'terminal': values[1] == 5}requested = json.load(sys.stdin)if requested is None:diff --git a/scripts/lib/testOwnedProcesses.js b/scripts/lib/testOwnedProcesses.jsindex 11fcb360d4..3acd680480 100644--- a/scripts/lib/testOwnedProcesses.js+++ b/scripts/lib/testOwnedProcesses.js@@ -7,17 +7,17 @@ const ownershipKeys = new Set(['PORTOS_TEST_TEMP_ROOT', 'TMPDIR', 'VITEST', 'NODfunction legacyBrowserProcesses(pids, exec) {try {- const output = exec('ps', [...(pids ? ['-p', pids.join(',')] : ['-A']), '-ww', '-o', 'pid=,ppid=,uid=,lstart=,args='], {+ const output = exec('ps', [...(pids ? ['-p', pids.join(',')] : ['-A']), '-ww', '-o', 'pid=,ppid=,uid=,stat=,lstart=,args='], {encoding: 'utf8', timeout: 5000, maxBuffer: 16 * 1024 * 1024,stdio: ['ignore', 'pipe', 'ignore'], env: { ...process.env, LC_ALL: 'C' },});return output.split('\n').flatMap(line => {- const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\w{3}\s+\w{3}\s+\d+\s+\d{2}:\d{2}:\d{2}\s+\d{4})\s+(.*)$/.exec(line);+ const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\w{3}\s+\w{3}\s+\d+\s+\d{2}:\d{2}:\d{2}\s+\d{4})\s+(.*)$/.exec(line);if (!match || Number(match[3]) !== process.getuid()) return [];// Compatibility only: this is the pre-existing browser flag rule. Flat// ps text is never used to infer generic argv or environment ownership.- const profile = /(?:^|\s)--user-data-dir=(\S+)/.exec(match[5])?.[1];- return [{ pid: Number(match[1]), ppid: Number(match[2]), start: `legacy:${match[4]}`,+ const profile = /(?:^|\s)--user-data-dir=(\S+)/.exec(match[6])?.[1];+ return [{ pid: Number(match[1]), ppid: Number(match[2]), start: `legacy:${match[5]}`, terminal: match[4].startsWith('Z'),args: profile ? [`--user-data-dir=${profile}`] : [], env: {} }];});} catch { return null; }@@ -28,8 +28,7 @@ function linuxIdentity(pid) {if (statSync(path).uid !== process.getuid()) return null;const stat = readFileSync(join(path, 'stat'), 'utf8');const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' ');- if (fields[0] === 'Z') return null;- return { pid, ppid: Number(fields[1]), start: fields[19] };+ return { pid, ppid: Number(fields[1]), start: fields[19], terminal: fields[0] === 'Z' };}function linuxProcess(pid) {@@ -117,3 +116,20 @@ export function processAncestors(records, pid = process.pid) {}return ancestors;}++/** A missing inventory row is not proof of exit: readers can lose permission. */+export function probeTestPid(pid) {+ try { process.kill(pid, 0); return true; }+ catch (error) { return error.code === 'ESRCH' ? false : null; }+}++export function testProcessSettlement(record, records, probe = probeTestPid) {+ const current = Array.isArray(records) ? records.find(item => item.pid === record.pid) : null;+ if (!current) return probe(record.pid) === false ? 'terminated' : 'unknown';+ if (typeof current.start !== 'string' || !current.start) return 'unknown';+ if (current.terminal === true) return 'terminated';+ // A fallback reader uses a different identity encoding; that is not PID reuse.+ if (current.start.startsWith('legacy:') !== record.start.startsWith('legacy:')) return 'unknown';+ if (current.start !== record.start) return 'terminated';+ return 'running';+}diff --git a/scripts/lib/testProcessAttribution.js b/scripts/lib/testProcessAttribution.jsnew file mode 100644index 0000000000..958c0b0062--- /dev/null+++ b/scripts/lib/testProcessAttribution.js@@ -0,0 +1,100 @@+import { channel } from 'node:diagnostics_channel';+import { createHash, randomUUID } from 'node:crypto';+import { mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';+import { join, relative, sep } from 'node:path';+import { fileURLToPath } from 'node:url';+import { readTestProcesses } from './testOwnedProcesses.js';++export const PROCESS_OWNERS_DIR = '.process-owners';+const repo = fileURLToPath(new URL('../../', import.meta.url));+const spawned = channel('child_process');+const limit = 64;+const safeSuite = value => typeof value === 'string' && value.length <= 240+ && /^(?:server|client|scripts|lib|autofixer)\/(?:[\w.-]+\/)*[\w.-]+\.(?:test|spec)\.[cm]?[jt]sx?$/.test(value)+ && !value.split('/').includes('..');+const suiteIdentity = value => {+ if (typeof value !== 'string') return 'unavailable';+ const path = relative(repo, value).split(sep).join('/');+ return safeSuite(path) ? path : 'unavailable';+};+const workerIdentity = value => /^\d{1,6}$/.test(value ?? '') ? String(value) : 'unavailable';+export const processIdentity = ({ pid, start }) => createHash('sha256').update(`${pid}:${start}`).digest('hex');++/** Capture the suite at child construction, never the suite active at cleanup. */+export function observeTestProcesses(getSuite, { root = process.env.PORTOS_TEST_TEMP_ROOT, read = readTestProcesses } = {}) {+ if (!root) return () => {};+ const directory = join(root, PROCESS_OWNERS_DIR);+ const path = join(directory, `${randomUUID()}.json`);+ const children = new Map();+ const worker = workerIdentity(process.env.VITEST_WORKER_ID);+ let phase = 'suite-active';+ let capped = false;+ const persist = () => {+ try {+ if (!children.size && !capped) { rmSync(path, { force: true }); return; }+ mkdirSync(directory, { recursive: true });+ writeFileSync(`${path}.tmp`, JSON.stringify({ capped, children: [...children.values()].map(item => item.row) }));+ renameSync(`${path}.tmp`, path);+ } catch { /* missing evidence is explicitly unavailable at containment */ }+ };+ const listener = ({ process: child }) => {+ if (children.size >= limit) { capped = true; persist(); return; }+ let suite = 'unavailable';+ try { suite = suiteIdentity(getSuite()); } catch { /* diagnostic context is optional */ }+ const row = { identity: null, suite, worker, phase, lifecycle: 'constructed' };+ const onSpawn = () => {+ try {+ const record = read([child.pid])?.find(item => item.pid === child.pid);+ if (record?.start && !record.terminal) row.identity = processIdentity(record);+ } catch { /* attribution remains unavailable, never guess by PID */ }+ row.lifecycle = 'spawned'; persist();+ };+ const onExit = () => { row.lifecycle = 'exit-observed'; persist(); };+ const onClose = () => { children.delete(child); persist(); };+ children.set(child, { row, onSpawn, onExit, onClose });+ child.once('spawn', onSpawn);+ child.once('exit', onExit);+ child.once('close', onClose);+ };+ spawned.subscribe(listener);+ return () => {+ spawned.unsubscribe(listener);+ phase = 'suite-ended';+ for (const [child, item] of children) {+ item.row.phase = phase;+ child.removeListener('spawn', item.onSpawn);+ child.removeListener('exit', item.onExit);+ child.removeListener('close', item.onClose);+ }+ persist();+ };+}++/** All on-disk input is reprojected; no raw process data enters the report. */+export function processAttribution(root, identity) {+ const unavailable = state => ({ suite: 'unavailable', worker: 'unavailable', phase: 'unavailable', lifecycle: 'unavailable', attribution: state });+ try {+ const directory = join(root, PROCESS_OWNERS_DIR);+ const names = readdirSync(directory);+ if (names.length > 256) return unavailable('capped');+ let capped = false;+ let match;+ for (const name of names) {+ if (!/^[\da-f-]{36}\.json$/.test(name)) continue;+ const path = join(directory, name);+ if (statSync(path).size > 48_000) return unavailable('capped');+ const data = JSON.parse(readFileSync(path, 'utf8'));+ if (!Array.isArray(data.children) || data.children.length > limit) return unavailable('unavailable');+ capped ||= data.capped === true;+ for (const item of data.children) {+ if (item.identity !== identity) continue;+ if (match) return unavailable('ambiguous');+ match = { suite: safeSuite(item.suite) ? item.suite : 'unavailable', worker: workerIdentity(item.worker),+ phase: ['suite-active', 'suite-ended'].includes(item.phase) ? item.phase : 'unavailable',+ lifecycle: ['constructed', 'spawned', 'exit-observed'].includes(item.lifecycle) ? item.lifecycle : 'unavailable',+ attribution: 'observed' };+ }+ }+ return match ?? unavailable(capped ? 'capped' : 'unavailable');+ } catch { return unavailable('unavailable'); }+}diff --git a/scripts/lib/vitestStaleRunRoots.js b/scripts/lib/vitestStaleRunRoots.jsindex 77254398e0..ed20220c83 100644--- a/scripts/lib/vitestStaleRunRoots.js+++ b/scripts/lib/vitestStaleRunRoots.js@@ -10,12 +10,18 @@* to the conservative 6h age rule.*/import { execFileSync } from 'node:child_process';-import { lstatSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { existsSync, lstatSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';import { basename, dirname, join } from 'node:path';-import { processAncestors, readTestProcesses, testProcessRoot } from './testOwnedProcesses.js';+import { processAncestors, probeTestPid, readTestProcesses, testProcessRoot, testProcessSettlement } from './testOwnedProcesses.js';+import { processAttribution, processIdentity } from './testProcessAttribution.js';import { killWithEscalation } from '../../server/lib/killWithEscalation.js';export const OWNER_FILE = '.owner.pid';+export const PENDING_FILE = '.process-settlement-pending';++function markPending(root) {+ try { writeFileSync(join(root, PENDING_FILE), 'unverified\n'); } catch { /* preserve in this snapshot even if the stamp fails */ }+}export const STALE_ROOT_AGE_MS = 6 * 60 * 60 * 1000;// `ps -o lstart=` has 1s resolution and the recorded start is derived from// process.uptime(), so allow slack before calling a pid "reused".@@ -123,9 +129,9 @@ function ownedSnapshot(parent, prefix, read) {const ancestors = processAncestors(records);if (!ancestors) return null;const owned = records.flatMap(record => {- if (!Number.isSafeInteger(record.pid) || record.pid <= 1 || typeof record.start !== 'string' || !record.start) return [];+ if (record.terminal || !Number.isSafeInteger(record.pid) || record.pid <= 1 || typeof record.start !== 'string' || !record.start) return [];const root = testProcessRoot(record, parent, prefix);- return root && !ancestors.has(record.pid) ? [{ pid: record.pid, start: record.start, root, kind: processKind(record), parent: ancestors.has(record.ppid) ? 'runner' : 'descendant' }] : [];+ return root && !ancestors.has(record.pid) ? [{ pid: record.pid, start: record.start, root, kind: processKind(record), parent: ancestors.has(record.ppid) ? 'runner' : 'descendant', identity: processIdentity(record) }] : [];});return { ancestors, owned };}@@ -148,17 +154,22 @@ export function terminatePid(record, { valid, kill = (pid, signal) => process.ki{ label: 'Orphaned test process', stillRunning: valid, delayMs: 2000 });}-/** Only individually proven processes are signalled; an unowned group member is never collateral cleanup. */+/** Only individually proven processes are signalled; an unowned group member is never collateral cleanup.+ * Synchronous settlement also serves Node's exit hook, where promises cannot be awaited.+ */export function killTestProcessesUnder(root, { read = readTestProcesses, probe = isOwnerAlive,- kill = (pid, signal) => process.kill(pid, signal), onKilled = () => {} } = {}) {+ kill = (pid, signal) => process.kill(pid, signal), onKilled = () => {}, onOutcome = () => {}, onInspection = () => {},+ probePid = probeTestPid, now = () => performance.now(), wait = ms => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms),+ settlementMs = 2000 } = {}) {let rootExists = true;- try { if (!lstatSync(root).isDirectory()) return 0; } catch (error) {+ try { const stat = lstatSync(root); if (!stat.isDirectory() || stat.isSymbolicLink()) return 0; } catch (error) {if (error.code !== 'ENOENT') return 0;rootExists = false;}const parent = dirname(root);const prefix = basename(root);const snapshot = ownedSnapshot(parent, prefix, read);+ onInspection(snapshot !== null);if (!snapshot) return 0;const owner = readOwner(root);if (rootExists && !owner) return 0;@@ -168,11 +179,39 @@ export function killTestProcessesUnder(root, { read = readTestProcesses, probe =&& Math.abs(startMs - owner.startMs) <= START_TOLERANCE_MS;if (!verifiedOwner && probe(owner) !== false) return 0;}- let killed = 0;+ const pending = [];for (const record of snapshot.owned) {- if (record.root !== root || !stillOwned(record, snapshot, parent, prefix, read)) continue;- try { kill(record.pid, 'SIGKILL'); killed++; } catch { continue; /* already gone */ }- onKilled({ kind: record.kind, parent: record.parent });+ if (record.root !== root) continue;+ const verified = stillOwned(record, snapshot, parent, prefix, read);+ if (!verified && testProcessSettlement(record, read([record.pid]), probePid) === 'terminated') continue;+ // Read the bounded attribution before signaling; it cannot be reconstructed+ // after the suite worker or run root disappears.+ const attribution = processAttribution(root, record.identity);+ let signal = 'not-attempted';+ if (verified) {+ try { kill(record.pid, 'SIGKILL'); signal = 'delivered'; } catch { signal = 'failed'; }+ }+ pending.push({ record, attribution, signal, settlement: 'unknown' });+ }+ const deadline = now() + settlementMs;+ while (pending.some(item => item.settlement !== 'terminated')) {+ const current = read(pending.map(item => item.record.pid));+ for (const item of pending) {+ if (item.settlement !== 'terminated') item.settlement = testProcessSettlement(item.record, current, probePid);+ }+ if (pending.every(item => item.settlement === 'terminated') || now() >= deadline) break;+ wait(Math.min(25, deadline - now()));+ }+ let killed = 0;+ for (const { record, attribution, signal, settlement } of pending) {+ const outcome = { kind: record.kind, parent: record.parent, ...attribution, signal,+ settlement: settlement === 'running' ? 'unconfirmed' : settlement };+ onOutcome(outcome);+ if (settlement !== 'terminated') markPending(root);+ if (settlement === 'terminated') {+ killed++;+ onKilled({ kind: record.kind, parent: record.parent });+ }}return killed;}@@ -180,17 +219,21 @@ export function killTestProcessesUnder(root, { read = readTestProcesses, probe =export function sweepOrphanedTestProcesses(tmpRoot, { prefix = 'pvt-', now = Date.now(), probe = isOwnerAlive,read = readTestProcesses, terminate = terminatePid } = {}) {const snapshot = ownedSnapshot(tmpRoot, prefix, read);- if (!snapshot) return { killed: 0, inspected: false, preserved: new Set() };+ if (!snapshot) return { signaled: 0, inspected: false, preserved: new Set() };const preserved = new Set();- let killed = 0;+ let signaled = 0;for (const record of snapshot.owned) {const valid = () => stillOwned(record, snapshot, tmpRoot, prefix, read)&& rootIsOrphaned(record.root, probe, Date.now());if (!rootIsOrphaned(record.root, probe, now) || !valid()) { preserved.add(record.root); continue; }- try { terminate(record, { valid }); killed++; } catch { /* already gone */ }+ // Escalation is deferred; retain this root until a later snapshot proves+ // its children gone. Signal scheduling is not verified termination.+ preserved.add(record.root);+ markPending(record.root);+ try { terminate(record, { valid }); signaled++; } catch { /* settlement unknown */ }}- if (killed) console.warn(`🧹 Stopped ${killed} test process${killed === 1 ? '' : 'es'} left running by an interrupted test run`);- return { killed, inspected: true, preserved };+ if (signaled) console.warn(`🧹 Requested termination of ${signaled} interrupted-run test processes; settlement pending`);+ return { signaled, inspected: true, preserved };}// Retain the established import boundary while callers adopt the broader name.@@ -213,7 +256,7 @@ export function sweepStaleRunRoots(tmpRoot, { prefix = 'pvt-', now = Date.now(),for (const name of names) {if (!name.startsWith(prefix)) continue;const path = join(tmpRoot, name);- if (preserved.has(path)) continue;+ if (preserved.has(path) || (!inspected && existsSync(join(path, PENDING_FILE)))) continue;try {const stat = lstatSync(path);if (!stat.isDirectory() || stat.isSymbolicLink()) continue;diff --git a/scripts/run-ci-tests.js b/scripts/run-ci-tests.jsindex 35d0aaf2b4..3c85bda154 100644--- a/scripts/run-ci-tests.js+++ b/scripts/run-ci-tests.js@@ -1,7 +1,7 @@#!/usr/bin/env nodeimport { spawn } from 'child_process';-import { mkdirSync, mkdtempSync, readFileSync, statSync, rmSync } from 'node:fs';+import { existsSync, mkdirSync, mkdtempSync, readFileSync, statSync, rmSync } from 'node:fs';import { tmpdir } from 'node:os';import { stripVTControlCharacters } from 'node:util';import { dirname, join } from 'path';@@ -224,9 +224,11 @@ export async function runNpm(scope, script, extraArgs, { env = process.env, signconst reportDir = mkdtempSync(join(hostRoot, 'portos-vitest-'));const reportPath = join(reportDir, 'failures.json');let cleanupUnverified = false;+ let processEvidence = false;try {writeOwnerFile(reportDir);const launched = await runNpmWithReport(scope, script, extraArgs, reportPath, env, signal, cancellationTimeoutMs);+ processEvidence = existsSync(join(reportDir, 'processes.json'));cleanupUnverified = launched.cleanupUnverified === true;if (cleanupUnverified) {console.error(`Test launcher cleanup unverified; diagnostics retained at ${reportDir}`);@@ -235,6 +237,7 @@ export async function runNpm(scope, script, extraArgs, { env = process.env, sign// Per-file outcomes and the run-level verdict feed the test memo; an interrupted run proves nothing.const result = {...launched,+ ...(processEvidence ? { diagnosticsPath: reportDir } : {}),fileResults: launched.interrupted ? [] : fileResultsFromReport(reportPath),runReport: launched.interrupted ? null : readRunReport(join(reportDir, 'run.json')),};@@ -249,7 +252,7 @@ export async function runNpm(scope, script, extraArgs, { env = process.env, sign}return result;} finally {- if (!durable && !cleanupUnverified) rmSync(reportDir, { recursive: true, force: true });+ if (!durable && !cleanupUnverified && !processEvidence) rmSync(reportDir, { recursive: true, force: true });}}@@ -277,7 +280,7 @@ function runNpmWithReport(scope, script, extraArgs, reportPath, env, abortSignal// the inherited group, so it can retire npm and Vitest even after this// launcher's event loop stopped or the launcher itself was killed.const processGroup = process.platform !== 'win32' && env.PORTOS_CI_TEST_SUPERVISED_GROUP !== '1';- const childEnv = { ...env, PORTOS_CI_DIAGNOSTICS_PATH: runReportPath };+ const childEnv = { ...env, PORTOS_CI_DIAGNOSTICS_PATH: runReportPath, PORTOS_TEST_PROCESS_DIAGNOSTICS_PATH: join(dirname(reportPath), 'processes.json') };// Only the workflow launcher owns durable attempts. Nested launcher// fixtures must not overwrite or add to the job's failure provenance.delete childEnv.PORTOS_CI_DIAGNOSTICS_DIR;diff --git a/scripts/run-ci-tests.test.js b/scripts/run-ci-tests.test.jsindex 5c3ced1de4..c70264998f 100644--- a/scripts/run-ci-tests.test.js+++ b/scripts/run-ci-tests.test.js@@ -800,6 +800,28 @@ describe('structured failure diagnostics', () => {}, 35_000);});+// A containment report written after the ordinary reporter snapshot must survive+// the launcher's nondurable-attempt cleanup, including otherwise green runs.+it('retains process settlement evidence from a completed launcher attempt', async () => {+ const fixture = mkdtempSync(join(tmpdir(), 'process-report-launcher-'));+ let diagnosticsPath;+ try {+ installFakeNpm(fixture, `require('node:fs').writeFileSync(process.env.PORTOS_TEST_PROCESS_DIAGNOSTICS_PATH,+ JSON.stringify({ version: 1, total: 1, terminated: 1, unsettled: 0, processes: [] }));`);+ const { runNpm } = await import('./run-ci-tests.js');+ const result = await runNpm('server', 'test:ci', [], {+ env: { ...process.env, PORTOS_CI_DIAGNOSTICS_DIR: '', PATH: `${fixture}${delimiter}${process.env.PATH}` },+ });+ diagnosticsPath = result.diagnosticsPath;+ expect(result.status).toBe(0);+ expect(diagnosticsPath).toBeTruthy();+ expect(JSON.parse(readFileSync(join(diagnosticsPath, 'processes.json'), 'utf8'))).toMatchObject({ total: 1, terminated: 1 });+ } finally {+ if (diagnosticsPath) rmSync(diagnosticsPath, { recursive: true, force: true });+ rmSync(fixture, { recursive: true, force: true });+ }+});+describe('report launcher subprocess lifecycle (#9804)', () => {const runnerUrl = new URL('./run-ci-tests.js', import.meta.url).href;diff --git a/scripts/testOwnedProcessRecovery.test.js b/scripts/testOwnedProcessRecovery.test.jsindex 6feffa64e6..8304ee91d0 100644--- a/scripts/testOwnedProcessRecovery.test.js+++ b/scripts/testOwnedProcessRecovery.test.js@@ -5,6 +5,7 @@ import { tmpdir } from 'node:os';import { join } from 'node:path';import { fileURLToPath } from 'node:url';import { STALE_ROOT_AGE_MS, killTestProcessesUnder, sweepStaleRunRoots, writeOwnerFile } from './lib/vitestStaleRunRoots.js';+import { observeTestProcesses } from './lib/testProcessAttribution.js';import { processAncestors, readTestProcesses, testProcessRoot } from './lib/testOwnedProcesses.js';import { isProcessAlive } from '../server/test/processAlive.js';@@ -43,8 +44,8 @@ describe('test process ownership boundaries', () => {it.skipIf(process.platform === 'win32')('retains only legacy browser ownership when the optional Darwin reader is unavailable', () => {const exec = vi.fn((command) => {if (command !== 'ps') throw Error('reader unavailable');- return `900001 1 ${process.getuid()} Fri Oct 16 12:00:00 2026 chrome --user-data-dir=/tmp/example/pvt-browser/profile\n`- + `900002 1 ${process.getuid()} Fri Oct 16 12:00:00 2026 node /tmp/example/pvt-script/file.js PORTOS_TEST_TEMP_ROOT=/tmp/example/pvt-env NODE_ENV=test\n`;+ return `900001 1 ${process.getuid()} S Fri Oct 16 12:00:00 2026 chrome --user-data-dir=/tmp/example/pvt-browser/profile\n`+ + `900002 1 ${process.getuid()} S Fri Oct 16 12:00:00 2026 node /tmp/example/pvt-script/file.js PORTOS_TEST_TEMP_ROOT=/tmp/example/pvt-env NODE_ENV=test\n`;});const records = readTestProcesses(null, { platform: 'darwin', exec });expect(records.map(record => testProcessRoot(record, '/tmp/example', 'pvt-'))).toEqual(['/tmp/example/pvt-browser', null]);@@ -66,7 +67,7 @@ describe('test process ownership boundaries', () => {expect(testProcessRoot({ args: ['chrome', `--user-data-dir=${root}/profile`] }, parent, 'pvt-')).toBe(root);});- it('reports only fixed process categories for successfully stopped owned children', () => {+ it('waits for terminal birth identity after signaling and distinguishes unconfirmed or unknown settlement', () => {const host = mkdtempSync(join(tmpdir(), 'recovery-diagnostics-'));const root = join(host, 'pvt-owned');mkdirSync(root);@@ -75,16 +76,26 @@ describe('test process ownership boundaries', () => {const child = { pid: 900001, ppid: process.pid, start: 'original',args: ['/example/node', '/example/vitest/dist/workers/forks.js', '--secret=fixture-secret'],env: { NODE_ENV: 'test', PORTOS_TEST_TEMP_ROOT: root } };- const onKilled = vi.fn();- const kill = vi.fn();try {- expect(killTestProcessesUnder(root, { read: () => [...ancestors, child], probe: () => false, kill, onKilled })).toBe(1);- expect(kill).toHaveBeenCalledExactlyOnceWith(child.pid, 'SIGKILL');- expect(onKilled).toHaveBeenCalledExactlyOnceWith({ kind: 'vitest', parent: 'runner' });- kill.mockImplementation(() => { throw Error('already gone'); });- onKilled.mockClear();- expect(killTestProcessesUnder(root, { read: () => [...ancestors, child], probe: () => false, kill, onKilled })).toBe(0);- expect(onKilled).not.toHaveBeenCalled();+ for (const mode of ['terminal', 'reused', 'unconfirmed', 'unknown', 'signal-failed']) {+ let time = 0;+ let signaled = false;+ const onKilled = vi.fn();+ const onOutcome = vi.fn();+ const read = () => !signaled || time === 0 ? [...ancestors, child]+ : mode === 'unknown' ? null+ : [...ancestors, { ...child, terminal: mode === 'terminal', start: mode === 'reused' ? 'replacement' : child.start }];+ const kill = vi.fn(() => { signaled = true; if (mode === 'signal-failed') throw Error('denied'); });+ const result = killTestProcessesUnder(root, { read, probe: () => false, probePid: () => true, kill, onKilled, onOutcome,+ now: () => time, wait: ms => { expect(onKilled).not.toHaveBeenCalled(); time += ms; }, settlementMs: 50 });+ const terminated = ['terminal', 'reused'].includes(mode);+ expect(result).toBe(terminated ? 1 : 0);+ expect(kill).toHaveBeenCalledExactlyOnceWith(child.pid, 'SIGKILL');+ expect(onOutcome).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ kind: 'vitest', parent: 'runner',+ signal: mode === 'signal-failed' ? 'failed' : 'delivered',+ settlement: terminated ? 'terminated' : mode === 'unknown' ? 'unknown' : 'unconfirmed' }));+ expect(JSON.stringify(onOutcome.mock.calls)).not.toMatch(/fixture-secret|900001|original|example/);+ }} finally { rmSync(host, { recursive: true, force: true }); }});@@ -102,14 +113,16 @@ describe('test process ownership boundaries', () => {try {sweepStaleRunRoots(host, { read, probe: () => ownerAlive });expect(kill).toHaveBeenCalledExactlyOnceWith(target.pid, 'SIGTERM');+ sweepStaleRunRoots(host, { read: () => null, probe: () => false });+ expect(existsSync(root)).toBe(true); // unknown settlement cannot erase a prior pending stamptarget.start = 'reused';vi.advanceTimersByTime(2000);expect(kill).toHaveBeenCalledTimes(1);- mkdirSync(root); writeFileSync(join(root, '.owner.pid'), '999999 0\n');+ expect(existsSync(root)).toBe(true); writeFileSync(join(root, '.owner.pid'), '999999 0\n');target.start = 'next';sweepStaleRunRoots(host, { read, probe: () => ownerAlive });expect(kill).toHaveBeenCalledTimes(2);- mkdirSync(root); writeOwnerFile(root); ownerAlive = true;+ expect(existsSync(root)).toBe(true); writeOwnerFile(root); ownerAlive = true;vi.advanceTimersByTime(2000);expect(kill).toHaveBeenCalledTimes(2);expect(killTestProcessesUnder(root, { read: () => ancestors, probe: () => true, kill })).toBe(0);@@ -155,7 +168,10 @@ describe.skipIf(process.platform === 'win32')('real interrupted-run recovery', (const script = join(dead, 'fixture.sh'); writeFileSync(script, 'echo ready\nread answer\n');const argumentOnly = await launch(['/bin/sh', script]);const unrelated = await launch([process.execPath, '-e', code], { OTHER: `PORTOS_TEST_TEMP_ROOT=${dead} NODE_ENV=test` });- const ownedLive = await launch([process.execPath, '-e', code], { NODE_ENV: 'test', PORTOS_TEST_TEMP_ROOT: live });+ const dispose = observeTestProcesses(() => fileURLToPath(import.meta.url), { root: live });+ let ownedLive;+ try { ownedLive = await launch([process.execPath, '-e', code], { NODE_ENV: 'test', PORTOS_TEST_TEMP_ROOT: live }); }+ finally { dispose(); }const inspected = readTestProcesses([environmentOnly.pid, argumentOnly.pid, unrelated.pid]);expect(inspected?.map(record => testProcessRoot(record, host, 'pvt-'))).toEqual([dead, dead, null]);sweepStaleRunRoots(host);@@ -165,7 +181,9 @@ describe.skipIf(process.platform === 'win32')('real interrupted-run recovery', (}, { timeout: 5000 });expect(isProcessAlive(unrelated.pid)).toBe(true);expect(isProcessAlive(ownedLive.pid)).toBe(true);- expect(killTestProcessesUnder(live)).toBe(1);+ const outcomes = [];+ expect(killTestProcessesUnder(live, { onOutcome: outcome => outcomes.push(outcome) })).toBe(1);+ expect(outcomes).toEqual([expect.objectContaining({ suite: 'scripts/testOwnedProcessRecovery.test.js', phase: 'suite-ended', lifecycle: 'spawned', settlement: 'terminated' })]);await vi.waitFor(() => expect(isProcessAlive(ownedLive.pid)).toBe(false));} finally {await Promise.all(children.map(child => new Promise(resolve => {diff --git a/scripts/testProcessAttribution.test.js b/scripts/testProcessAttribution.test.jsnew file mode 100644index 0000000000..62e34e458d--- /dev/null+++ b/scripts/testProcessAttribution.test.js@@ -0,0 +1,63 @@+import { channel } from 'node:diagnostics_channel';+import { EventEmitter } from 'node:events';+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { fileURLToPath } from 'node:url';+import { afterEach, expect, it, vi } from 'vitest';+import { observeTestProcesses, processAttribution, processIdentity } from './lib/testProcessAttribution.js';+import { killTestProcessesUnder } from './lib/vitestStaleRunRoots.js';+import { cleanupRunTempRoot } from './vitestTempRootSetup.js';++const originalExit = process.exitCode;+afterEach(() => { process.exitCode = originalExit; vi.restoreAllMocks(); });++// Regression: a suite-ended child must be attributed before the root disappears,+// while a successful signal cannot remove that root until terminal identity exists.+it.each(['terminated', 'unconfirmed', 'unknown'])('retains sanitized spawn ownership and %s settlement through run cleanup', settlement => {+ const host = mkdtempSync(join(tmpdir(), 'process-evidence-'));+ const root = mkdtempSync(join(host, 'pvt-'));+ const reportPath = join(host, 'processes.json');+ const records = [{ pid: process.pid, ppid: process.ppid }, { pid: process.ppid, ppid: 1 }];+ const record = { pid: 900001, ppid: process.pid, start: 'birth-private-sentinel', args: ['/example/node', 'private-argument'],+ env: { NODE_ENV: 'test', PORTOS_TEST_TEMP_ROOT: root } };+ writeFileSync(join(root, '.owner.pid'), '999999 0\n');+ let suite = fileURLToPath(import.meta.url);+ const dispose = observeTestProcesses(() => suite, { root, read: () => [record] });+ const child = Object.assign(new EventEmitter(), { pid: record.pid });+ vi.spyOn(console, 'warn').mockImplementation(() => {});+ vi.spyOn(console, 'error').mockImplementation(() => {});+ try {+ channel('child_process').publish({ process: child });+ child.emit('spawn');+ suite = '/private/not-the-spawning-suite.test.js';+ dispose();+ expect(processAttribution(root, processIdentity(record))).toMatchObject({ suite: 'scripts/testProcessAttribution.test.js', phase: 'suite-ended', lifecycle: 'spawned' });+ expect(processAttribution(root, processIdentity({ ...record, start: 'reused' })).attribution).toBe('unavailable');+ let time = 0;+ const contain = (path, callbacks) => killTestProcessesUnder(path, { ...callbacks,+ read: () => time > 0 && settlement === 'unknown' ? null : [...records, { ...record, terminal: time > 0 && settlement === 'terminated' }],+ probe: () => false, probePid: () => true, kill: () => {}, now: () => time, wait: ms => { time += ms; }, settlementMs: 50 });+ cleanupRunTempRoot(root, { contain, reportPath, platform: 'linux' });+ const report = JSON.parse(readFileSync(reportPath, 'utf8'));+ expect(report.processes).toEqual([expect.objectContaining({ suite: 'scripts/testProcessAttribution.test.js', phase: 'suite-ended', lifecycle: 'spawned', signal: 'delivered', settlement })]);+ expect(existsSync(root)).toBe(settlement !== 'terminated');+ if (settlement !== 'terminated') expect(process.exitCode).toBe(1);+ expect(JSON.stringify(report)).not.toMatch(/private|900001|999999|birth-|argument|\/tmp\/|\/Users\//);+ } finally { dispose(); rmSync(host, { recursive: true, force: true }); }+});++it('retains the root and an explicit diagnostic gap when process inspection fails', () => {+ const host = mkdtempSync(join(tmpdir(), 'process-inspection-'));+ const root = mkdtempSync(join(host, 'pvt-'));+ const reportPath = join(host, 'processes.json');+ vi.spyOn(console, 'warn').mockImplementation(() => {});+ vi.spyOn(console, 'error').mockImplementation(() => {});+ try {+ cleanupRunTempRoot(root, { reportPath, platform: 'linux',+ contain: (path, callbacks) => killTestProcessesUnder(path, { ...callbacks, read: () => null }) });+ expect(JSON.parse(readFileSync(reportPath, 'utf8'))).toMatchObject({ inspected: false, total: 0, terminated: 0, unsettled: 1 });+ expect(process.exitCode).toBe(1);+ expect(existsSync(root)).toBe(true);+ } finally { rmSync(host, { recursive: true, force: true }); }+});diff --git a/scripts/vitestProcessEnvironment.js b/scripts/vitestProcessEnvironment.jsnew file mode 100644index 0000000000..cd5031b0d9--- /dev/null+++ b/scripts/vitestProcessEnvironment.js@@ -0,0 +1,20 @@+import { builtinEnvironments } from 'vitest/runtime';+import { observeTestProcesses } from './lib/testProcessAttribution.js';++// Keep the observer in Node's environment realm: client transforms replace+// import.meta.url with HTTP URLs. Environment teardown also runs for skipped+// suites and failed collection, unlike an afterAll observer cleanup.+export default {+ ...builtinEnvironments['happy-dom'],+ name: 'portos-observed-happy-dom',+ async setup(global, options) {+ const environment = await builtinEnvironments['happy-dom'].setup(global, options);+ const dispose = observeTestProcesses(() => global.__vitest_worker__?.filepath);+ return {+ async teardown() {+ try { dispose(); }+ finally { await environment.teardown(global); }+ },+ };+ },+};diff --git a/scripts/vitestTempRoot.test.js b/scripts/vitestTempRoot.test.jsindex 9b7aa4e9b9..d121461180 100644--- a/scripts/vitestTempRoot.test.js+++ b/scripts/vitestTempRoot.test.js@@ -168,6 +168,9 @@ describe.skipIf(process.platform === 'win32')('orphaned test browser sweep', ()sweepStaleRunRoots(host);expect(await gone(orphan.pid)).toBe(true);+ // The first sweep retains ownership while its asynchronous escalation settles.+ expect(existsSync(dead)).toBe(true);+ sweepStaleRunRoots(host);expect(existsSync(dead)).toBe(false);expect(isProcessAlive(owned.pid)).toBe(true);diff --git a/scripts/vitestTempRootSetup.js b/scripts/vitestTempRootSetup.jsindex 24d2165667..4c067664ec 100644--- a/scripts/vitestTempRootSetup.js+++ b/scripts/vitestTempRootSetup.js@@ -3,16 +3,19 @@* redirects temp APIs before Vitest creates its module-transform cache.* The server's exclusive capture projects share this single global owner.*/-import { lstatSync, readdirSync, rmSync } from 'node:fs';-import { join } from 'node:path';+import { randomUUID } from 'node:crypto';+import { fileURLToPath } from 'node:url';+import { PROCESS_OWNERS_DIR } from './lib/testProcessAttribution.js';+import { lstatSync, mkdirSync, readdirSync, rmSync, writeFileSync } from 'node:fs';+import { dirname, join } from 'node:path';import { describeOwnedBrowserObservation, describeStageFixtureRoot, readOwnedBrowserObservations } from '../server/test/ownedBrowserDiagnostics.js';-import { OWNER_FILE, killTestProcessesUnder } from './lib/vitestStaleRunRoots.js';+import { OWNER_FILE, PENDING_FILE, killTestProcessesUnder } from './lib/vitestStaleRunRoots.js';export function setup({ vitest }) {const root = process.env.PORTOS_TEST_TEMP_ROOT;// Vitest runs global teardown before closing its worker pool. Its close hook// runs after pool.close(), so live runner workers are not mistaken for leaks.- vitest.onClose(() => cleanupRunTempRoot(root));+ vitest.onClose(() => cleanupRunTempRoot(root, { reportPath: process.env.PORTOS_TEST_PROCESS_DIAGNOSTICS_PATH, durable: true }));}// Vitest's nanoid transform scratch still exists when global teardown runs.@@ -91,18 +94,44 @@ export function isEffectivelyEmpty(path) {return children.every((child) => isEffectivelyEmpty(join(path, child)));}-export function cleanupRunTempRoot(root = process.env.PORTOS_TEST_TEMP_ROOT) {+export function cleanupRunTempRoot(root = process.env.PORTOS_TEST_TEMP_ROOT, { contain = killTestProcessesUnder, reportPath, durable = false, platform = process.platform } = {}) {if (!root) return;// A process a suite failed to tear down would outlive the run (#10840).- const kinds = new Map();- const processes = process.platform === 'win32' ? 0 : killTestProcessesUnder(root, {- onKilled: ({ kind, parent }) => {- const key = `kind=${kind} parent=${parent}`;- kinds.set(key, (kinds.get(key) ?? 0) + 1);+ const outcomes = [];+ let total = 0;+ let unsettled = 0;+ let inspected = true;+ const processes = platform === 'win32' ? 0 : contain(root, {+ onInspection: value => { inspected = value; if (!value) unsettled++; },+ onOutcome: outcome => {+ total++;+ if (outcome.settlement !== 'terminated') unsettled++;+ if (outcomes.length < 64) outcomes.push(outcome);},});- if (processes) console.warn(`⚠️ test process leak: stopped ${processes} process${processes === 1 ? '' : 'es'} still running under the run temp root`);- for (const [kind, count] of kinds) console.warn(`⚠️ stopped owned test processes: ${kind} count=${count}`);+ if (total || !inspected) {+ const report = { version: 1, inspected, boundary: 'pool-closed', total, terminated: processes, unsettled, capped: total > outcomes.length, processes: outcomes };+ const retainedPath = join(fileURLToPath(new URL('../.cache/test-process-diagnostics/', import.meta.url)), `${randomUUID()}.json`);+ const destinations = reportPath ? [reportPath, ...(durable ? [retainedPath] : [])] : [retainedPath];+ let retained = true;+ try {+ for (const destination of destinations) {+ mkdirSync(dirname(destination), { recursive: true });+ writeFileSync(destination, JSON.stringify(report));+ }+ } catch {+ retained = false;+ console.error('❌ owned test process evidence could not be persisted');+ unsettled++;+ }+ console.warn(`⚠️ test process leak: signal/settlement evidence=${retained ? 'retained' : 'unavailable'} inspected=${inspected}; found=${total} terminated=${processes} unsettled=${unsettled}`);+ for (const outcome of outcomes) console.warn(`⚠️ owned test process: ${Object.entries(outcome).map(([key, value]) => `${key}=${value}`).join(' ')}`);+ }+ if (unsettled) {+ process.exitCode = 1;+ console.error('❌ owned test process settlement unverified; retaining run temp root');+ return;+ }let entries;try {entries = readdirSync(root);@@ -110,7 +139,7 @@ export function cleanupRunTempRoot(root = process.env.PORTOS_TEST_TEMP_ROOT) {return;}entries = entries- .filter((name) => name !== OWNER_FILE && name !== OWNERS_FILE)+ .filter((name) => name !== OWNER_FILE && name !== PENDING_FILE && name !== OWNERS_FILE && name !== PROCESS_OWNERS_DIR).filter((name) => !VITEST_INTERNAL_SCRATCH_DIR.test(name)).filter((name) => !TOOLCHAIN_CACHE_PREFIXES.has(groupLeakPrefix(name))).filter((name) => !isDotnetRegistrationCache(root, name))diff --git a/server/test/mediaRegistryEnvironment.js b/server/test/mediaRegistryEnvironment.jsindex a87bb8ee1c..cfa4d3c3f8 100644--- a/server/test/mediaRegistryEnvironment.js+++ b/server/test/mediaRegistryEnvironment.js@@ -1,3 +1,4 @@+import { observeTestProcesses } from '../../scripts/lib/testProcessAttribution.js';import { observeOwnedBrowsers } from './ownedBrowserDiagnostics.js';import { builtinEnvironments } from 'vitest/runtime';import { mkdtempSync, rmSync } from 'node:fs';@@ -13,6 +14,7 @@ export default {viteEnvironment: 'ssr',async setup(global, options) {const node = await builtinEnvironments.node.setup(global, options);+ const disposeProcesses = observeTestProcesses(() => global.__vitest_worker__?.filepath);const disposeObservations = observeOwnedBrowsers(() => global.__vitest_worker__?.filepath);try {const prior = process.env.PORTOS_MEDIA_MODELS_FILE;@@ -22,6 +24,7 @@ export default {async teardown() {try {disposeObservations();+ disposeProcesses();if (root) {if (prior === undefined) delete process.env.PORTOS_MEDIA_MODELS_FILE;else process.env.PORTOS_MEDIA_MODELS_FILE = prior;@@ -36,6 +39,7 @@ export default {// setup can fail before Vitest receives our teardown owner. Preserve that// original error while ending any partial diagnostic subscription.disposeObservations();+ disposeProcesses();throw error;}},