Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix([issue-12536]): preserve late test process exit provenance

[._.]/ Adam Eivy581b92299205
diff --git a/docs/GITHUB_ACTIONS.md b/docs/GITHUB_ACTIONS.md
index da444d2cb0..b475d2f667 100644
--- a/docs/GITHUB_ACTIONS.md
+++ b/docs/GITHUB_ACTIONS.md
@@ -1064,6 +1064,24 @@ test and a minimal Windows reproduction before changing production metadata
handling (#8152). Microsoft documents these fields in its
[application crash troubleshooting guide](https://learn.microsoft.com/en-us/troubleshoot/windows-server/performance/troubleshoot-application-service-crashing-behavior).
+### Final test process exit evidence
+
+The run report retains the exit-code snapshot from Vitest's normal hooks and
+adds the later Node `exit` event code and `process.exitCode` in a synchronous,
+best-effort write. The invocation report records the observed command boundary
+(`direct npm process`, or `cmd.exe -> npm.cmd` when a batch wrapper is needed), its child `exit` and `close`
+events, and the launcher's effective status. Collection prints only bounded
+numeric status and allowlisted boundary names. Missing exit evidence stays an
+explicit diagnostic gap; diagnostic I/O never changes the original result.
+
+An exit event is later evidence, not proof of a cause or of every subsequent
+exit listener's behavior. When a Windows batch wrapper is needed, npm's own process exit is still not
+observed independently of `cmd.exe`; a passing Vitest exit with a failing
+command boundary narrows the remaining investigation to that boundary. Forced
+termination can prevent any exit-event write. Preserve the first failing job
+and compare these observations on a hosted recurrence rather than classifying
+a green rerun or a synthetic fixture as a causal fix (#12536).
+
### External cancellation and one automatic retry
Not every cancel comes from this repository. When several PRs build at once,
diff --git a/scripts/ci-test-diagnostics.js b/scripts/ci-test-diagnostics.js
index 2aac87ca30..025469b6db 100644
--- a/scripts/ci-test-diagnostics.js
+++ b/scripts/ci-test-diagnostics.js
@@ -27,6 +27,16 @@ export function persistTestDiagnostic(path, value) {
}
}
+// No raw process identity or command arguments enter this projection.
+export function processExitDiagnostic(run) {
+ const final = run.processExit;
+ if (!Number.isInteger(final?.eventCode)
+ || (final.exitCode !== null && !Number.isInteger(final.exitCode))) {
+ return 'Diagnostic gap: Vitest process exit event unavailable; final process status is unverified.';
+ }
+ return `Vitest process exit event: code=${final.eventCode}; exitCode=${final.exitCode}.`;
+}
+
function readReport(path, limit = 100_000) {
if (statSync(path).size > limit) throw new Error('report exceeds limit');
return JSON.parse(readFileSync(path, 'utf8'));
@@ -62,6 +72,13 @@ export async function collectTestDiagnostics(root, {
lines.push(`Launcher ${invocation.completed ? 'completed' : 'interrupted or completion not recorded'}; `
+ `exit=${Number.isInteger(invocation.status) ? invocation.status : 'unavailable'}; `
+ `signal=${/^SIG[A-Z]+$/.test(invocation.signal) ? invocation.signal : 'none'}.`);
+ const boundary = ['direct npm process', 'cmd.exe -> npm.cmd'].includes(invocation.commandBoundary) ? invocation.commandBoundary : 'unavailable';
+ lines.push(`Observed command boundary: ${boundary}.`);
+ for (const [event, value] of [['exit', invocation.childExit], ['close', invocation.childClose]]) {
+ lines.push(`Command ${event}: code=${Number.isInteger(value?.code) ? value.code : 'unavailable'}; `
+ + `signal=${/^SIG[A-Z]+$/.test(value?.signal) ? value.signal : 'none'}.`);
+ }
+ if (boundary === 'cmd.exe -> npm.cmd') lines.push('Diagnostic gap: npm process exit is not observed independently of the Windows command wrapper.');
if (Array.isArray(invocation.selectors)) lines.push(`Selected inputs (up to 100): ${invocation.selectors.slice(0, 100).map(diagnosticTestFile).join(', ') || 'full suite or no file selectors'}`);
} catch {
lines.push('Diagnostic gap: invocation metadata unavailable; launcher completion is unverified.');
@@ -71,6 +88,7 @@ export async function collectTestDiagnostics(root, {
if (!Array.isArray(run.activeModules) || !Array.isArray(run.completedModules)
|| !Number.isInteger(run.completedCount) || !Number.isInteger(run.errorCount)) throw Error('invalid progress');
const reason = ['passed', 'failed', 'interrupted', 'not-finished'].includes(run.reason) ? run.reason : 'unavailable';
+ lines.push(processExitDiagnostic(run));
lines.push(`Vitest progress: reason=${reason}; completed=${run.completedCount}; unhandledErrors=${run.errorCount}.`);
lines.push(`Queued/unfinished modules (up to 100): ${run.activeModules.slice(0, 100).map(diagnosticTestFile).join(', ') || 'none recorded'}`);
lines.push(`Last completed modules (up to 100): ${run.completedModules.slice(-100).map(diagnosticTestFile).join(', ') || 'none recorded'}`);
diff --git a/scripts/ci-test-diagnostics.test.js b/scripts/ci-test-diagnostics.test.js
index 7436c30791..0ea90fb9e2 100644
--- a/scripts/ci-test-diagnostics.test.js
+++ b/scripts/ci-test-diagnostics.test.js
@@ -79,6 +79,8 @@ describe('independent CI test failure recovery (#11552)', () => {
const output = await launchAndCollect(false);
expect(output).toContain('Launcher completed; exit=17');
expect(output).toContain('reason=failed; completed=1');
+ expect(output).toContain('Vitest process exit event: code=17; exitCode=17.');
+ expect(output).toContain('Command exit: code=17; signal=none.');
expect(output).toContain('Failed suites (up to 10): server/lib/synthetic.test.js');
});
@@ -89,9 +91,24 @@ describe('independent CI test failure recovery (#11552)', () => {
expect(output).toContain('structured test report missing');
});
+ it.each([0, 17])('diagnostic write failure preserves process exit %i', (code) => {
+ const result = spawnSync(process.execPath, ['--input-type=module', '-e', `
+ import reporter from ${JSON.stringify(reporter)};
+ const hooks = reporter();
+ hooks.onInit({ onClose(callback) { callback(); } });
+ hooks.onTestRunEnd([], [], 'passed');
+ process.exitCode = ${code};
+ `], { encoding: 'utf8', timeout: 15_000,
+ env: { ...process.env, PORTOS_CI_DIAGNOSTICS_PATH: new URL(import.meta.url).pathname + '/missing/run.json' } });
+ expect(result.error).toBeUndefined();
+ expect(result.status).toBe(code);
+ expect(result.stderr).toContain('Unable to persist CI test diagnostics.');
+ });
+
it('recovers bounded completed outcomes without final JSON or private error details', async () => {
const root = mkdtempSync(join(tmpdir(), 'ci-diagnostics-outcomes-'));
const prior = process.env.PORTOS_CI_DIAGNOSTICS_PATH;
+ const priorExitListeners = new Set(process.listeners('exit'));
try {
const attempt = join(root, 'portos-vitest-fixture'); mkdirSync(attempt);
const path = join(attempt, 'run.json');
@@ -108,8 +125,10 @@ describe('independent CI test failure recovery (#11552)', () => {
const lines = await collectTestDiagnostics(root, { logger: { warn() {} }, diagnose: async () => {}, writeSummary() {} });
expect(lines.join('\n')).toContain('server/lib/synthetic.test.js=failed');
expect(lines.join('\n')).toContain('structured test report missing');
+ expect(lines.join('\n')).toContain('Vitest process exit event unavailable');
expect(lines.join('\n')).not.toContain('private-sentinel');
} finally {
+ for (const listener of process.listeners('exit')) if (!priorExitListeners.has(listener)) process.removeListener('exit', listener);
if (prior === undefined) delete process.env.PORTOS_CI_DIAGNOSTICS_PATH;
else process.env.PORTOS_CI_DIAGNOSTICS_PATH = prior;
rmSync(root, { recursive: true, force: true });
diff --git a/scripts/run-ci-tests.js b/scripts/run-ci-tests.js
index 2fa7cf29ca..c13f2045c5 100644
--- a/scripts/run-ci-tests.js
+++ b/scripts/run-ci-tests.js
@@ -11,7 +11,7 @@ import { isDirectlyInvoked } from './lib/directInvocation.js';
import { sweepStaleRunRoots, writeOwnerFile } from './lib/vitestStaleRunRoots.js';
import { formatErrorAnnotation, writeStepSummary } from './lib/githubOutput.js';
import { reportWindowsCrashDiagnostics } from './ci-windows-crash-diagnostics.js';
-import { diagnosticTestFile, persistTestDiagnostic } from './ci-test-diagnostics.js';
+import { diagnosticTestFile, persistTestDiagnostic, processExitDiagnostic } from './ci-test-diagnostics.js';
import { browserTestProfileArgs, browserTestProfileFiles, assertBrowserProfileExecuted, BROWSER_FIDELITY_ONLY_FILES } from './lib/browserTestProfiles.js';
import {
clearVitestDependencies, createToolVersionResolver, currentCommit, fileResultsFromReport, findVitestDependenciesFile,
@@ -162,6 +162,7 @@ export function structuredFailureDiagnostics(reportPath, runReportPath) {
throw new Error('invalid run report schema');
}
lines.push(`Vitest run: reason=${run.reason}, exitCode=${run.exitCode}, unhandledErrors=${run.errorCount}`);
+ lines.push(processExitDiagnostic(run));
if (run.teardownTimeout) lines.push('Vitest teardown timed out.');
if (run.errors.length) lines.push(`Vitest unhandled errors (up to 10):\n${run.errors.slice(0, 10).map(error => error.slice(0, 8000)).join('\n')}`);
} catch {
@@ -259,7 +260,6 @@ function runNpmWithReport(scope, script, extraArgs, reportPath, env, abortSignal
selectors: extraArgs.filter(arg => !arg.startsWith('-')).slice(0, 100)
.map(arg => diagnosticTestFile(arg.startsWith('../') ? arg.slice(3) : `${scope}/${arg.replace(/^\.\//, '')}`)),
};
- persistTestDiagnostic(invocationPath, invocation);
const args = ['run', script, '--prefix', scope];
args.push('--', ...extraArgs, '--reporter=json', `--outputFile.json=${reportPath}`,
`--reporter=${join(repoRoot, 'scripts', 'vitestRunDiagnostics.js')}`);
@@ -267,6 +267,8 @@ function runNpmWithReport(scope, script, extraArgs, reportPath, env, abortSignal
// `shell:false` and throws EINVAL, so this script could never run on a
// Windows checkout. See server/lib/bufferedSpawn.js.
const { command, args: spawnArgs } = prepareCliSpawn('npm', args, env);
+ invocation.commandBoundary = command === 'cmd.exe' ? 'cmd.exe -> npm.cmd' : 'direct npm process';
+ persistTestDiagnostic(invocationPath, invocation);
return new Promise((resolve) => {
// npm is a launcher, not the entire test tree. Give this invocation its
// own POSIX group; Windows killProcessTree uses taskkill /T instead. The
@@ -319,6 +321,8 @@ function runNpmWithReport(scope, script, extraArgs, reportPath, env, abortSignal
child.stderr.on('data', (chunk) => tee(process.stderr, chunk));
// A spawn error also emits close. Settle only there, after the pipes close.
child.on('error', (error) => { spawnError = error; });
+ let childExit = null;
+ child.on('exit', (code, signal) => { childExit = { code, signal }; });
const finish = (code, signal) => {
if (settled) return;
settled = true;
@@ -333,7 +337,8 @@ function runNpmWithReport(scope, script, extraArgs, reportPath, env, abortSignal
process.removeListener('SIGTERM', onSigterm);
abortSignal?.removeEventListener('abort', onAbort);
const status = interrupted ? (interrupted === 'SIGINT' ? 130 : 143) : code ?? 1;
- persistTestDiagnostic(invocationPath, { ...invocation, completed: !cleanupUnverified, status, signal: signal ?? interrupted ?? null });
+ persistTestDiagnostic(invocationPath, { ...invocation, completed: !cleanupUnverified, status, signal: signal ?? interrupted ?? null,
+ childExit, childClose: { code, signal } });
if (status !== 0) {
console.error(`Test launcher exited: code=${code}, signal=${signal ?? 'none'}${interrupted ? `, interrupted=${interrupted}` : ''}`);
console.error(structuredFailureDiagnostics(reportPath, runReportPath));
diff --git a/scripts/run-ci-tests.test.js b/scripts/run-ci-tests.test.js
index 37825110ee..10c843012c 100644
--- a/scripts/run-ci-tests.test.js
+++ b/scripts/run-ci-tests.test.js
@@ -552,6 +552,47 @@ describe('native crash diagnostics before retry/cancellation (#8152)', () => {
describe('structured failure diagnostics', () => {
+ // This uniquely pins shutdown after the ordinary reporter snapshot through
+ // real Vitest assertions, npm, and the launcher's durable collection boundary.
+ it('retains passing assertions and late process exit provenance without changing status (#12536)', async () => {
+ const fixture = mkdtempSync(join(tmpdir(), 'portos-final-exit-'));
+ try {
+ writeFileSync(join(fixture, 'fixture.test.js'), "test('passing control', () => expect(true).toBe(true));");
+ const lateReporter = join(fixture, 'late-reporter.mjs');
+ writeFileSync(lateReporter, `export default function () { return {
+ onInit(vitest) { vitest.onClose(() => { process.once('beforeExit', () => { process.exitCode = Number(process.env.FIXTURE_FINAL_CODE); }); }); }
+ }; }`);
+ writeFileSync(join(fixture, 'vitest.config.mjs'), FIXTURE_CONFIG);
+ for (const code of [17, 0]) {
+ const durable = join(fixture, 'durable-' + code);
+ const args = ['--config', join(fixture, 'vitest.config.mjs'), '--root', fixture, './fixture.test.js', '--reporter=default', '--reporter=' + lateReporter];
+ const env = { ...process.env, NODE_ENV: 'test', NODE_DISABLE_COMPILE_CACHE: '1',
+ PORTOS_CI_DIAGNOSTICS_DIR: durable, FIXTURE_FINAL_CODE: String(code) };
+ delete env.PORTOS_TEST_TEMP_ROOT;
+ const result = spawnSync(process.execPath, ['--input-type=module', '-e', `
+ import { runNpm } from ${JSON.stringify(new URL('./run-ci-tests.js', import.meta.url).href)};
+ process.exitCode = (await runNpm('server', 'test:ci', ${JSON.stringify(args)})).status;
+ `], { encoding: 'utf8', timeout: 30_000, env });
+ expect(result.error).toBeUndefined();
+ expect(result.status, result.stderr).toBe(code);
+ expect(stripVTControlCharacters(result.stdout)).toMatch(/Test Files\s+1 passed/);
+ const attempt = join(durable, readdirSync(durable)[0]);
+ const run = JSON.parse(readFileSync(join(attempt, 'run.json'), 'utf8'));
+ expect(run).toMatchObject({ reason: 'passed', errorCount: 0,
+ processExit: { eventCode: code, exitCode: code } });
+ // The diagnostic reporter's close callback ran before the late callback.
+ expect(run.exitCode ?? 0).toBe(0);
+ const invocation = JSON.parse(readFileSync(join(attempt, 'invocation.json'), 'utf8'));
+ expect(invocation).toMatchObject({ completed: true, status: code,
+ childExit: { code, signal: null }, childClose: { code, signal: null } });
+ expect(['direct npm process', 'cmd.exe -> npm.cmd']).toContain(invocation.commandBoundary);
+ const { collectTestDiagnostics } = await import('./ci-test-diagnostics.js');
+ const lines = await collectTestDiagnostics(durable, { logger: { warn() {} }, diagnose: async () => {}, writeSummary() {} });
+ expect(lines.join('\n')).toContain(`Vitest process exit event: code=${code}; exitCode=${code}.`);
+ }
+ } finally { rmSync(fixture, { recursive: true, force: true }); }
+ }, 65_000);
+
it('reports a run-level rejection and worker cause when every test passed (#11244)', () => {
const fixture = mkdtempSync(join(tmpdir(), 'portos-diagnostics-'));
try {
diff --git a/scripts/vitestRunDiagnostics.js b/scripts/vitestRunDiagnostics.js
index 8f47470b5c..8c52402c61 100644
--- a/scripts/vitestRunDiagnostics.js
+++ b/scripts/vitestRunDiagnostics.js
@@ -13,19 +13,29 @@ function errorDiagnostic(error, depth = 0) {
const moduleStates = new Set(['queued', 'pending', 'passed', 'failed', 'skipped']);
export default function vitestRunDiagnostics() {
+ const path = process.env.PORTOS_CI_DIAGNOSTICS_PATH;
+ let snapshotExitCode = null;
let diagnostics = { reason: 'not-finished', errorCount: 0, errors: [], completedCount: 0 };
const activeModules = new Set();
const completedModules = [];
const moduleResults = [];
- const persist = () => {
- const path = process.env.PORTOS_CI_DIAGNOSTICS_PATH;
+ const persist = (processExit) => {
+ if (!processExit) snapshotExitCode = process.exitCode ?? null;
if (!path) return;
persistTestDiagnostic(path, { ...diagnostics, activeModules: [...activeModules].slice(0, 100),
- completedModules, moduleResults, exitCode: process.exitCode ?? null });
+ completedModules, moduleResults, exitCode: snapshotExitCode,
+ ...(processExit ? { processExit } : {}) });
};
return {
onInit(vitest) {
- vitest.onClose(persist);
+ // Exit listeners can only do synchronous I/O. Preserve the ordinary hook
+ // snapshot separately: the event is later evidence, not a causal verdict.
+ const onExit = (eventCode) => {
+ const processExit = { eventCode, exitCode: process.exitCode ?? null };
+ persist(processExit);
+ };
+ if (path) process.once('exit', onExit);
+ vitest.onClose(() => persist());
persist();
},
onTestModuleQueued(module) {