Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix([issue-12536]): preserve late test process exit provenance
[._.]/ Adam Eivy581b92299205
diff --git a/docs/GITHUB_ACTIONS.md b/docs/GITHUB_ACTIONS.mdindex da444d2cb0..b475d2f667 100644--- a/docs/GITHUB_ACTIONS.md+++ b/docs/GITHUB_ACTIONS.md@@ -1064,6 +1064,24 @@ test and a minimal Windows reproduction before changing production metadatahandling (#8152). Microsoft documents these fields in its[application crash troubleshooting guide](https://learn.microsoft.com/en-us/troubleshoot/windows-server/performance/troubleshoot-application-service-crashing-behavior).+### Final test process exit evidence++The run report retains the exit-code snapshot from Vitest's normal hooks and+adds the later Node `exit` event code and `process.exitCode` in a synchronous,+best-effort write. The invocation report records the observed command boundary+(`direct npm process`, or `cmd.exe -> npm.cmd` when a batch wrapper is needed), its child `exit` and `close`+events, and the launcher's effective status. Collection prints only bounded+numeric status and allowlisted boundary names. Missing exit evidence stays an+explicit diagnostic gap; diagnostic I/O never changes the original result.++An exit event is later evidence, not proof of a cause or of every subsequent+exit listener's behavior. When a Windows batch wrapper is needed, npm's own process exit is still not+observed independently of `cmd.exe`; a passing Vitest exit with a failing+command boundary narrows the remaining investigation to that boundary. Forced+termination can prevent any exit-event write. Preserve the first failing job+and compare these observations on a hosted recurrence rather than classifying+a green rerun or a synthetic fixture as a causal fix (#12536).+### External cancellation and one automatic retryNot every cancel comes from this repository. When several PRs build at once,diff --git a/scripts/ci-test-diagnostics.js b/scripts/ci-test-diagnostics.jsindex 2aac87ca30..025469b6db 100644--- a/scripts/ci-test-diagnostics.js+++ b/scripts/ci-test-diagnostics.js@@ -27,6 +27,16 @@ export function persistTestDiagnostic(path, value) {}}+// No raw process identity or command arguments enter this projection.+export function processExitDiagnostic(run) {+ const final = run.processExit;+ if (!Number.isInteger(final?.eventCode)+ || (final.exitCode !== null && !Number.isInteger(final.exitCode))) {+ return 'Diagnostic gap: Vitest process exit event unavailable; final process status is unverified.';+ }+ return `Vitest process exit event: code=${final.eventCode}; exitCode=${final.exitCode}.`;+}+function readReport(path, limit = 100_000) {if (statSync(path).size > limit) throw new Error('report exceeds limit');return JSON.parse(readFileSync(path, 'utf8'));@@ -62,6 +72,13 @@ export async function collectTestDiagnostics(root, {lines.push(`Launcher ${invocation.completed ? 'completed' : 'interrupted or completion not recorded'}; `+ `exit=${Number.isInteger(invocation.status) ? invocation.status : 'unavailable'}; `+ `signal=${/^SIG[A-Z]+$/.test(invocation.signal) ? invocation.signal : 'none'}.`);+ const boundary = ['direct npm process', 'cmd.exe -> npm.cmd'].includes(invocation.commandBoundary) ? invocation.commandBoundary : 'unavailable';+ lines.push(`Observed command boundary: ${boundary}.`);+ for (const [event, value] of [['exit', invocation.childExit], ['close', invocation.childClose]]) {+ lines.push(`Command ${event}: code=${Number.isInteger(value?.code) ? value.code : 'unavailable'}; `+ + `signal=${/^SIG[A-Z]+$/.test(value?.signal) ? value.signal : 'none'}.`);+ }+ if (boundary === 'cmd.exe -> npm.cmd') lines.push('Diagnostic gap: npm process exit is not observed independently of the Windows command wrapper.');if (Array.isArray(invocation.selectors)) lines.push(`Selected inputs (up to 100): ${invocation.selectors.slice(0, 100).map(diagnosticTestFile).join(', ') || 'full suite or no file selectors'}`);} catch {lines.push('Diagnostic gap: invocation metadata unavailable; launcher completion is unverified.');@@ -71,6 +88,7 @@ export async function collectTestDiagnostics(root, {if (!Array.isArray(run.activeModules) || !Array.isArray(run.completedModules)|| !Number.isInteger(run.completedCount) || !Number.isInteger(run.errorCount)) throw Error('invalid progress');const reason = ['passed', 'failed', 'interrupted', 'not-finished'].includes(run.reason) ? run.reason : 'unavailable';+ lines.push(processExitDiagnostic(run));lines.push(`Vitest progress: reason=${reason}; completed=${run.completedCount}; unhandledErrors=${run.errorCount}.`);lines.push(`Queued/unfinished modules (up to 100): ${run.activeModules.slice(0, 100).map(diagnosticTestFile).join(', ') || 'none recorded'}`);lines.push(`Last completed modules (up to 100): ${run.completedModules.slice(-100).map(diagnosticTestFile).join(', ') || 'none recorded'}`);diff --git a/scripts/ci-test-diagnostics.test.js b/scripts/ci-test-diagnostics.test.jsindex 7436c30791..0ea90fb9e2 100644--- a/scripts/ci-test-diagnostics.test.js+++ b/scripts/ci-test-diagnostics.test.js@@ -79,6 +79,8 @@ describe('independent CI test failure recovery (#11552)', () => {const output = await launchAndCollect(false);expect(output).toContain('Launcher completed; exit=17');expect(output).toContain('reason=failed; completed=1');+ expect(output).toContain('Vitest process exit event: code=17; exitCode=17.');+ expect(output).toContain('Command exit: code=17; signal=none.');expect(output).toContain('Failed suites (up to 10): server/lib/synthetic.test.js');});@@ -89,9 +91,24 @@ describe('independent CI test failure recovery (#11552)', () => {expect(output).toContain('structured test report missing');});+ it.each([0, 17])('diagnostic write failure preserves process exit %i', (code) => {+ const result = spawnSync(process.execPath, ['--input-type=module', '-e', `+ import reporter from ${JSON.stringify(reporter)};+ const hooks = reporter();+ hooks.onInit({ onClose(callback) { callback(); } });+ hooks.onTestRunEnd([], [], 'passed');+ process.exitCode = ${code};+ `], { encoding: 'utf8', timeout: 15_000,+ env: { ...process.env, PORTOS_CI_DIAGNOSTICS_PATH: new URL(import.meta.url).pathname + '/missing/run.json' } });+ expect(result.error).toBeUndefined();+ expect(result.status).toBe(code);+ expect(result.stderr).toContain('Unable to persist CI test diagnostics.');+ });+it('recovers bounded completed outcomes without final JSON or private error details', async () => {const root = mkdtempSync(join(tmpdir(), 'ci-diagnostics-outcomes-'));const prior = process.env.PORTOS_CI_DIAGNOSTICS_PATH;+ const priorExitListeners = new Set(process.listeners('exit'));try {const attempt = join(root, 'portos-vitest-fixture'); mkdirSync(attempt);const path = join(attempt, 'run.json');@@ -108,8 +125,10 @@ describe('independent CI test failure recovery (#11552)', () => {const lines = await collectTestDiagnostics(root, { logger: { warn() {} }, diagnose: async () => {}, writeSummary() {} });expect(lines.join('\n')).toContain('server/lib/synthetic.test.js=failed');expect(lines.join('\n')).toContain('structured test report missing');+ expect(lines.join('\n')).toContain('Vitest process exit event unavailable');expect(lines.join('\n')).not.toContain('private-sentinel');} finally {+ for (const listener of process.listeners('exit')) if (!priorExitListeners.has(listener)) process.removeListener('exit', listener);if (prior === undefined) delete process.env.PORTOS_CI_DIAGNOSTICS_PATH;else process.env.PORTOS_CI_DIAGNOSTICS_PATH = prior;rmSync(root, { recursive: true, force: true });diff --git a/scripts/run-ci-tests.js b/scripts/run-ci-tests.jsindex 2fa7cf29ca..c13f2045c5 100644--- a/scripts/run-ci-tests.js+++ b/scripts/run-ci-tests.js@@ -11,7 +11,7 @@ import { isDirectlyInvoked } from './lib/directInvocation.js';import { sweepStaleRunRoots, writeOwnerFile } from './lib/vitestStaleRunRoots.js';import { formatErrorAnnotation, writeStepSummary } from './lib/githubOutput.js';import { reportWindowsCrashDiagnostics } from './ci-windows-crash-diagnostics.js';-import { diagnosticTestFile, persistTestDiagnostic } from './ci-test-diagnostics.js';+import { diagnosticTestFile, persistTestDiagnostic, processExitDiagnostic } from './ci-test-diagnostics.js';import { browserTestProfileArgs, browserTestProfileFiles, assertBrowserProfileExecuted, BROWSER_FIDELITY_ONLY_FILES } from './lib/browserTestProfiles.js';import {clearVitestDependencies, createToolVersionResolver, currentCommit, fileResultsFromReport, findVitestDependenciesFile,@@ -162,6 +162,7 @@ export function structuredFailureDiagnostics(reportPath, runReportPath) {throw new Error('invalid run report schema');}lines.push(`Vitest run: reason=${run.reason}, exitCode=${run.exitCode}, unhandledErrors=${run.errorCount}`);+ lines.push(processExitDiagnostic(run));if (run.teardownTimeout) lines.push('Vitest teardown timed out.');if (run.errors.length) lines.push(`Vitest unhandled errors (up to 10):\n${run.errors.slice(0, 10).map(error => error.slice(0, 8000)).join('\n')}`);} catch {@@ -259,7 +260,6 @@ function runNpmWithReport(scope, script, extraArgs, reportPath, env, abortSignalselectors: extraArgs.filter(arg => !arg.startsWith('-')).slice(0, 100).map(arg => diagnosticTestFile(arg.startsWith('../') ? arg.slice(3) : `${scope}/${arg.replace(/^\.\//, '')}`)),};- persistTestDiagnostic(invocationPath, invocation);const args = ['run', script, '--prefix', scope];args.push('--', ...extraArgs, '--reporter=json', `--outputFile.json=${reportPath}`,`--reporter=${join(repoRoot, 'scripts', 'vitestRunDiagnostics.js')}`);@@ -267,6 +267,8 @@ function runNpmWithReport(scope, script, extraArgs, reportPath, env, abortSignal// `shell:false` and throws EINVAL, so this script could never run on a// Windows checkout. See server/lib/bufferedSpawn.js.const { command, args: spawnArgs } = prepareCliSpawn('npm', args, env);+ invocation.commandBoundary = command === 'cmd.exe' ? 'cmd.exe -> npm.cmd' : 'direct npm process';+ persistTestDiagnostic(invocationPath, invocation);return new Promise((resolve) => {// npm is a launcher, not the entire test tree. Give this invocation its// own POSIX group; Windows killProcessTree uses taskkill /T instead. The@@ -319,6 +321,8 @@ function runNpmWithReport(scope, script, extraArgs, reportPath, env, abortSignalchild.stderr.on('data', (chunk) => tee(process.stderr, chunk));// A spawn error also emits close. Settle only there, after the pipes close.child.on('error', (error) => { spawnError = error; });+ let childExit = null;+ child.on('exit', (code, signal) => { childExit = { code, signal }; });const finish = (code, signal) => {if (settled) return;settled = true;@@ -333,7 +337,8 @@ function runNpmWithReport(scope, script, extraArgs, reportPath, env, abortSignalprocess.removeListener('SIGTERM', onSigterm);abortSignal?.removeEventListener('abort', onAbort);const status = interrupted ? (interrupted === 'SIGINT' ? 130 : 143) : code ?? 1;- persistTestDiagnostic(invocationPath, { ...invocation, completed: !cleanupUnverified, status, signal: signal ?? interrupted ?? null });+ persistTestDiagnostic(invocationPath, { ...invocation, completed: !cleanupUnverified, status, signal: signal ?? interrupted ?? null,+ childExit, childClose: { code, signal } });if (status !== 0) {console.error(`Test launcher exited: code=${code}, signal=${signal ?? 'none'}${interrupted ? `, interrupted=${interrupted}` : ''}`);console.error(structuredFailureDiagnostics(reportPath, runReportPath));diff --git a/scripts/run-ci-tests.test.js b/scripts/run-ci-tests.test.jsindex 37825110ee..10c843012c 100644--- a/scripts/run-ci-tests.test.js+++ b/scripts/run-ci-tests.test.js@@ -552,6 +552,47 @@ describe('native crash diagnostics before retry/cancellation (#8152)', () => {describe('structured failure diagnostics', () => {+ // This uniquely pins shutdown after the ordinary reporter snapshot through+ // real Vitest assertions, npm, and the launcher's durable collection boundary.+ it('retains passing assertions and late process exit provenance without changing status (#12536)', async () => {+ const fixture = mkdtempSync(join(tmpdir(), 'portos-final-exit-'));+ try {+ writeFileSync(join(fixture, 'fixture.test.js'), "test('passing control', () => expect(true).toBe(true));");+ const lateReporter = join(fixture, 'late-reporter.mjs');+ writeFileSync(lateReporter, `export default function () { return {+ onInit(vitest) { vitest.onClose(() => { process.once('beforeExit', () => { process.exitCode = Number(process.env.FIXTURE_FINAL_CODE); }); }); }+ }; }`);+ writeFileSync(join(fixture, 'vitest.config.mjs'), FIXTURE_CONFIG);+ for (const code of [17, 0]) {+ const durable = join(fixture, 'durable-' + code);+ const args = ['--config', join(fixture, 'vitest.config.mjs'), '--root', fixture, './fixture.test.js', '--reporter=default', '--reporter=' + lateReporter];+ const env = { ...process.env, NODE_ENV: 'test', NODE_DISABLE_COMPILE_CACHE: '1',+ PORTOS_CI_DIAGNOSTICS_DIR: durable, FIXTURE_FINAL_CODE: String(code) };+ delete env.PORTOS_TEST_TEMP_ROOT;+ const result = spawnSync(process.execPath, ['--input-type=module', '-e', `+ import { runNpm } from ${JSON.stringify(new URL('./run-ci-tests.js', import.meta.url).href)};+ process.exitCode = (await runNpm('server', 'test:ci', ${JSON.stringify(args)})).status;+ `], { encoding: 'utf8', timeout: 30_000, env });+ expect(result.error).toBeUndefined();+ expect(result.status, result.stderr).toBe(code);+ expect(stripVTControlCharacters(result.stdout)).toMatch(/Test Files\s+1 passed/);+ const attempt = join(durable, readdirSync(durable)[0]);+ const run = JSON.parse(readFileSync(join(attempt, 'run.json'), 'utf8'));+ expect(run).toMatchObject({ reason: 'passed', errorCount: 0,+ processExit: { eventCode: code, exitCode: code } });+ // The diagnostic reporter's close callback ran before the late callback.+ expect(run.exitCode ?? 0).toBe(0);+ const invocation = JSON.parse(readFileSync(join(attempt, 'invocation.json'), 'utf8'));+ expect(invocation).toMatchObject({ completed: true, status: code,+ childExit: { code, signal: null }, childClose: { code, signal: null } });+ expect(['direct npm process', 'cmd.exe -> npm.cmd']).toContain(invocation.commandBoundary);+ const { collectTestDiagnostics } = await import('./ci-test-diagnostics.js');+ const lines = await collectTestDiagnostics(durable, { logger: { warn() {} }, diagnose: async () => {}, writeSummary() {} });+ expect(lines.join('\n')).toContain(`Vitest process exit event: code=${code}; exitCode=${code}.`);+ }+ } finally { rmSync(fixture, { recursive: true, force: true }); }+ }, 65_000);+it('reports a run-level rejection and worker cause when every test passed (#11244)', () => {const fixture = mkdtempSync(join(tmpdir(), 'portos-diagnostics-'));try {diff --git a/scripts/vitestRunDiagnostics.js b/scripts/vitestRunDiagnostics.jsindex 8f47470b5c..8c52402c61 100644--- a/scripts/vitestRunDiagnostics.js+++ b/scripts/vitestRunDiagnostics.js@@ -13,19 +13,29 @@ function errorDiagnostic(error, depth = 0) {const moduleStates = new Set(['queued', 'pending', 'passed', 'failed', 'skipped']);export default function vitestRunDiagnostics() {+ const path = process.env.PORTOS_CI_DIAGNOSTICS_PATH;+ let snapshotExitCode = null;let diagnostics = { reason: 'not-finished', errorCount: 0, errors: [], completedCount: 0 };const activeModules = new Set();const completedModules = [];const moduleResults = [];- const persist = () => {- const path = process.env.PORTOS_CI_DIAGNOSTICS_PATH;+ const persist = (processExit) => {+ if (!processExit) snapshotExitCode = process.exitCode ?? null;if (!path) return;persistTestDiagnostic(path, { ...diagnostics, activeModules: [...activeModules].slice(0, 100),- completedModules, moduleResults, exitCode: process.exitCode ?? null });+ completedModules, moduleResults, exitCode: snapshotExitCode,+ ...(processExit ? { processExit } : {}) });};return {onInit(vitest) {- vitest.onClose(persist);+ // Exit listeners can only do synchronous I/O. Preserve the ordinary hook+ // snapshot separately: the event is later evidence, not a causal verdict.+ const onExit = (eventCode) => {+ const processExit = { eventCode, exitCode: process.exitCode ?? null };+ persist(processExit);+ };+ if (path) process.once('exit', onExit);+ vitest.onClose(() => persist());persist();},onTestModuleQueued(module) {