Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { execFileSync, spawn, spawnSync } from 'node:child_process';
import { EventEmitter } from 'node:events';
import { PassThrough } from 'node:stream';
import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, utimesSync, writeFileSync } from 'fs';
import { tmpdir } from 'os';
import { delimiter, dirname, join, relative } from 'path';
import { stripVTControlCharacters } from 'node:util';
import { afterEach, describe, expect, it, vi } from 'vitest';
import {
structuredFailureDiagnostics,
extractCrashedTestFile,
hasOtherTestFailures,
planCrashRetry,
recordVitestDuration,
repoRelativeFromCrashPath,
parseShard,
requiresSourceFiles,
spawnNpm,
toRunnerPath,
} from './run-ci-tests.js';
import { killProcessTree } from '../server/lib/bufferedSpawn.js';
import * as nodeFs from 'node:fs';
import * as nodePath from 'node:path';
import { computeTestMemoKeys, createDiskFs, relatedTests } from './lib/testMemo.js';
import { vitestShardSlice } from './lib/testMemoRunner.js';
import { guardInputSnapshot } from './lib/pregateReport.js';
import { toWorkspacePath } from './lib/workspaceTests.js';
import { BROWSER_SUITES } from './lib/browserSuites.js';
import { staticImportClosure } from '../server/lib/staticImportGraph.js';
import { destroyGitSandbox, makeGitSandbox } from '../server/lib/gitTestRepo.js';
import { readTestProcesses } from './lib/testOwnedProcesses.js';
import { OWNER_FILE, STALE_ROOT_AGE_MS, sweepStaleRunRoots } from './lib/vitestStaleRunRoots.js';
import { workflowJobs } from './lib/workflowJobs.js';
import { buildCiTestPlan, collectPlanInputs } from './ci-test-plan.js';
// Synthetic crash/retry cases must not print a fabricated crash as a real
// Actions annotation. Duration/summary tests retain real I/O.
vi.mock('./lib/githubOutput.js', async (importOriginal) => ({
...await importOriginal(),
formatErrorAnnotation: vi.fn((title, message) => `${title}: ${message}`),
}));
// Keep nested Vitest's own scratch under its separately owned pvt-* root,
// just like the workspace configs; reporter diagnostics stay outside it.
const FIXTURE_CONFIG = `
import { bootstrapVitestTempRoot } from ${JSON.stringify(new URL('./lib/vitestTempRoot.js', import.meta.url).href)};
// Beside the config, not in <fixture>/node_modules/.vite, so removing the fixture removes the cache.
const cacheDir = import.meta.dirname + '/.vitest-cache';
bootstrapVitestTempRoot();
export default { cacheDir,
test: { globals: true, maxWorkers: 1, testTimeout: 60000,
globalSetup: [${JSON.stringify(new URL('./vitestTempRootSetup.js', import.meta.url).href)}] } };
`;
const WORKFLOW = readFileSync(join(import.meta.dirname, '..', '.github', 'workflows', 'ci.yml'), 'utf8');
describe('parseShard', () => {
it('reads a slice only when the matrix actually split the suite', () => {
expect(parseShard(undefined)).toBeNull();
expect(parseShard('')).toBeNull();
expect(parseShard('1/1')).toBeNull();
expect(parseShard('2/3')).toEqual({ index: 2, count: 3 });
expect(() => parseShard('2')).toThrow(/<index>\/<count>/);
});
});
/** A fake `npm` on PATH that prints its arguments and the selection file it was handed. */
function installFakeNpm(fixture, body) {
const npmScript = join(fixture, 'npm.js');
writeFileSync(npmScript, body);
writeFileSync(join(fixture, 'npm'), `#!${process.execPath}\n${body}`, { mode: 0o755 });
writeFileSync(join(fixture, 'npm.cmd'), `@"${process.execPath}" "${npmScript}" %*\r\n`);
}
const REPORTING_NPM = 'process.stdout.write(JSON.stringify({ args: process.argv.slice(2), '
+ 'selection: JSON.parse(require("fs").readFileSync(process.env.PORTOS_CI_TEST_SELECTION, "utf8")) }) + "\\n");';
it('hands Vitest the selection as a file, sliced by the runner for the matrix shard, never as --shard', () => {
const fixture = mkdtempSync(join(tmpdir(), 'portos-related-shard-'));
try {
installFakeNpm(fixture, REPORTING_NPM);
// The planner fans both scoped modes out (ci-test-plan.js fansOut), so
// each must hand Vitest its slice rather than run the whole selection twice.
// Related selection is closure-based: this runner imports
// scripts/lib/directInvocation.js, so the runner's own test is related to it.
const files = ['server/lib/example.test.js', 'scripts/example.test.js'];
for (const [mode, shard, sources, expected] of [
['related', '', ['scripts/lib/directInvocation.js'], ['../scripts/run-ci-tests.test.js', 'lib/example.test.js', '../scripts/example.test.js']],
['files', '', [], ['lib/example.test.js', '../scripts/example.test.js']],
// Two explicit files split three ways: shard 2 of 3 gets exactly one of them.
['files', '2/3', [], [vitestShardSlice(files, { repoRoot: join(import.meta.dirname, '..'), scope: 'server', index: 2, count: 3 })].flat().map((rel) => toWorkspacePath('server', rel))],
]) {
const result = spawnSync(process.execPath, [join(import.meta.dirname, 'run-ci-tests.js'), 'server'], {
encoding: 'utf8', timeout: 120000,
env: { ...process.env, PATH: `${fixture}${delimiter}${process.env.PATH}`,
CI_TEST_MODE: mode, CI_SHARD: shard, PORTOS_TEST_MEMO: 'off',
CI_TEST_SOURCES: JSON.stringify(sources),
CI_TEST_FILES: JSON.stringify(files),
GITHUB_ENV: '', GITHUB_STEP_SUMMARY: '' },
});
expect(result.error).toBeUndefined();
expect(result.status, result.stderr).toBe(0);
const { args, selection } = JSON.parse(result.stdout.split('\n').find(line => line.startsWith('{"args"')));
expect(args.slice(0, 5), mode).toEqual(['run', 'test:ci', '--prefix', 'server', '--']);
expect(args.some(arg => arg.startsWith('--shard')), mode).toBe(false);
expect(args, mode).not.toContain('--experimental.recordDependencies');
expect(selection.exclude, mode).toEqual([]);
expect(selection.include, `${mode} ${shard}`).toEqual(expect.arrayContaining(expected));
if (mode === 'files') expect(selection.include, shard).toHaveLength(expected.length);
}
} finally {
rmSync(fixture, { recursive: true, force: true });
}
}, 150_000);
it('hands the real client fixture consumers to Vitest for a shared browser cleanup change', () => {
const fixture = mkdtempSync(join(tmpdir(), 'portos-client-helper-selection-'));
try {
installFakeNpm(fixture, REPORTING_NPM);
const source = 'scripts/lib/ownedBrowserCleanup.js';
const inputs = collectPlanInputs({ baseSha: 'HEAD', changedFiles: [source], cwd: join(import.meta.dirname, '..') });
const plan = buildCiTestPlan(inputs.changedFiles, inputs);
expect(plan.full).toBe(false);
expect(plan.build).toBe(false);
expect(plan.server.mode).toBe('related');
expect(plan.server.sources).toContain(source);
expect(plan.browserFiles.length).toBeGreaterThan(0);
expect(plan.client.mode).toBe('files');
const result = spawnSync(process.execPath, [join(import.meta.dirname, 'run-ci-tests.js'), 'client'], {
encoding: 'utf8', timeout: 120000,
env: { ...process.env, PATH: `${fixture}${delimiter}${process.env.PATH}`,
CI_TEST_MODE: plan.client.mode, CI_SHARD: '', PORTOS_TEST_MEMO: 'off',
CI_TEST_SOURCES: JSON.stringify(plan.client.sources), CI_TEST_FILES: JSON.stringify(plan.client.files),
PORTOS_PREGATE_GUARDS: '', GITHUB_ENV: '', GITHUB_STEP_SUMMARY: '' },
});
expect(result.error).toBeUndefined();
expect(result.status, result.stderr).toBe(0);
const { selection } = JSON.parse(result.stdout.split('\n').find(line => line.startsWith('{"args"')));
expect(selection.include).toEqual(expect.arrayContaining([
'src/test/browserFixture.test.js',
'src/components/ThemeSwitcher.browser.test.js',
'src/components/goals/GoalsTreeView.browser.test.js',
'src/components/ui/OverflowMenu.browser.test.js',
]));
expect(selection.include.sort()).toEqual(plan.client.files.map(file => toWorkspacePath('client', file)).sort());
} finally {
rmSync(fixture, { recursive: true, force: true });
}
}, 150_000);
it('runs the db scope through the server workspace\'s test:db:ci script with the same selection file', () => {
const fixture = mkdtempSync(join(tmpdir(), 'portos-db-scope-'));
try {
installFakeNpm(fixture, REPORTING_NPM);
const result = spawnSync(process.execPath, [join(import.meta.dirname, 'run-ci-tests.js'), 'db'], {
encoding: 'utf8', timeout: 120000,
env: { ...process.env, PATH: `${fixture}${delimiter}${process.env.PATH}`,
CI_TEST_MODE: 'files', CI_SHARD: '', PORTOS_TEST_MEMO: 'off',
CI_TEST_FILES: JSON.stringify(['server/services/dbAdmin.db.test.js']),
GITHUB_ENV: '', GITHUB_STEP_SUMMARY: '' },
});
expect(result.status, result.stderr).toBe(0);
const { args, selection } = JSON.parse(result.stdout.split('\n').find(line => line.startsWith('{"args"')));
expect(args.slice(0, 5)).toEqual(['run', 'test:db:ci', '--prefix', 'server', '--']);
expect(selection.include).toEqual(['services/dbAdmin.db.test.js']);
expect(result.stdout).toMatch(/⏱ db files|⏱ db selected tests/);
} finally {
rmSync(fixture, { recursive: true, force: true });
}
}, 150_000);
describe('test memo through the real runner', () => {
it('runs a selected file once, then skips it while its inputs are unchanged, and runs it again when they change', () => {
const fixture = mkdtempSync(join(tmpdir(), 'portos-memo-e2e-'));
const memoDir = join(fixture, 'memo');
try {
// A fake npm that reports every selected file as passed, so the contract
// covers the runner's bookkeeping without a nested Vitest run.
installFakeNpm(fixture, [
'const fs = require("fs");',
'const selection = JSON.parse(fs.readFileSync(process.env.PORTOS_CI_TEST_SELECTION, "utf8"));',
'const out = process.argv.find(a => a.startsWith("--outputFile.json=")).slice("--outputFile.json=".length);',
`const repo = ${JSON.stringify(join(import.meta.dirname, '..'))};`,
'const testResults = selection.include.map(rel => ({ name: require("path").resolve(repo, "server", rel), status: "passed", assertionResults: [{ status: "passed" }] }));',
'fs.writeFileSync(out, JSON.stringify({ testResults }));',
// The run-level report scripts/vitestRunDiagnostics.js would write; PORTOS_FAKE_LATE_ERROR simulates a rejection after the last test.
'fs.writeFileSync(process.env.PORTOS_CI_DIAGNOSTICS_PATH, JSON.stringify({ reason: "passed", errorCount: process.env.PORTOS_FAKE_LATE_ERROR ? 1 : 0, errors: [] }));',
'process.stdout.write("RAN " + JSON.stringify(selection.include) + "\\n");',
'process.stdout.write("ARGS " + JSON.stringify(process.argv.slice(2)) + "\\n");',
].join('\n'));
const test = 'scripts/lib/releaseTitle.test.js';
const run = (extraEnv = {}) => spawnSync(process.execPath, [join(import.meta.dirname, 'run-ci-tests.js'), 'server'], {
encoding: 'utf8', timeout: 120000,
env: { ...process.env, PATH: `${fixture}${delimiter}${process.env.PATH}`,
CI_TEST_MODE: 'files', CI_TEST_FILES: JSON.stringify([test]), CI_TEST_SOURCES: '[]',
// An inherited PORTOS_TEST_MEMO=off (a developer's shell, a recording run) must not turn this contract off,
// and an inherited CI_SHARD (a matrix job) must not slice the one selected file away.
PORTOS_TEST_MEMO: '', CI_SHARD: '', PORTOS_TEST_MEMO_DIR: memoDir, PORTOS_TEST_MEMO_VERBOSE: '1', GITHUB_ENV: '', GITHUB_STEP_SUMMARY: '', GITHUB_EVENT_NAME: 'push', ...extraEnv },
});
// A green-looking report with a run-level error records nothing.
const tainted = run({ PORTOS_FAKE_LATE_ERROR: '1' });
expect(tainted.status, tainted.stderr).toBe(0);
expect(tainted.stdout).toContain('RAN ');
expect(tainted.stdout).not.toContain('recorded');
// Pull-request jobs restore trusted memos but never publish them. Their
// green test result must not pay to compute a disposable replacement.
const pullRequestMemo = join(fixture, 'pull-request-memo');
for (let attempt = 0; attempt < 2; attempt++) {
const pr = run({ GITHUB_EVENT_NAME: 'pull_request', PORTOS_TEST_MEMO_DIR: pullRequestMemo });
expect(pr.status, pr.stderr).toBe(0);
expect(pr.stdout).toContain('RAN ["../scripts/lib/releaseTitle.test.js"]');
expect(pr.stdout).not.toContain('recorded');
expect(pr.stdout).not.toContain('--experimental.recordDependencies');
expect(existsSync(pullRequestMemo) && readdirSync(pullRequestMemo).length > 0).toBe(false);
}
const first = run();
expect(first.status, first.stderr).toBe(0);
expect(first.stdout).toContain('🧠 Test memo: skipped 0 of 1 selected files (closure unchanged); running 1.');
expect(first.stdout).toContain('RAN ["../scripts/lib/releaseTitle.test.js"]');
// Two entries when this checkout's Vitest cache already holds a depfile for the file (a recorded key), one otherwise.
expect(first.stdout).toMatch(/recorded [12] green entries for 1 passed files/);
expect(first.stdout).toContain('--experimental.recordDependencies');
const second = run();
expect(second.status, second.stderr).toBe(0);
expect(second.stdout).toContain('🧠 Test memo: skipped 1 of 1 selected files (closure unchanged); running 0.');
expect(second.stdout).toMatch(new RegExp(`skipped ${test.replace(/[./]/g, '\\$&')} \\((?:recorded|floor) key\\)`));
expect(second.stdout).not.toContain('RAN ');
const trustedPr = run({ GITHUB_EVENT_NAME: 'pull_request' });
expect(trustedPr.status, trustedPr.stderr).toBe(0);
expect(trustedPr.stdout).toContain('skipped 1 of 1 selected files');
expect(trustedPr.stdout).not.toContain('RAN ');
// A gating environment variable is part of the key.
const other = run({ PORTOS_TEST_QUIET: process.env.PORTOS_TEST_QUIET === '1' ? '' : '1' });
expect(other.stdout).toContain('skipped 0 of 1 selected files');
// Off restores the pre-memo behavior: no summary, the file runs.
const off = run({ PORTOS_TEST_MEMO: 'off' });
expect(off.stdout).not.toContain('🧠');
expect(off.stdout).toContain('RAN ');
} finally {
rmSync(fixture, { recursive: true, force: true });
}
}, 200_000);
it('selects the same files as `vitest related` on a fixture graph with a dynamic import', () => {
const fixture = mkdtempSync(join(tmpdir(), 'portos-related-parity-'));
try {
const host = join(fixture, 'host');
mkdirSync(host);
writeFileSync(join(fixture, 'vitest.config.mjs'), FIXTURE_CONFIG);
writeFileSync(join(fixture, 'a.js'), 'export const a = 1;\n');
writeFileSync(join(fixture, 'b.js'), "export const load = () => import('./c.js');\n");
writeFileSync(join(fixture, 'c.js'), 'export const c = 1;\n');
writeFileSync(join(fixture, 'a.test.js'), "import { a } from './a.js';\ntest('a', () => { expect(a).toBe(1); });\n");
writeFileSync(join(fixture, 'b.test.js'), "import { load } from './b.js';\ntest('b', async () => { expect((await load()).c).toBe(1); });\n");
writeFileSync(join(fixture, 'plain.test.js'), "test('plain', () => { expect(1).toBe(1); });\n");
// Vitest's JS entry through node: the .bin shim cannot be started by spawnSync on Windows.
const vitest = join(import.meta.dirname, '..', 'server', 'node_modules', 'vitest', 'vitest.mjs');
const env = { ...process.env, NODE_ENV: 'test', TMPDIR: host, TMP: host, TEMP: host, NODE_DISABLE_COMPILE_CACHE: '1', PORTOS_CI_TEST_SELECTION: '' };
delete env.PORTOS_TEST_TEMP_ROOT;
const viaVitest = (source) => {
const report = join(fixture, `${source}.json`);
const result = spawnSync(process.execPath, [vitest, 'related', `./${source}`, '--run', '--config', join(fixture, 'vitest.config.mjs'), '--root', fixture,
'--reporter=json', `--outputFile=${report}`, '--passWithNoTests'], { cwd: fixture, encoding: 'utf8', env, timeout: 60000 });
expect(result.error).toBeUndefined();
return JSON.parse(readFileSync(report, 'utf8')).testResults.map(suite => suite.name.replace(/\\/g, '/').split('/').pop()).sort();
};
const keys = computeTestMemoKeys(['a.test.js', 'b.test.js', 'plain.test.js'], { fs: createDiskFs(fixture, nodeFs, nodePath), workspace: '.', env: {} });
for (const source of ['a.js', 'b.js', 'c.js']) {
expect(relatedTests(keys, [source]).sort(), source).toEqual(viaVitest(source));
}
expect(relatedTests(keys, ['c.js'])).toEqual(['b.test.js']);
} finally {
rmSync(fixture, { recursive: true, force: true });
}
}, 120_000);
});
// Guard admission hashes tree/index/status/refs. Keep synthetic proofs off
// the shared checkout, where other real suites may create source fixtures.
async function createRunnerSandbox(files) {
const sandbox = await makeGitSandbox({ prefix: 'portos-runner-proof-' });
const { scratch, repo } = sandbox;
try {
const sourceRoot = join(import.meta.dirname, '..');
for (const source of staticImportClosure(join(sourceRoot, 'scripts/run-ci-tests.js')).files) {
const destination = join(repo, relative(sourceRoot, source));
mkdirSync(dirname(destination), { recursive: true });
copyFileSync(source, destination);
}
writeFileSync(join(repo, 'package.json'), '{"type":"module"}');
for (const path of files) {
mkdirSync(dirname(join(repo, path)), { recursive: true });
writeFileSync(join(repo, path), '// synthetic runner selection fixture\n');
}
const git = (...args) => execFileSync('git', args, { cwd: repo, encoding: 'utf8' }).trim();
git('config', 'core.hooksPath', join(scratch, 'empty-hooks'));
git('add', '--all');
git('commit', '-qm', 'runner selection fixture');
return sandbox;
} catch (error) {
await destroyGitSandbox(scratch);
throw error;
}
}
it.each(['files', 'full'])('validates the guard proof before the actual %s Vitest selection with the persistent memo off', async (mode) => {
const guard = 'server/lib/index.test.js';
const { scratch: fixture, repo: root } = await createRunnerSandbox([guard, 'scripts/example.test.js']);
try {
installFakeNpm(fixture, 'if (process.env.PORTOS_PREGATE_GUARDS) throw new Error("Guard proof leaked to test process");\n' + REPORTING_NPM);
const result = spawnSync(process.execPath, [join(root, 'scripts/run-ci-tests.js'), 'server'], {
cwd: root, encoding: 'utf8', timeout: 120000,
env: { ...process.env, PATH: `${fixture}${delimiter}${process.env.PATH}`, CI_SHARD: '',
CI_TEST_MODE: mode, CI_TEST_FILES: JSON.stringify([guard, 'scripts/example.test.js']),
CI_TEST_SOURCES: '["scripts/lib/directInvocation.js"]',
PORTOS_PREGATE_GUARDS: JSON.stringify({ files: [guard], snapshot: mode === 'files' ? guardInputSnapshot(root) : 'stale' }),
PORTOS_TEST_MEMO: 'off', GITHUB_ENV: '', GITHUB_STEP_SUMMARY: '' },
});
expect(result.status, result.stderr).toBe(0);
if (mode === 'files') expect(result.stdout).toContain('Reused 1 structural guards');
else expect(result.stdout).not.toContain('Reused');
const { selection } = JSON.parse(result.stdout.split('\n').find(line => line.startsWith('{"args"')));
if (mode === 'files') expect(selection.include).not.toContain('lib/index.test.js');
else expect(selection.include).toContain('lib/index.test.js');
expect(selection.include).toContain('../scripts/example.test.js');
} finally {
await destroyGitSandbox(fixture);
}
// Include the child's existing 120s budget plus real Git setup/teardown.
// Windows can otherwise expire the outer 30s test while that child is valid.
}, 150_000);
it('keeps explicit browser selectors with their owner after structural guard reuse', async () => {
const guard = 'server/lib/index.test.js';
const browser = 'server/services/musicVideo/documentTemplates/shared/toonWorld.browser.test.js';
const ordinary = 'scripts/example.test.js';
const { scratch: fixture, repo: root } = await createRunnerSandbox([guard, browser, ordinary]);
const run = ({ owner, files, proof }) => spawnSync(process.execPath, [join(root, 'scripts/run-ci-tests.js'), 'server'], {
cwd: root, encoding: 'utf8', timeout: 10000,
env: { ...process.env, PATH: `${fixture}${delimiter}${process.env.PATH}`, CI_SHARD: '',
CI_TEST_MODE: 'files', CI_TEST_FILES: JSON.stringify(files), CI_TEST_SOURCES: '[]',
CI: 'true', PORTOS_CI_BROWSER_OWNER: owner, PORTOS_BROWSER_TEST_PROFILE: undefined,
PORTOS_PREGATE_GUARDS: proof ? JSON.stringify({ files: [guard], snapshot: proof }) : '',
PORTOS_TEST_MEMO: 'off', GITHUB_ENV: '', GITHUB_STEP_SUMMARY: '' },
});
try {
expect(BROWSER_SUITES).toContain(browser);
installFakeNpm(fixture, REPORTING_NPM);
const proof = guardInputSnapshot(root);
expect(proof).toEqual(expect.any(String));
// The failing pregate boundary: the guard is already proven and the only
// remaining explicit selector belongs to the separate browser stage.
const unitOnly = run({ owner: 'unit', files: [guard, browser], proof });
expect(unitOnly.error).toBeUndefined();
expect(unitOnly.status, unitOnly.stderr).toBe(0);
expect(unitOnly.stdout).toContain('Reused 1 structural guards');
expect(unitOnly.stdout).toContain('No server tests left to run');
expect(unitOnly.stdout).not.toContain('{"args"');
// A mixed selection still runs its ordinary contract, and dedicated
// browser / local / Windows callers keep explicitly selected coverage.
const mixed = run({ owner: 'unit', files: [ordinary, browser] });
expect(mixed.status, mixed.stderr).toBe(0);
const selectionOf = (result) => JSON.parse(result.stdout.split('\n').find(line => line.startsWith('{"args"'))).selection.include;
expect(selectionOf(mixed)).toEqual(['../scripts/example.test.js']);
const owned = run({ owner: 'browser', files: [browser] });
expect(owned.status, owned.stderr).toBe(0);
expect(selectionOf(owned)).toEqual([toWorkspacePath('server', browser)]);
const local = run({ owner: '', files: [browser] });
expect(local.status, local.stderr).toBe(0);
expect(selectionOf(local)).toEqual([toWorkspacePath('server', browser)]);
// Control: a real on-disk change invalidates the same proof and runs the
// guard, preserving production's exact-input admission under CI=true.
writeFileSync(join(root, 'initial.txt'), 'changed after proof');
const changed = run({ owner: 'unit', files: [guard, browser], proof });
expect(changed.status, changed.stderr).toBe(0);
expect(changed.stdout).not.toContain('Reused');
expect(selectionOf(changed)).toEqual([toWorkspacePath('server', guard)]);
} finally {
await destroyGitSandbox(fixture);
}
});
it('keeps explicit DB selectors with the DB runner after structural guard reuse and test memo subtraction (#13455)', async () => {
const guard = 'server/lib/index.test.js';
const dbTest = 'server/services/githubNotifications.db.test.js';
const ordinary = 'scripts/example.test.js';
const { scratch: fixture, repo: root } = await createRunnerSandbox([guard, dbTest, ordinary]);
const run = ({ files, proof }) => spawnSync(process.execPath, [join(root, 'scripts/run-ci-tests.js'), 'server'], {
cwd: root, encoding: 'utf8', timeout: 10000,
env: { ...process.env, PATH: `${fixture}${delimiter}${process.env.PATH}`, CI_SHARD: '',
CI_TEST_MODE: 'files', CI_TEST_FILES: JSON.stringify(files), CI_TEST_SOURCES: '[]',
CI: 'true', PORTOS_PREGATE_GUARDS: proof ? JSON.stringify({ files: [guard], snapshot: proof }) : '',
PORTOS_TEST_MEMO: 'off', GITHUB_ENV: '', GITHUB_STEP_SUMMARY: '' },
});
try {
installFakeNpm(fixture, REPORTING_NPM);
const proof = guardInputSnapshot(root);
expect(proof).toEqual(expect.any(String));
// When the structural guard is reused, the DB test must not be handed to the server runner as its sole selector.
const result = run({ files: [guard, dbTest], proof });
expect(result.error).toBeUndefined();
expect(result.status, result.stderr).toBe(0);
expect(result.stdout).toContain('Reused 1 structural guards');
expect(result.stdout).toContain('No server tests left to run');
expect(result.stdout).not.toContain('{"args"');
// A mixed selection runs only the ordinary non-DB server contract.
const mixed = run({ files: [ordinary, dbTest] });
expect(mixed.status, mixed.stderr).toBe(0);
const selectionOf = (res) => JSON.parse(res.stdout.split('\n').find(line => line.startsWith('{"args"'))).selection.include;
expect(selectionOf(mixed)).toEqual(['../scripts/example.test.js']);
// A DB-only selection prints "No server tests selected" and exits 0 without invoking Vitest.
const soleDb = run({ files: [dbTest] });
expect(soleDb.status, soleDb.stderr).toBe(0);
expect(soleDb.stdout).toContain('No server tests selected');
expect(soleDb.stdout).not.toContain('{"args"');
} finally {
await destroyGitSandbox(fixture);
}
});
it('rejects empty browser coverage except explicitly deferred smoke-only inputs', () => {
for (const [profile, files, expected] of [
['fidelity', [], 1], ['fidelity', ['server/lib/filmLook.browser.test.js'], 1],
['full', [], 1], ['smoke', [], 1],
['smoke', ['server/routes/musicVideoRichAuthoring.browser.test.js'], 0],
]) {
const result = spawnSync(process.execPath, [join(import.meta.dirname, 'run-ci-tests.js'), 'server'], {
encoding: 'utf8', timeout: 10000,
env: { ...process.env, PORTOS_BROWSER_TEST_PROFILE: profile, CI_TEST_MODE: 'files',
CI_TEST_FILES: JSON.stringify(files), GITHUB_ENV: '', GITHUB_STEP_SUMMARY: '' },
});
expect(result.error).toBeUndefined();
expect(result.status, `${profile}: ${result.stderr}`).toBe(expected);
}
});
describe('ci.yml shard wiring', () => {
const runners = Object.entries(workflowJobs(WORKFLOW)).filter(([, body]) => /(?:run|watch)-ci-tests\.js/.test(body));
it('builds every test runner matrix from the planner and hands each slice to the runner', () => {
expect(runners.map(([id]) => id).sort()).toEqual(['browser', 'client', 'database', 'server', 'windows-server']);
// `browser` runs the planner's cross-workspace browser suites in `files`
// mode (#10312): an exact file list has nothing to shard, so it is the one
// runner without a matrix.
// `database` is one serial job (a single `portos_test`), also without a matrix.
for (const [id, body] of runners.filter(([id]) => id !== 'browser' && id !== 'database')) {
// The fan-out is decided by the impact job, never hardcoded here: a
// scoped plan must collapse to one runner, and a job-level `if` cannot
// read `matrix` to skip the extra shards itself.
expect(body, id).toMatch(/shard: \$\{\{ fromJSON\(needs\.impact\.outputs\.\w+_shards\) \}\}/);
expect(body, id).toContain('CI_SHARD: ${{ matrix.shard }}/${{ strategy.job-total }}');
// Shards race to save one immutable cache entry; without the shard in
// the key the winner's 1/n of the transform artifacts is all that persists.
expect(body, id).toMatch(/key: vitest-\w+-\$\{\{ runner\.os \}\}-\$\{\{ hashFiles\([^)]*\) \}\}-\$\{\{ matrix\.shard \}\}of\$\{\{ strategy\.job-total \}\}/);
}
});
it('memoizes the serial database job through the db scope', () => {
const body = workflowJobs(WORKFLOW).database;
const start = body.indexOf('- name: Run DB-backed tests (portos_test)');
const step = body.slice(start, body.indexOf('- name:', start + 1));
expect(step).toContain('run: node scripts/run-ci-tests.js db');
expect(step).toContain('CI_TEST_MODE: full');
expect(step).toContain('PGDATABASE: portos_test');
expect(step).toContain('PORTOS_TEST_MEMO_DIR:');
expect(body).toMatch(/key: test-memo-[^\n]*-db-\$\{\{ github\.run_id \}\}/);
expect(body).toContain("steps.db-tests.outcome == 'success' && github.event_name != 'pull_request'");
});
it('runs the planner-selected browser suites on their own job with prerequisites required (#10312)', () => {
const body = workflowJobs(WORKFLOW).browser;
expect(body).toContain("if: needs.impact.outputs.browser_files != '[]'");
const start = body.indexOf('- name: Run cross-workspace browser suites');
expect(start).toBeGreaterThan(0);
const step = body.slice(start, body.indexOf('- name:', start + 1));
expect(step).toContain('CI_TEST_MODE: files');
expect(step).toContain('CI_TEST_FILES: ${{ needs.impact.outputs.browser_files }}');
expect(step).toContain("PORTOS_REQUIRE_BROWSER_SUITES: '1'");
// The client workspace must be installed before the suites bundle it.
expect(body.indexOf('npm ci --prefix client')).toBeGreaterThan(0);
expect(body.indexOf('npm ci --prefix client')).toBeLessThan(start);
});
it('deduplicates only Linux browser ownership and runs PR browser suites as smoke', () => {
const jobs = workflowJobs(WORKFLOW);
expect(jobs.server).toContain('PORTOS_CI_BROWSER_OWNER: unit');
expect(jobs['windows-server']).not.toContain('PORTOS_CI_BROWSER_OWNER');
expect(jobs.browser).toContain("github.event_name == 'pull_request' && github.base_ref != 'release' && inputs.full != true && 'smoke' || 'full'");
expect(jobs.database).not.toContain('Run cross-workspace browser suites');
expect(jobs.gate).toContain('CI_GATE_RESULT_BROWSER: ${{ needs.browser.result }}');
expect(jobs['full-gate']).toContain("(github.event_name != 'pull_request' || github.base_ref == 'release' || inputs.full == true)");
expect(jobs.gate).toContain('CI_GATE_RESULT_LINUX_CONTAINMENT:');
expect(jobs.gate).toContain('CI_GATE_RESULT_WINDOWS_SERVER:');
});
it('runs once-only steps on the first shard alone', () => {
// Smoke boot, the production build, and the Scalar-removal pin are not
// sharded work; on every shard they would triple the cost for no coverage.
for (const step of ['Smoke-boot server', 'Build client', 'Check API Explorer bundle has no Scalar chunks']) {
const start = WORKFLOW.indexOf(`- name: ${step}\n`);
expect(start, step).toBeGreaterThan(0);
const condition = WORKFLOW.slice(start).match(/\n {8}if: (.*)\n/)[1];
expect(condition, step).toMatch(/&& matrix\.shard == 1$/);
}
});
it('keeps client lint out of the sharded test jobs entirely', () => {
// Lint used to be a shard-1 step on the client job, running in front of the
// slowest shard's tests. Its own job runs it in parallel and keeps a
// lint-only diff from starting a test job; the `lint` job's own comment
// says why it cannot simply move to a different shard. Both halves are
// pinned here: no runner job invokes the linter, and lint is not a matrix.
const jobs = workflowJobs(WORKFLOW);
for (const [id, body] of runners) expect(body, id).not.toContain('run-ci-lint.js');
expect(jobs.lint).toBeTruthy();
expect(jobs.lint).toContain('run: node scripts/run-ci-lint.js');
expect(jobs.lint).not.toContain('strategy:');
// No `matrix.shard` at all: lint is not a matrix, so a cache key, a step
// name, above all a shard-pinned `if:` naming it is the regression.
expect(jobs.lint).not.toMatch(/matrix\.shard/);
// A scoped plan emits `client_shards: [1]`, so a later-shard pin would skip
// lint on every impact-scoped pull request. Its gate is the plan's lint mode.
expect(jobs.lint).toContain("if: needs.impact.outputs.lint_mode != 'skip'");
// The aggregate gate is the single required check, so a job it does not
// observe can fail without blocking a merge.
expect(jobs.gate).toContain('CI_GATE_RESULT_LINT: ${{ needs.lint.result }}');
expect(jobs.gate).toMatch(/needs: \[[^\]]*\blint\b/);
});
});
describe('toRunnerPath', () => {
it('maps repo paths onto each workspace runner root', () => {
expect(toRunnerPath('client', 'client/src/lib/index.test.js')).toBe('./src/lib/index.test.js');
expect(toRunnerPath('server', 'server/lib/index.test.js')).toBe('./lib/index.test.js');
expect(toRunnerPath('server', 'scripts/checkNodeVersion.test.js')).toBe('../scripts/checkNodeVersion.test.js');
expect(toRunnerPath('client', 'server/lib/postScoring.js')).toBe('../server/lib/postScoring.js');
});
});
describe('recordVitestDuration', () => {
const originalSummary = process.env.GITHUB_STEP_SUMMARY;
let summaryDir;
afterEach(() => {
if (originalSummary === undefined) delete process.env.GITHUB_STEP_SUMMARY;
else process.env.GITHUB_STEP_SUMMARY = originalSummary;
if (summaryDir) {
rmSync(summaryDir, { recursive: true, force: true });
summaryDir = undefined;
}
});
it('writes wall time to the GitHub step summary when one is configured', () => {
summaryDir = mkdtempSync(join(tmpdir(), 'vitest-duration-'));
const summaryPath = join(summaryDir, 'summary.md');
process.env.GITHUB_STEP_SUMMARY = summaryPath;
recordVitestDuration('server', 'full suite', Date.now() - 1500);
expect(readFileSync(summaryPath, 'utf8')).toMatch(/^⏱ server full suite: 1\.\ds\n$/);
});
});
describe('requiresSourceFiles', () => {
it('fails closed only when related mode has no source selector', () => {
expect(requiresSourceFiles('related', [])).toBe(true);
expect(requiresSourceFiles('related', ['server/services/auth.js'])).toBe(false);
expect(requiresSourceFiles('files', [])).toBe(false);
expect(requiresSourceFiles('full', [])).toBe(false);
});
});
describe('extractCrashedTestFile', () => {
// Verbatim shape of Vitest's own message for a forked worker that aborted
// natively (issue 8152 — a Windows 0xC0000409 fail-fast), trimmed to the
// two lines the regex actually needs.
const CRASH_OUTPUT = [
'⎯⎯⎯⎯⎯⎯ Unhandled Errors ⎯⎯⎯⎯⎯⎯',
'Error: [vitest-pool]: Worker forks emitted error.',
'Caused by: Error: Worker exited unexpectedly with exit code 3221226505 during started state while running test file D:/a/PortOS/PortOS/server/services/sprites/importer.test.js',
' Test Files 798 passed | 2 skipped (801)',
].join('\n');
it('pulls the crashed file out of a native worker-crash error', () => {
expect(extractCrashedTestFile(CRASH_OUTPUT)).toBe(
'D:/a/PortOS/PortOS/server/services/sprites/importer.test.js',
);
});
it('finds nothing in an ordinary assertion failure or empty output', () => {
expect(extractCrashedTestFile('FAIL server/lib/foo.test.js > bar\nExpected 1 to be 2')).toBeNull();
expect(extractCrashedTestFile('')).toBeNull();
expect(extractCrashedTestFile(undefined)).toBeNull();
});
});
describe('repoRelativeFromCrashPath', () => {
it('maps an absolute Windows or POSIX runner path onto its workspace', () => {
expect(repoRelativeFromCrashPath('D:/a/PortOS/PortOS/server/services/sprites/importer.test.js'))
.toBe('server/services/sprites/importer.test.js');
expect(repoRelativeFromCrashPath('/home/runner/work/PortOS/PortOS/client/src/lib/foo.test.js'))
.toBe('client/src/lib/foo.test.js');
});
it('returns null when the path names neither workspace', () => {
expect(repoRelativeFromCrashPath('D:/a/PortOS/PortOS/scripts/foo.test.js')).toBeNull();
expect(repoRelativeFromCrashPath('')).toBeNull();
});
});
describe('planCrashRetry', () => {
const CRASH_OUTPUT = 'Caused by: Error: Worker exited unexpectedly with exit code 3221226505 '
+ 'during started state while running test file D:/a/PortOS/PortOS/server/services/sprites/importer.test.js';
it('plans a same-workspace single-file retry for a matching crash', () => {
expect(planCrashRetry('server', CRASH_OUTPUT)).toEqual({
retry: true,
relPath: 'server/services/sprites/importer.test.js',
selector: './services/sprites/importer.test.js',
});
});
it('removes reporter color resets from the retry filename', () => {
expect(planCrashRetry('server', '\u001b[31m' + CRASH_OUTPUT + '\u001b[39m')).toEqual({
retry: true,
relPath: 'server/services/sprites/importer.test.js',
selector: './services/sprites/importer.test.js',
});
});
it('declines to retry a crash reported in the other workspace', () => {
// A server-shard crash never gets replayed as a client-scoped selector.
expect(planCrashRetry('client', CRASH_OUTPUT)).toEqual({
retry: false,
crashedPath: 'D:/a/PortOS/PortOS/server/services/sprites/importer.test.js',
});
});
it('declines to retry an ordinary test failure', () => {
expect(planCrashRetry('server', 'FAIL server/lib/foo.test.js\nExpected true to be false')).toEqual({
retry: false,
});
});
});
describe('hasOtherTestFailures', () => {
// The real end-of-run summary block from the crash this fixes (issue 8152):
// one crashed file counted in neither "Test Files" nor "Tests", and exactly
// one unhandled error — the crash itself, nothing else.
const CLEAN_RUN_SUMMARY = [
'Test Files 798 passed | 2 skipped (801)',
' Tests 15077 passed | 61 skipped (15145)',
' Errors 1 error',
].join('\n');
it('reads false from a summary with no failures and exactly one (the crash\'s own) unhandled error', () => {
expect(hasOtherTestFailures(CLEAN_RUN_SUMMARY)).toBe(false);
});
it('reads true when either summary line reports a real failure, whichever order it lists the counts', () => {
expect(hasOtherTestFailures(CLEAN_RUN_SUMMARY.replace(
'Test Files 798 passed', 'Test Files 1 failed | 797 passed',
))).toBe(true);
expect(hasOtherTestFailures(CLEAN_RUN_SUMMARY.replace(
'Tests 15077 passed', 'Tests 15076 passed | 1 failed',
))).toBe(true);
});
it('parses colored summaries without masking assertion failures', () => {
const colored = CLEAN_RUN_SUMMARY.replace(/(\d+ (?:passed|error))/g, '\u001b[32m$1\u001b[39m');
expect(hasOtherTestFailures(colored)).toBe(false);
expect(hasOtherTestFailures(colored.replace('798 passed', '797 passed | \u001b[31m1 failed'))).toBe(true);
});
it('reads true when a second, unrelated unhandled error shares the run with the crash', () => {
expect(hasOtherTestFailures(CLEAN_RUN_SUMMARY.replace('1 error', '2 errors'))).toBe(true);
});
it('fails closed (assumes a real failure) when any summary line is missing', () => {
expect(hasOtherTestFailures('')).toBe(true);
expect(hasOtherTestFailures('Test Files 798 passed | 2 skipped (801)')).toBe(true);
expect(hasOtherTestFailures('some unrelated crash output with no summary at all')).toBe(true);
});
});
describe('native crash diagnostics before retry/cancellation (#8152)', () => {
const output = 'Worker exited unexpectedly with exit code 3221226505 during started state while running test file '
+ 'D:/a/PortOS/PortOS/server/services/sprites/walk.test.js\n'
+ 'Test Files 798 passed | 2 skipped (801)\nTests 15077 passed | 61 skipped (15145)\nErrors 1 error';
it('captures each failed invocation before returning nonzero with the same one-retry ceiling', async () => {
const order = [];
const run = vi.fn(async () => {
order.push('run');
return { status: 1, output };
});
const diagnose = vi.fn(async () => { order.push('diagnose'); });
expect(await spawnNpm('server', 'test:ci', ['--experimental.recordDependencies'], 'fixture', { run, diagnose })).toBe(1);
expect(order).toEqual(['run', 'diagnose', 'run', 'diagnose']);
// The retry narrows the selectors to the crashed file but keeps the run's flags, so its record is taken too.
expect(run.mock.calls[1]).toEqual(['server', 'test:ci', ['./services/sprites/walk.test.js', '--experimental.recordDependencies']]);
});
it('refuses an explicitly empty browser profile before invoking tests', async () => {
vi.stubEnv('PORTOS_BROWSER_TEST_PROFILE', '');
const run = vi.fn();
try {
await expect(spawnNpm('server', 'test:ci', ['selected.browser.test.js'], 'invalid', { run }))
.rejects.toThrow('Unknown browser test profile');
expect(run).not.toHaveBeenCalled();
} finally { vi.unstubAllEnvs(); }
});
it('keeps browser profile selection on a recovered worker retry', async () => {
vi.stubEnv('PORTOS_BROWSER_TEST_PROFILE', 'smoke');
try {
const run = vi.fn().mockResolvedValueOnce({ status: 1, output })
.mockResolvedValueOnce({ status: 0, output: 'passed' });
expect(await spawnNpm('server', 'test:ci', ['selected.browser.test.js'], 'smoke', {
run, diagnose: vi.fn(),
})).toBe(0);
const filter = '--testNamePattern=^(?!.*\\[fidelity\\]).*$';
expect(run.mock.calls[0][2]).toEqual(['selected.browser.test.js', filter]);
expect(run.mock.calls[1][2]).toEqual(['./services/sprites/walk.test.js', filter]);
} finally {
vi.unstubAllEnvs();
}
});
it('retains diagnostic evidence for recovered crashes without clearing unrelated assertion failures', async () => {
for (const [originalOutput, expected] of [[output, 0], [output.replace('798 passed', '797 passed | 1 failed'), 1]]) {
const run = vi.fn().mockResolvedValueOnce({ status: 1, output: originalOutput })
.mockResolvedValueOnce({ status: 0, output: 'passed' });
const diagnose = vi.fn();
expect(await spawnNpm('server', 'test:ci', [], 'fixture', { run, diagnose })).toBe(expected);
expect(diagnose).toHaveBeenCalledTimes(1);
expect(run).toHaveBeenCalledTimes(2);
}
});
it('leaves assertion failures and interrupted runs outside native diagnostics/retry', async () => {
for (const result of [{ status: 1, output: 'FAIL assertion' }, { status: 143, output, interrupted: 'SIGTERM' }]) {
const run = vi.fn().mockResolvedValue(result);
const diagnose = vi.fn();
expect(await spawnNpm('server', 'test:ci', [], 'fixture', { run, diagnose })).toBe(result.status);
expect(run).toHaveBeenCalledTimes(1);
expect(diagnose).not.toHaveBeenCalled();
}
});
});
describe('structured failure diagnostics', () => {
// This uniquely pins shutdown after the ordinary reporter snapshot through
// real Vitest assertions, npm, and the launcher's durable collection boundary.
it('retains passing assertions and late process exit provenance without changing status (#12536)', async () => {
const fixture = mkdtempSync(join(tmpdir(), 'portos-final-exit-'));
try {
writeFileSync(join(fixture, 'fixture.test.js'), "test('passing control', () => expect(true).toBe(true));");
const lateReporter = join(fixture, 'late-reporter.mjs');
writeFileSync(lateReporter, `export default function () { return {
onInit(vitest) { vitest.onClose(() => { process.once('beforeExit', () => { process.exitCode = Number(process.env.FIXTURE_FINAL_CODE); }); }); }
}; }`);
writeFileSync(join(fixture, 'vitest.config.mjs'), FIXTURE_CONFIG);
for (const code of [17, 0]) {
const durable = join(fixture, 'durable-' + code);
const args = ['--config', join(fixture, 'vitest.config.mjs'), '--root', fixture, './fixture.test.js', '--reporter=default', '--reporter=' + lateReporter];
const env = { ...process.env, NODE_ENV: 'test', NODE_DISABLE_COMPILE_CACHE: '1',
PORTOS_CI_DIAGNOSTICS_DIR: durable, FIXTURE_FINAL_CODE: String(code) };
delete env.PORTOS_TEST_TEMP_ROOT;
const result = spawnSync(process.execPath, ['--input-type=module', '-e', `
import { runNpm } from ${JSON.stringify(new URL('./run-ci-tests.js', import.meta.url).href)};
process.exitCode = (await runNpm('server', 'test:ci', ${JSON.stringify(args)})).status;
`], { encoding: 'utf8', timeout: 30_000, env });
expect(result.error).toBeUndefined();
expect(result.status, result.stderr).toBe(code);
expect(stripVTControlCharacters(result.stdout)).toMatch(/Test Files\s+1 passed/);
const attempt = join(durable, readdirSync(durable)[0]);
const run = JSON.parse(readFileSync(join(attempt, 'run.json'), 'utf8'));
expect(run).toMatchObject({ reason: 'passed', errorCount: 0,
processExit: { eventCode: code, exitCode: code } });
// The diagnostic reporter's close callback ran before the late callback.
expect(run.exitCode ?? 0).toBe(0);
const invocation = JSON.parse(readFileSync(join(attempt, 'invocation.json'), 'utf8'));
expect(invocation).toMatchObject({ completed: true, status: code,
childExit: { code, signal: null }, childClose: { code, signal: null } });
expect(['direct npm process', 'cmd.exe -> npm.cmd']).toContain(invocation.commandBoundary);
const { collectTestDiagnostics } = await import('./ci-test-diagnostics.js');
const lines = await collectTestDiagnostics(durable, { logger: { warn() {} }, diagnose: async () => {}, writeSummary() {} });
expect(lines.join('\n')).toContain(`Vitest process exit event: code=${code}; exitCode=${code}.`);
}
} finally { rmSync(fixture, { recursive: true, force: true }); }
}, 65_000);
it('reports a run-level rejection and worker cause when every test passed (#11244)', () => {
const fixture = mkdtempSync(join(tmpdir(), 'portos-diagnostics-'));
try {
const host = join(fixture, 'host');
mkdirSync(host);
writeFileSync(join(fixture, 'vitest.config.mjs'), FIXTURE_CONFIG);
writeFileSync(join(fixture, 'fixture.test.js'), `
test('passing assertion with unhandled rejection', async () => {
Promise.reject(new Error('synthetic run-level rejection', {
cause: new Error('Worker exited unexpectedly with exit code 3221226505'),
}));
await new Promise(resolve => setImmediate(resolve));
expect(true).toBe(true);
});`);
const runnerUrl = new URL('./run-ci-tests.js', import.meta.url).href;
const args = ['--config', join(fixture, 'vitest.config.mjs'), '--root', fixture, './fixture.test.js'];
const env = { ...process.env, NODE_ENV: 'test', TMPDIR: host, TMP: host, TEMP: host, NODE_DISABLE_COMPILE_CACHE: '1' };
delete env.PORTOS_TEST_TEMP_ROOT;
const result = spawnSync(process.execPath, ['--input-type=module', '-e',
`import { runNpm } from ${JSON.stringify(runnerUrl)};
import { passedTestFiles } from ${JSON.stringify(new URL('./lib/testMemoRunner.js', import.meta.url).href)};
const result = await runNpm('server', 'test:ci', ${JSON.stringify(args)});
console.log('MEMO ' + JSON.stringify({ status: result.status, runReport: result.runReport, files: result.fileResults.map(f => f.status),
passed: passedTestFiles(result.fileResults, { repoRoot: ${JSON.stringify(fixture)}, status: result.status, runReport: result.runReport }) }));
process.exitCode = result.status;`,
], { encoding: 'utf8', timeout: 30_000, env });
expect(result.error).toBeUndefined();
expect(result.status).toBe(1);
expect(stripVTControlCharacters(result.stdout)).toMatch(/Test Files\s+1 passed/);
// The JSON report calls the file passed; the process and run-level report say otherwise, and the memo believes them.
const memo = JSON.parse(result.stdout.split('\n').find(line => line.startsWith('MEMO ')).slice(5));
expect(memo).toMatchObject({ status: 1, files: ['passed'], passed: [] });
expect(memo.runReport.errorCount).toBe(1);
const diagnostic = result.stderr.slice(result.stderr.indexOf('Test launcher exited:'));
expect(diagnostic).toContain('code=1, signal=none');
expect(diagnostic).toContain('unhandledErrors=1');
const errors = diagnostic.slice(diagnostic.indexOf('Vitest unhandled errors'), diagnostic.indexOf('Inconclusive'));
expect(errors).toContain('synthetic run-level rejection');
expect(errors).toContain('fixture.test.js:');
expect(errors).toContain('Worker exited unexpectedly with exit code 3221226505');
expect(diagnostic).toContain('Vitest output tail');
expect(readdirSync(host)).toEqual([]);
} finally {
rmSync(fixture, { recursive: true, force: true });
}
}, 35_000);
it('recovers a real bail failure with default and GitHub reporters even if their summary is incomplete', () => {
const fixture = mkdtempSync(join(tmpdir(), 'portos-diagnostics-'));
try {
const host = join(fixture, 'host');
mkdirSync(host);
const report = join(fixture, 'report.json');
writeFileSync(join(fixture, 'fixture.test.js'),
"test('synthetic assertion diagnostic', () => { expect(1, 'synthetic mismatch').toBe(2); });");
writeFileSync(join(fixture, 'vitest.config.mjs'),
FIXTURE_CONFIG);
const runnerUrl = new URL('./run-ci-tests.js', import.meta.url).href;
const args = ['--config', join(fixture, 'vitest.config.mjs'), '--root', fixture, './fixture.test.js'];
const durable = join(fixture, 'durable');
const env = { ...process.env, NODE_ENV: 'test', TMPDIR: host, TMP: host, TEMP: host, NODE_DISABLE_COMPILE_CACHE: '1',
PORTOS_CI_DIAGNOSTICS_DIR: durable };
delete env.PORTOS_TEST_TEMP_ROOT;
const result = spawnSync(process.execPath, ['--input-type=module', '-e',
`import { runNpm } from ${JSON.stringify(runnerUrl)};
const result = await runNpm('server', 'test:ci', ${JSON.stringify(args)});
process.exitCode = result.status;`,
], { encoding: 'utf8', timeout: 30_000, env });
expect(result.error).toBeUndefined();
expect(result.status).toBe(1);
expect(readdirSync(host)).toEqual([]);
const attempts = readdirSync(durable);
expect(attempts).toHaveLength(1);
const invocation = JSON.parse(readFileSync(join(durable, attempts[0], 'invocation.json'), 'utf8'));
expect(invocation).toMatchObject({ completed: true, status: 1 });
const progress = JSON.parse(readFileSync(join(durable, attempts[0], 'run.json'), 'utf8'));
// Vitest reports bail=1 as an interrupted run, even with a failed suite.
expect(progress).toMatchObject({ reason: 'interrupted', completedCount: 1, activeModules: [],
completedModules: ['<outside-repository>/fixture.test.js'] });
expect(result.stdout).toContain('Test Files');
expect(result.stdout).toContain('Tests');
expect(result.stderr).toContain('Failed Tests');
// Inspect only the structured fallback, independent of terminal summaries.
const diagnostic = result.stderr.slice(result.stderr.indexOf('Structured Vitest'));
expect(diagnostic).toContain('fixture.test.js > synthetic assertion diagnostic');
expect(diagnostic).toContain('synthetic mismatch');
expect(diagnostic).toContain('Structured Vitest failure diagnostics');
writeFileSync(report, '{');
expect(structuredFailureDiagnostics(report)).toContain('Inconclusive');
writeFileSync(report, JSON.stringify({ testResults: [] }));
expect(structuredFailureDiagnostics(report)).toContain('Inconclusive');
expect(structuredFailureDiagnostics(join(fixture, 'missing.json'))).toContain('Inconclusive');
} finally {
rmSync(fixture, { recursive: true, force: true });
}
}, 35_000);
});
// A containment report written after the ordinary reporter snapshot must survive
// the launcher's nondurable-attempt cleanup, including otherwise green runs.
it('retains process settlement evidence from a completed launcher attempt', async () => {
const fixture = mkdtempSync(join(tmpdir(), 'process-report-launcher-'));
let diagnosticsPath;
try {
installFakeNpm(fixture, `require('node:fs').writeFileSync(process.env.PORTOS_TEST_PROCESS_DIAGNOSTICS_PATH,
JSON.stringify({ version: 1, total: 1, terminated: 1, unsettled: 0, processes: [] }));`);
const { runNpm } = await import('./run-ci-tests.js');
const result = await runNpm('server', 'test:ci', [], {
env: { ...process.env, PORTOS_CI_DIAGNOSTICS_DIR: '', PATH: `${fixture}${delimiter}${process.env.PATH}` },
});
diagnosticsPath = result.diagnosticsPath;
expect(result.status).toBe(0);
expect(diagnosticsPath).toBeTruthy();
expect(JSON.parse(readFileSync(join(diagnosticsPath, 'processes.json'), 'utf8'))).toMatchObject({ total: 1, terminated: 1 });
} finally {
if (diagnosticsPath) rmSync(diagnosticsPath, { recursive: true, force: true });
rmSync(fixture, { recursive: true, force: true });
}
});
describe('report launcher subprocess lifecycle (#9804)', () => {
const runnerUrl = new URL('./run-ci-tests.js', import.meta.url).href;
it('retains temporary diagnostics when cancellation cannot verify child cleanup', async () => {
const host = mkdtempSync(join(tmpdir(), 'launcher-unverified-'));
const child = Object.assign(new EventEmitter(), {
stdout: new PassThrough(), stderr: new PassThrough(), unref: vi.fn(),
});
const errors = vi.spyOn(console, 'error').mockImplementation(() => {});
vi.useFakeTimers();
vi.doMock('child_process', async importOriginal => ({ ...await importOriginal(), spawn: () => child }));
vi.doMock('node:os', async importOriginal => ({ ...await importOriginal(), tmpdir: () => host }));
vi.doMock('../server/lib/bufferedSpawn.js', async importOriginal => ({
...await importOriginal(), killProcessTree: vi.fn(),
}));
vi.resetModules();
try {
const { runNpm } = await import('./run-ci-tests.js');
const controller = new AbortController();
const pending = runNpm('server', 'test:ci', [], {
env: { ...process.env, PORTOS_CI_DIAGNOSTICS_DIR: '' },
signal: controller.signal, cancellationTimeoutMs: 100,
});
controller.abort();
await vi.advanceTimersByTimeAsync(100);
const result = await pending;
expect(result).toMatchObject({ status: 143, interrupted: 'SIGTERM', cleanupUnverified: true });
expect(result.error.message).toContain('cleanup unverified');
expect(result.diagnosticsPath).toBe(join(host, readdirSync(host)[0]));
expect(JSON.parse(readFileSync(join(result.diagnosticsPath, 'invocation.json'), 'utf8')))
.toMatchObject({ completed: false, status: 143, signal: 'SIGTERM', childExit: null, childClose: null });
expect(errors).toHaveBeenCalledWith(expect.stringContaining(result.diagnosticsPath));
} finally {
vi.useRealTimers();
vi.doUnmock('child_process');
vi.doUnmock('node:os');
vi.doUnmock('../server/lib/bufferedSpawn.js');
vi.resetModules();
errors.mockRestore();
child.stdout.destroy();
child.stderr.destroy();
rmSync(host, { recursive: true, force: true });
}
});
function launch(host, name, { blocking = false, abortOnSignal = false } = {}) {
const fixture = join(host, name);
mkdirSync(fixture);
const ready = join(fixture, 'ready.json');
writeFileSync(join(fixture, 'vitest.config.mjs'),
FIXTURE_CONFIG);
writeFileSync(join(fixture, 'fixture.test.js'), blocking ? `
import { spawn, execFileSync } from 'node:child_process';
import { renameSync, writeFileSync } from 'node:fs';
test('blocking fixture', async () => {
// A descendant that ignores graceful signals must be escalated too.
const descendant = spawn(process.execPath, ['-e',
"process.on('SIGTERM', () => {}); process.on('SIGINT', () => {}); setInterval(() => {}, 1000); process.send('ready');"],
{ stdio: ['ignore', 'ignore', 'ignore', 'ipc'] });
await new Promise(resolve => descendant.once('message', resolve));
writeFileSync(${JSON.stringify(ready + '.tmp')}, JSON.stringify({ worker: process.pid, descendant: descendant.pid,
group: process.platform === 'win32' ? null : Number(execFileSync('ps', ['-o', 'pgid=', '-p', String(process.pid)], { encoding: 'utf8' }).trim()) }));
renameSync(${JSON.stringify(ready + '.tmp')}, ${JSON.stringify(ready)});
await new Promise(() => {});
});` : "test('success fixture', () => { expect(true).toBe(true); });");
const args = ['--config', join(fixture, 'vitest.config.mjs'), '--root', fixture, './fixture.test.js'];
const env = { ...process.env, TMPDIR: host, TMP: host, TEMP: host, NODE_ENV: 'test', NODE_DISABLE_COMPILE_CACHE: '1' };
delete env.PORTOS_TEST_TEMP_ROOT;
const child = spawn(process.execPath, ['--input-type=module', '-e', `
import { runNpm } from ${JSON.stringify(runnerUrl)};
const controller = new AbortController();
if (${abortOnSignal}) process.once('SIGUSR2', () => controller.abort());
const result = await runNpm('server', 'test:ci', ${JSON.stringify(args)},
${abortOnSignal} ? { signal: controller.signal } : {});
process.exitCode = result.status;`], { env, stdio: ['ignore', 'pipe', 'pipe'] });
let output = '';
child.stdout.on('data', chunk => { output += chunk; });
child.stderr.on('data', chunk => { output += chunk; });
const closed = new Promise((resolve, reject) => {
const timeout = setTimeout(() => {
if (existsSync(ready)) {
const pids = JSON.parse(readFileSync(ready, 'utf8'));
if (pids.group) killProcessTree({ pid: pids.group }, 'SIGKILL', { processGroup: true });
}
killProcessTree(child, 'SIGKILL');
reject(Error('report launcher exceeded its subprocess budget'));
}, 40000);
child.once('error', error => { clearTimeout(timeout); reject(error); });
child.once('close', (code, signal) => {
clearTimeout(timeout);
resolve({ code, signal, output });
});
});
// Shared teardown: reads ready.json whenever present (even if the caller's
// waitFor threw first), kills the detached process group plus worker and
// descendant, and awaits the launcher's close before the caller removes host.
async function cleanup() {
killProcessTree(child, 'SIGKILL');
if (existsSync(ready)) {
const pids = JSON.parse(readFileSync(ready, 'utf8'));
if (pids.group) killProcessTree({ pid: pids.group }, 'SIGKILL', { processGroup: true });
for (const pid of [pids.worker, pids.descendant]) if (pid) killProcessTree({ pid }, 'SIGKILL');
}
await Promise.allSettled([closed]);
}
return { child, closed, ready, cleanup };
}
async function waitFor(check, message) {
const deadline = Date.now() + 15000;
while (!check()) {
if (Date.now() > deadline) throw Error(message);
await new Promise(resolve => setTimeout(resolve, 25));
}
}
function ownedRoot(host, pid) {
return readdirSync(host).filter(name => name.startsWith('portos-vitest-'))
.map(name => join(host, name))
.filter(root => existsSync(join(root, OWNER_FILE)))
.find(root => readFileSync(join(root, OWNER_FILE), 'utf8').startsWith(`${pid} `));
}
function alive(pid) {
try { process.kill(pid, 0); return true; }
catch (error) { if (error.code === 'ESRCH') return false; throw error; }
}
// Windows cannot deliver POSIX SIGINT/SIGTERM to a Node process; ordinary
// outcomes and recovery below still exercise its real npm.cmd boundary.
describe.skipIf(process.platform === 'win32')('graceful signals', () => {
for (const signal of ['SIGINT', 'SIGTERM', 'SIGUSR2']) {
it(`${signal} fails the stage, stops descendants, and removes the report root`, async () => {
const host = mkdtempSync(join(tmpdir(), 'rcl-'));
let run;
let pids;
try {
run = launch(host, 'cancel', { blocking: true, abortOnSignal: signal === 'SIGUSR2' });
await waitFor(() => existsSync(run.ready), 'test worker did not start');
pids = JSON.parse(readFileSync(run.ready, 'utf8'));
expect(ownedRoot(host, run.child.pid)).toBeTruthy();
run.child.kill(signal);
const result = await run.closed;
expect(result.code, result.output).toBe(signal === 'SIGINT' ? 130 : 143);
expect(ownedRoot(host, run.child.pid)).toBeUndefined();
await waitFor(() => !alive(pids.worker) && !alive(pids.descendant), 'owned test processes survived cancellation');
} finally {
await run?.cleanup();
rmSync(host, { recursive: true, force: true });
}
}, 35000);
}
});
it('recovers a killed owner on the next successful launch while preserving a concurrent old live root and host entries', async () => {
const host = mkdtempSync(join(tmpdir(), 'rcl-'));
const runs = [];
const descendants = [];
try {
const old = new Date(Date.now() - STALE_ROOT_AGE_MS * 2);
for (const name of ['unrelated-old', 'portos-vitest-legacy-old', 'portos-vitest-legacy-fresh']) {
mkdirSync(join(host, name));
if (name.endsWith('-old')) utimesSync(join(host, name), old, old);
}
const live = launch(host, 'live', { blocking: true });
runs.push(live);
await waitFor(() => existsSync(live.ready), 'live worker did not start');
const livePids = JSON.parse(readFileSync(live.ready, 'utf8'));
descendants.push(livePids.worker, livePids.descendant);
const liveRoot = ownedRoot(host, live.child.pid);
expect(liveRoot).toBeTruthy();
utimesSync(liveRoot, old, old);
const doomed = launch(host, 'doomed', { blocking: true });
runs.push(doomed);
await waitFor(() => existsSync(doomed.ready), 'doomed worker did not start');
const doomedPids = JSON.parse(readFileSync(doomed.ready, 'utf8'));
descendants.push(doomedPids.worker, doomedPids.descendant);
const deadRoot = ownedRoot(host, doomed.child.pid);
expect(deadRoot).toBeTruthy();
// Abruptly kill only this fixture's launcher/tree. Its finally cannot run.
if (process.platform === 'win32') killProcessTree(doomed.child, 'SIGKILL');
else {
doomed.child.kill('SIGKILL');
killProcessTree({ pid: doomedPids.group }, 'SIGKILL', { processGroup: true });
}
await doomed.closed;
await waitFor(() => !alive(doomed.child.pid), 'doomed launcher survived SIGKILL');
expect(existsSync(deadRoot)).toBe(true);
const next = launch(host, 'success');
runs.push(next);
const result = await next.closed;
expect(result.code, result.output).toBe(0);
expect(ownedRoot(host, next.child.pid)).toBeUndefined();
// Unreadable unrelated host processes now make automatic reclamation
// conservative. Prove retirement against the complete fixture inventory.
sweepStaleRunRoots(host, { prefix: 'portos-vitest-',
read: ids => readTestProcesses(ids ?? [process.pid, process.ppid, ...descendants, ...runs.map(run => run.child.pid)]) });
expect(existsSync(deadRoot)).toBe(false);
expect(existsSync(liveRoot)).toBe(true);
expect(alive(live.child.pid)).toBe(true);
expect(existsSync(join(host, 'unrelated-old'))).toBe(true);
// Windows has no process snapshot reader: dead-owner evidence permits
// cleanup above, but age alone cannot prove a pid-less root is unused.
expect(existsSync(join(host, 'portos-vitest-legacy-old'))).toBe(process.platform === 'win32');
expect(existsSync(join(host, 'portos-vitest-legacy-fresh'))).toBe(true);
} finally {
await Promise.allSettled(runs.map(run => run.cleanup()));
rmSync(host, { recursive: true, force: true });
}
// Nothing may outlive teardown to recreate host (or its pvt-* roots) later.
await waitFor(() => descendants.every(pid => !alive(pid)), 'fixture processes survived teardown');
expect(existsSync(host)).toBe(false);
}, 65000);
});