Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Select related tests by closure and skip memoized files in the CI runner

[._.]/ Adam Eivyba2f4db42422
diff --git a/client/package.json b/client/package.json
index f588b42f28..c0412bde1f 100644
--- a/client/package.json
+++ b/client/package.json
@@ -15,7 +15,6 @@
"preview": "vite preview --configLoader runner",
"test": "vitest run --configLoader runner",
"test:ci": "vitest run --bail=1 --silent=passed-only --reporter=default --reporter=github-actions --configLoader runner",
- "test:ci:related": "vitest related --run --bail=1 --silent=passed-only --reporter=default --reporter=github-actions --configLoader runner",
"test:watch": "vitest --configLoader runner"
},
"dependencies": {
diff --git a/scripts/lib/testMemoRunner.js b/scripts/lib/testMemoRunner.js
new file mode 100644
index 0000000000..870c1a1a92
--- /dev/null
+++ b/scripts/lib/testMemoRunner.js
@@ -0,0 +1,205 @@
+/**
+ * Test memo orchestration for `scripts/run-ci-tests.js`: decide which of the
+ * plan's selected files a trusted green run already verified, and record the
+ * new green results afterwards. The same code runs under `npm run pregate`
+ * and in CI, so there is one opinion about what the memo covers.
+ *
+ * Selection is closure-based (`testMemo.js`): in `related` mode a test is
+ * selected when a changed source is in its module closure, or when its edges
+ * could not all be resolved. The planner's explicit files are always added.
+ * Vitest then receives the exact list through a selection file
+ * (`vitestRunnerSelection` in `scripts/vitestCiPool.js`).
+ *
+ * Recording: Vitest's `experimental.recordDependencies` depfile names the
+ * modules each test actually loaded. A passed file gets a memo record (that
+ * list) and an entry under its recorded key, plus an entry under its floor
+ * key when the static closure is fully resolvable — so the next run can hit
+ * either.
+ */
+
+import { spawnSync } from 'node:child_process';
+import * as nodeFs from 'node:fs';
+import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import * as nodePath from 'node:path';
+import { join, relative, sep } from 'node:path';
+import { WORKSPACE_BASE_INPUTS, createDiskFs, createTestMemoAnalyzer, relatedTests } from './testMemo.js';
+import { readMemoEntry, readMemoRecord, writeMemoEntry, writeMemoRecord } from './testMemoStore.js';
+import { listWorkspaceTests, toRepoPath, toWorkspacePath } from './workspaceTests.js';
+
+const uniqueSorted = (values) => [...new Set(values)].sort();
+const VIRTUAL_MODULE = /(?:^|\/)@vite\/|^\0|^virtual:/;
+
+/**
+ * The runner's selection and memo verdicts for one workspace.
+ *
+ * @param {object} options
+ * @param {'server'|'client'} options.scope
+ * @param {'full'|'files'|'related'} options.mode
+ * @param {string[]} options.repoFiles planner-selected test files (repo-relative)
+ * @param {string[]} options.repoSources changed sources for related mode (repo-relative)
+ * @param {string|null} options.memoDir null when the memo is off
+ * @returns {{ selected: string[], run: string[], skipped: Array<{test: string, via: 'recorded'|'floor'}>,
+ * reasons: Map<string, string>, analyze: (test: string, recorded: string[]) => object, verdicts: Map }}
+ */
+export function planTestMemo({ scope, mode, repoFiles, repoSources, memoDir, repoRoot, env = process.env, platform, arch, nodeMajor, listTests = listWorkspaceTests, readRecord = readMemoRecord, readEntry = readMemoEntry }) {
+ const workspaceTests = listTests(scope, { repoRoot, platform, env }).map((rel) => toRepoPath(scope, rel));
+ const candidates = mode === 'files' ? uniqueSorted(repoFiles) : uniqueSorted([...workspaceTests, ...repoFiles]);
+ const analyzer = createTestMemoAnalyzer({ fs: createDiskFs(repoRoot, nodeFs, nodePath), workspaceRel: scope, env, platform, arch, nodeMajor });
+ const baseClosure = analyzer.workspaceBase(WORKSPACE_BASE_INPUTS[scope]);
+ const analyze = (test, recorded) => analyzer.analyzeTest(test, { baseClosure, recorded });
+
+ const verdicts = new Map();
+ for (const test of candidates) verdicts.set(test, analyze(test, memoDir ? readRecord(memoDir, scope, test) : null));
+
+ const selected = mode === 'related'
+ ? uniqueSorted([...relatedTests(verdicts, repoSources), ...repoFiles])
+ : candidates;
+
+ const skipped = [];
+ const reasons = new Map();
+ const run = [];
+ for (const test of selected) {
+ const info = verdicts.get(test);
+ const via = !memoDir ? null
+ : info.recorded?.memoizable && readEntry(memoDir, info.recorded.key, { test, workspace: scope }) ? 'recorded'
+ : info.floor.memoizable && readEntry(memoDir, info.floor.key, { test, workspace: scope }) ? 'floor'
+ : null;
+ if (via) { skipped.push({ test, via }); continue; }
+ run.push(test);
+ if (!info.floor.memoizable) reasons.set(test, info.recorded?.memoizable ? 'no green entry yet (recorded key)' : info.recorded?.reason ?? info.floor.reason);
+ }
+ return { selected, run, skipped, reasons, analyze, verdicts };
+}
+
+export function memoSummaryLine(plan) {
+ return `🧠 Test memo: skipped ${plan.skipped.length} of ${plan.selected.length} selected files (closure unchanged); running ${plan.run.length}.`;
+}
+
+/**
+ * Write the selection Vitest will read (`PORTOS_CI_TEST_SELECTION`). `include`
+ * null keeps the config's globs (a full run with memo-excluded files).
+ */
+export function writeSelectionFile({ scope, include, exclude }) {
+ const dir = mkdtempSync(join(tmpdir(), 'portos-test-selection-'));
+ const path = join(dir, 'selection.json');
+ writeFileSync(path, JSON.stringify({
+ include: include === null ? null : include.map((test) => toWorkspacePath(scope, test)),
+ exclude: exclude.map((test) => toWorkspacePath(scope, test)),
+ }));
+ return { dir, path };
+}
+
+/**
+ * Per-file outcomes from Vitest's JSON report: `{ file, status, passed, failed, skipped }`.
+ * Any read or schema problem yields an empty list — nothing gets memoized.
+ */
+export function fileResultsFromReport(reportPath) {
+ try {
+ const report = JSON.parse(readFileSync(reportPath, 'utf8'));
+ if (!Array.isArray(report?.testResults)) return [];
+ return report.testResults.map((suite) => {
+ const assertions = Array.isArray(suite.assertionResults) ? suite.assertionResults : [];
+ const count = (status) => assertions.filter((assertion) => assertion?.status === status).length;
+ return { file: String(suite.name ?? ''), status: suite.status, passed: count('passed'), failed: count('failed'), skipped: assertions.length - count('passed') - count('failed') };
+ }).filter((result) => result.file);
+ } catch {
+ return [];
+ }
+}
+
+/**
+ * Files a run proved green: status passed, at least one passed case, no
+ * failed case, and no skipped case unless a name pattern deliberately skips
+ * cases (a browser profile). A capability-gated skip is otherwise invisible
+ * to the key, so a file that skipped anything is not recorded.
+ */
+export function passedTestFiles(fileResults, { repoRoot, namePattern = false }) {
+ const out = [];
+ for (const result of fileResults) {
+ if (result.status !== 'passed' || result.passed < 1 || result.failed > 0) continue;
+ if (result.skipped > 0 && !namePattern) continue;
+ const rel = relative(repoRoot, result.file).split(sep).join('/');
+ if (rel.startsWith('..') || nodePath.isAbsolute(rel)) continue;
+ out.push(rel);
+ }
+ return out;
+}
+
+/**
+ * Vitest's recorded depfile (`<workspace>/.cache/vitest/dependencies.json`):
+ * test -> modules it loaded, both repo-relative. Null when absent or unreadable.
+ */
+export function readVitestDependencies(path, scope) {
+ try {
+ const parsed = JSON.parse(readFileSync(path, 'utf8'));
+ if (parsed?.version !== 1 || typeof parsed.projects !== 'object') return null;
+ const out = new Map();
+ for (const project of Object.values(parsed.projects)) {
+ if (!Array.isArray(project?.modules) || typeof project.files !== 'object') continue;
+ for (const [test, indexes] of Object.entries(project.files)) {
+ if (!Array.isArray(indexes)) continue;
+ // Vite's own virtual modules (`/@vite/client` under happy-dom) are not files.
+ const modules = indexes.map((index) => project.modules[index])
+ .filter((module) => typeof module === 'string' && !VIRTUAL_MODULE.test(module));
+ out.set(toRepoPath(scope, test), modules.map((module) => toRepoPath(scope, module)));
+ }
+ }
+ return out;
+ } catch {
+ return null;
+ }
+}
+
+/** Best-effort commit identity for the entry; never a reason to fail the run. */
+export function currentCommit(repoRoot, env = process.env) {
+ if (env.GITHUB_SHA) return env.GITHUB_SHA;
+ const result = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: repoRoot, encoding: 'utf8' });
+ return result.status === 0 ? result.stdout.trim() : null;
+}
+
+/**
+ * Record green results: a memo record and entries for every passed file the
+ * plan knows. Returns how many entries were written.
+ */
+export function recordTestMemo({ scope, repoRoot, memoDir, plan, passed, dependencies, source, commit, writeEntry = writeMemoEntry, writeRecord = writeMemoRecord }) {
+ let written = 0;
+ for (const test of passed) {
+ const info = plan.verdicts.get(test);
+ if (!info) continue;
+ const meta = { test, workspace: scope, source, commit };
+ if (info.floor.memoizable) { writeEntry(memoDir, info.floor.key, meta); written += 1; }
+ const modules = dependencies?.get(test);
+ if (!modules) continue;
+ const recorded = plan.analyze(test, modules).recorded;
+ if (!recorded?.memoizable) continue;
+ writeRecord(memoDir, scope, test, modules);
+ writeEntry(memoDir, recorded.key, meta);
+ written += 1;
+ }
+ return written;
+}
+
+/**
+ * Vitest writes the depfile under its config-hashed cache directory
+ * (`<workspace>/.cache/vitest/vitest/<hash>/dependencies.json`); the newest
+ * one is this run's. Null when none exists.
+ */
+export function findVitestDependenciesFile(repoRoot, scope) {
+ const root = join(repoRoot, scope, '.cache', 'vitest');
+ let newest = null;
+ const walk = (dir, depth) => {
+ let entries;
+ try { entries = nodeFs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
+ for (const entry of entries) {
+ const path = join(dir, entry.name);
+ if (entry.isDirectory() && depth < 3) walk(path, depth + 1);
+ else if (entry.isFile() && entry.name === 'dependencies.json') {
+ const mtime = nodeFs.statSync(path).mtimeMs;
+ if (!newest || mtime > newest.mtime) newest = { path, mtime };
+ }
+ }
+ };
+ walk(root, 0);
+ return newest?.path ?? null;
+}
diff --git a/scripts/lib/testMemoRunner.test.js b/scripts/lib/testMemoRunner.test.js
new file mode 100644
index 0000000000..63bc6860a8
--- /dev/null
+++ b/scripts/lib/testMemoRunner.test.js
@@ -0,0 +1,128 @@
+/**
+ * Runner-side memo contract over a synthetic repository on disk: memoized
+ * files leave the launch list, related selection follows the closure, and
+ * entries are written only for files a run reported green.
+ */
+
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, describe, expect, it } from 'vitest';
+import {
+ fileResultsFromReport, findVitestDependenciesFile, passedTestFiles, planTestMemo, readVitestDependencies, recordTestMemo,
+ writeSelectionFile,
+} from './testMemoRunner.js';
+import { readMemoEntry, readMemoRecord, writeMemoEntry } from './testMemoStore.js';
+
+const FILES = {
+ 'server/vitest.config.js': 'export default {};\n',
+ 'server/vitest.setup.js': 'export const setup = true;\n',
+ 'scripts/vitestCiPool.js': 'export const vitestCiPool = () => ({});\n',
+ 'server/package.json': '{"name":"server"}',
+ 'server/package-lock.json': '{}',
+ 'server/node_modules/vitest/package.json': '{"name":"vitest","version":"5.0.4"}',
+ 'server/lib/a.js': 'export const a = 1;\n',
+ 'server/lib/b.js': "export const load = () => import('./lazy.js');\n",
+ 'server/lib/lazy.js': 'export const lazy = 1;\n',
+ 'server/lib/a.test.js': "import { it } from 'vitest';\nimport { a } from './a.js';\nit('a', () => {});\n",
+ 'server/lib/b.test.js': "import { it } from 'vitest';\nimport { load } from './b.js';\nit('b', () => {});\n",
+ 'server/lib/c.test.js': "import { it } from 'vitest';\nit('c', () => {});\n",
+};
+const TESTS = ['lib/a.test.js', 'lib/b.test.js', 'lib/c.test.js'];
+
+function writeRepo(root, files) {
+ for (const [rel, content] of Object.entries(files)) {
+ mkdirSync(join(root, rel, '..'), { recursive: true });
+ writeFileSync(join(root, rel), content);
+ }
+}
+
+describe('planTestMemo and recordTestMemo', () => {
+ let root;
+ let memoDir;
+ afterEach(() => { for (const dir of [root, memoDir]) if (dir) rmSync(dir, { recursive: true, force: true }); });
+
+ const plan = (overrides = {}) => planTestMemo({
+ scope: 'server', mode: 'files', repoFiles: TESTS.map((t) => `server/${t}`), repoSources: [], memoDir, repoRoot: root,
+ env: { NODE_ENV: 'test' }, platform: 'linux', arch: 'x64', nodeMajor: 24, listTests: () => TESTS, ...overrides,
+ });
+
+ it('subtracts memoized files, selects related files from the closure, and records only passed files', () => {
+ root = mkdtempSync(join(tmpdir(), 'portos-memo-repo-'));
+ memoDir = mkdtempSync(join(tmpdir(), 'portos-memo-store-'));
+ writeRepo(root, FILES);
+
+ const cold = plan();
+ expect(cold.selected).toEqual(['server/lib/a.test.js', 'server/lib/b.test.js', 'server/lib/c.test.js']);
+ expect(cold.run).toEqual(cold.selected);
+ expect(cold.skipped).toEqual([]);
+
+ const passedReport = {
+ testResults: [
+ { name: join(root, 'server/lib/a.test.js'), status: 'passed', assertionResults: [{ status: 'passed' }] },
+ { name: join(root, 'server/lib/b.test.js'), status: 'failed', assertionResults: [{ status: 'failed' }] },
+ { name: join(root, 'server/lib/c.test.js'), status: 'passed', assertionResults: [{ status: 'passed' }, { status: 'skipped' }] },
+ ],
+ };
+ const reportPath = join(root, 'report.json');
+ writeFileSync(reportPath, JSON.stringify(passedReport));
+ const passed = passedTestFiles(fileResultsFromReport(reportPath), { repoRoot: root });
+ expect(passed).toEqual(['server/lib/a.test.js']);
+ expect(passedTestFiles(fileResultsFromReport(reportPath), { repoRoot: root, namePattern: true })).toEqual(['server/lib/a.test.js', 'server/lib/c.test.js']);
+
+ expect(findVitestDependenciesFile(root, 'server')).toBeNull();
+ const dependenciesPath = join(root, 'server', '.cache', 'vitest', 'vitest', 'confighash', 'dependencies.json');
+ mkdirSync(join(dependenciesPath, '..'), { recursive: true });
+ writeFileSync(dependenciesPath, JSON.stringify({ version: 1, projects: { parallel: {
+ modules: ['lib/a.test.js', 'lib/a.js', 'lib/b.test.js', 'lib/b.js'], globalSetup: [],
+ files: { 'lib/a.test.js': [0, 1], 'lib/b.test.js': [2, 3] },
+ } } }));
+ expect(findVitestDependenciesFile(root, 'server')).toBe(dependenciesPath);
+ const dependencies = readVitestDependencies(dependenciesPath, 'server');
+ expect(readVitestDependencies(null, 'server')).toBeNull();
+ expect(dependencies.get('server/lib/a.test.js')).toEqual(['server/lib/a.test.js', 'server/lib/a.js']);
+
+ const written = recordTestMemo({ scope: 'server', repoRoot: root, memoDir, plan: cold, passed, dependencies, source: 'local', commit: 'abc' });
+ expect(written).toBe(2);
+ expect(readMemoRecord(memoDir, 'server', 'server/lib/a.test.js')).toEqual(['server/lib/a.js', 'server/lib/a.test.js']);
+ expect(readMemoRecord(memoDir, 'server', 'server/lib/b.test.js')).toBeNull();
+ expect(readMemoEntry(memoDir, cold.verdicts.get('server/lib/a.test.js').floor.key, { test: 'server/lib/a.test.js', workspace: 'server' })).toMatchObject({ source: 'local', commit: 'abc' });
+ expect(readMemoEntry(memoDir, cold.verdicts.get('server/lib/b.test.js').floor.key, { test: 'server/lib/b.test.js', workspace: 'server' })).toBeNull();
+
+ const warm = plan();
+ expect(warm.skipped).toEqual([{ test: 'server/lib/a.test.js', via: 'recorded' }]);
+ expect(warm.run).toEqual(['server/lib/b.test.js', 'server/lib/c.test.js']);
+
+ // The recorded key survives a change to a module the green run never loaded.
+ writeFileSync(join(root, 'server/lib/lazy.js'), 'export const lazy = 2;\n');
+ expect(plan().skipped).toEqual([{ test: 'server/lib/a.test.js', via: 'recorded' }]);
+ // A change to a loaded module invalidates it.
+ writeFileSync(join(root, 'server/lib/a.js'), 'export const a = 2;\n');
+ expect(plan().skipped).toEqual([]);
+
+ // Related selection follows static and dynamic edges; explicit files are always kept.
+ const related = plan({ mode: 'related', repoFiles: ['server/lib/c.test.js'], repoSources: ['server/lib/lazy.js'] });
+ expect(related.selected).toEqual(['server/lib/b.test.js', 'server/lib/c.test.js']);
+ expect(plan({ mode: 'related', repoFiles: [], repoSources: ['server/lib/a.js'] }).selected).toEqual(['server/lib/a.test.js']);
+
+ // The memo off: nothing is read, nothing is skipped.
+ writeMemoEntry(memoDir, plan().verdicts.get('server/lib/c.test.js').floor.key, { test: 'server/lib/c.test.js', workspace: 'server', source: 'local' });
+ expect(plan().skipped).toEqual([{ test: 'server/lib/c.test.js', via: 'floor' }]);
+ expect(plan({ memoDir: null }).skipped).toEqual([]);
+ });
+
+ it('writes the selection Vitest reads as workspace-relative paths', () => {
+ const selection = writeSelectionFile({ scope: 'server', include: ['server/lib/a.test.js', 'scripts/x.test.js'], exclude: [] });
+ try {
+ expect(JSON.parse(readFileSync(selection.path, 'utf8'))).toEqual({ include: ['lib/a.test.js', '../scripts/x.test.js'], exclude: [] });
+ } finally {
+ rmSync(selection.dir, { recursive: true, force: true });
+ }
+ const full = writeSelectionFile({ scope: 'client', include: null, exclude: ['client/src/a.test.jsx'] });
+ try {
+ expect(JSON.parse(readFileSync(full.path, 'utf8'))).toEqual({ include: null, exclude: ['src/a.test.jsx'] });
+ } finally {
+ rmSync(full.dir, { recursive: true, force: true });
+ }
+ });
+});
diff --git a/scripts/run-ci-tests.js b/scripts/run-ci-tests.js
index 67637e14e6..9cca0d2c50 100644
--- a/scripts/run-ci-tests.js
+++ b/scripts/run-ci-tests.js
@@ -13,6 +13,11 @@ import { formatErrorAnnotation, writeStepSummary } from './lib/githubOutput.js';
import { reportWindowsCrashDiagnostics } from './ci-windows-crash-diagnostics.js';
import { diagnosticTestFile, persistTestDiagnostic } from './ci-test-diagnostics.js';
import { browserTestProfileArgs, browserTestProfileFiles, assertBrowserProfileExecuted, BROWSER_FIDELITY_ONLY_FILES } from './lib/browserTestProfiles.js';
+import {
+ currentCommit, fileResultsFromReport, findVitestDependenciesFile, memoSummaryLine, passedTestFiles, planTestMemo,
+ readVitestDependencies, recordTestMemo, writeSelectionFile,
+} from './lib/testMemoRunner.js';
+import { resolveMemoDir } from './lib/testMemoStore.js';
const repoRoot = join(dirname(fileURLToPath(import.meta.url)), '..');
@@ -110,10 +115,6 @@ export function toRunnerPath(scope, path) {
return `../${path}`;
}
-export function relatedInputs(sourceFiles, selectedFiles) {
- return [...new Set([...sourceFiles, ...selectedFiles])];
-}
-
/**
* Vitest selector flags for this runner's slice of the selected suite. `CI_SHARD` is
* `<index>/<count>` from the job matrix (ci.yml). A single shard passes nothing,
@@ -209,12 +210,14 @@ export async function runNpm(scope, script, extraArgs, { env = process.env, sign
let cleanupUnverified = false;
try {
writeOwnerFile(reportDir);
- const result = await runNpmWithReport(scope, script, extraArgs, reportPath, env, signal, cancellationTimeoutMs);
- cleanupUnverified = result.cleanupUnverified === true;
+ const launched = await runNpmWithReport(scope, script, extraArgs, reportPath, env, signal, cancellationTimeoutMs);
+ cleanupUnverified = launched.cleanupUnverified === true;
if (cleanupUnverified) {
console.error(`Test launcher cleanup unverified; diagnostics retained at ${reportDir}`);
- return { ...result, diagnosticsPath: reportDir };
+ return { ...launched, diagnosticsPath: reportDir };
}
+ // Per-file outcomes feed the test memo; an interrupted run proves nothing.
+ const result = { ...launched, fileResults: launched.interrupted ? [] : fileResultsFromReport(reportPath) };
const profile = env.PORTOS_BROWSER_TEST_PROFILE;
if (profile !== undefined && result.status === 0) {
try {
@@ -328,7 +331,7 @@ function runNpmWithReport(scope, script, extraArgs, reportPath, env, abortSignal
}
export async function spawnNpm(scope, script, extraArgs, label, {
- run = runNpm, diagnose = reportWindowsCrashDiagnostics,
+ run = runNpm, diagnose = reportWindowsCrashDiagnostics, onResult = () => {},
} = {}) {
console.log(`Running ${scope} ${label}${extraArgs.length ? ` (${extraArgs.length} selector argument(s))` : ''}.`);
const startedAt = Date.now();
@@ -336,6 +339,7 @@ export async function spawnNpm(scope, script, extraArgs, label, {
? browserTestProfileArgs(process.env.PORTOS_BROWSER_TEST_PROFILE) : [];
const result = await run(scope, script, [...extraArgs, ...profileArgs]);
recordVitestDuration(scope, label, startedAt);
+ onResult(result);
if (result.error) {
console.error(result.error.message);
return 1;
@@ -360,6 +364,7 @@ export async function spawnNpm(scope, script, extraArgs, label, {
const retryStartedAt = Date.now();
const retry = await run(scope, script, [selector, ...profileArgs]);
recordVitestDuration(scope, `crash retry: ${relPath}`, retryStartedAt);
+ onResult(retry);
if (!retry.error && retry.status === 0) {
if (hasOtherTestFailures(result.output)) {
// The crash retry only clears the crash itself — a real assertion
@@ -396,7 +401,6 @@ async function main() {
? browserTestProfileFiles(process.env.PORTOS_BROWSER_TEST_PROFILE, inputFiles) : inputFiles;
const repoSources = JSON.parse(process.env.CI_TEST_SOURCES || '[]');
const selectedFiles = repoFiles.map((path) => toRunnerPath(scope, path));
- const sourceFiles = repoSources.map((path) => toRunnerPath(scope, path));
if (!['full', 'files', 'related'].includes(mode)) {
console.error(`Unsupported CI test mode: ${mode}`);
@@ -408,50 +412,72 @@ async function main() {
process.exit(2);
}
- if (mode === 'full') {
- const shard = shardArgs(process.env.CI_SHARD);
- const label = shard.length ? `full suite shard ${process.env.CI_SHARD}` : 'full suite';
- // process.exitCode, not process.exit(): the runner now streams the
- // child's output through this process's own stdout/stderr, and
- // process.exit() can terminate before those writes (and this function's
- // own diagnostic lines) finish flushing. Setting exitCode lets Node drain
- // normally once main() returns.
+ if (mode === 'files' && selectedFiles.length === 0) {
+ const profile = process.env.PORTOS_BROWSER_TEST_PROFILE;
+ const deferredOnly = profile === 'smoke' && inputFiles.length > 0
+ && inputFiles.every(file => BROWSER_FIDELITY_ONLY_FILES.includes(file));
+ if (profile !== undefined && !deferredOnly) throw Error('Browser profile selected no executable cases.');
+ console.log(`No ${scope} tests selected.${deferredOnly ? ' Fidelity-only inputs require separate evidence.' : ''}`);
+ process.exitCode = 0;
+ return;
+ }
+
+ const shard = shardArgs(process.env.CI_SHARD);
+ const label = mode === 'full' ? (shard.length ? `full suite shard ${process.env.CI_SHARD}` : 'full suite')
+ : mode === 'files' ? 'selected tests' : 'related and contract tests';
+ // process.exitCode, not process.exit(): the runner streams the child's
+ // output through this process's own stdout/stderr, and process.exit() can
+ // terminate before those writes (and this function's own diagnostic lines)
+ // finish flushing. Setting exitCode lets Node drain normally once main()
+ // returns.
+ const memoDir = resolveMemoDir();
+ if (mode === 'full' && !memoDir) {
process.exitCode = await spawnNpm(scope, 'test:ci', shard, label);
return;
}
- if (mode === 'files') {
- if (selectedFiles.length === 0) {
- const profile = process.env.PORTOS_BROWSER_TEST_PROFILE;
- const deferredOnly = profile === 'smoke' && inputFiles.length > 0
- && inputFiles.every(file => BROWSER_FIDELITY_ONLY_FILES.includes(file));
- if (profile !== undefined && !deferredOnly) throw Error('Browser profile selected no executable cases.');
- console.log(`No ${scope} tests selected.${deferredOnly ? ' Fidelity-only inputs require separate evidence.' : ''}`);
- process.exitCode = 0;
- return;
+ // The selection is closure-based (scripts/lib/testMemo.js): in related
+ // mode a test runs when a changed source is in its module closure or when
+ // its edges cannot all be resolved, plus the planner's explicit files.
+ // Vitest used to resolve `related` itself from CI_TEST_SOURCES; computing
+ // the set here is what lets the memo subtract the files a trusted green
+ // run already verified, and the list reaches Vitest as a file because a
+ // related set can exceed the Windows command-line limit.
+ const plan = planTestMemo({ scope, mode, repoFiles, repoSources, memoDir, repoRoot });
+ if (memoDir) {
+ console.log(memoSummaryLine(plan));
+ if (process.env.PORTOS_TEST_MEMO_VERBOSE === '1') {
+ for (const { test, via } of plan.skipped) console.log(`🧠 skipped ${test} (${via} key)`);
}
- // The planner fans an exact-file plan out only when it selects more files
- // than the shard count (ci-test-plan.js fansOut), so the slice is never empty.
- process.exitCode = await spawnNpm(scope, 'test:ci', [...selectedFiles, ...shardArgs(process.env.CI_SHARD)], 'selected tests');
+ }
+ if (plan.run.length === 0) {
+ console.log(`No ${scope} tests left to run.`);
+ process.exitCode = 0;
return;
}
-
- // Feed Vitest the actual changed source files instead of asking `list
- // --changed` to print every individual test name into a buffered subprocess.
- // The old discovery pass took 282 seconds on PR #5296, overflowed Node's
- // spawnSync buffer, discarded its work, then reran the same graph. `related`
- // builds that graph once and immediately executes it.
- // A test file passed to `vitest related` is itself selected, so changed tests
- // and structural guards can share the source graph's one Vitest invocation.
- // Running them in a second exact-file process repeated any changed test that
- // already imported the source; on PR #5299 that rebuilt the atlas twice and
- // added 27.5 seconds after the related run had already passed it.
- process.exitCode = await spawnNpm(
- scope,
- 'test:ci:related',
- [...relatedInputs(sourceFiles, selectedFiles), ...shardArgs(process.env.CI_SHARD)],
- 'related and contract tests',
- );
+ const skipped = plan.skipped.map(({ test }) => test);
+ const selection = writeSelectionFile({ scope, include: mode === 'full' ? null : plan.run, exclude: mode === 'full' ? skipped : [] });
+ process.env.PORTOS_CI_TEST_SELECTION = selection.path;
+ // A memo-trimmed selection can leave one shard of the matrix empty, which
+ // Vitest otherwise reports as a failure.
+ const extraArgs = [...shard, ...(skipped.length ? ['--passWithNoTests'] : []), ...(memoDir ? ['--experimental.recordDependencies'] : [])];
+ const passed = [];
+ try {
+ process.exitCode = await spawnNpm(scope, 'test:ci', extraArgs, label, {
+ onResult: (result) => passed.push(...passedTestFiles(result.fileResults ?? [], {
+ repoRoot, namePattern: process.env.PORTOS_BROWSER_TEST_PROFILE !== undefined,
+ })),
+ });
+ } finally {
+ rmSync(selection.dir, { recursive: true, force: true });
+ }
+ if (!memoDir || passed.length === 0) return;
+ const written = recordTestMemo({
+ scope, repoRoot, memoDir, plan, passed: [...new Set(passed)],
+ dependencies: readVitestDependencies(findVitestDependenciesFile(repoRoot, scope), scope),
+ source: process.env.CI ? 'ci' : 'local', commit: currentCommit(repoRoot),
+ });
+ console.log(`🧠 Test memo: recorded ${written} green entries for ${new Set(passed).size} passed files.`);
}
if (isDirectlyInvoked(import.meta.url)) await main();
diff --git a/scripts/run-ci-tests.test.js b/scripts/run-ci-tests.test.js
index 0c76022d50..cf2d73c7c0 100644
--- a/scripts/run-ci-tests.test.js
+++ b/scripts/run-ci-tests.test.js
@@ -13,7 +13,6 @@ import {
hasOtherTestFailures,
planCrashRetry,
recordVitestDuration,
- relatedInputs,
repoRelativeFromCrashPath,
requiresSourceFiles,
shardArgs,
@@ -21,6 +20,9 @@ import {
toRunnerPath,
} from './run-ci-tests.js';
import { killProcessTree } from '../server/lib/bufferedSpawn.js';
+import * as nodeFs from 'node:fs';
+import * as nodePath from 'node:path';
+import { computeTestMemoKeys, createDiskFs, relatedTests } from './lib/testMemo.js';
import { OWNER_FILE, STALE_ROOT_AGE_MS } from './lib/vitestStaleRunRoots.js';
import { workflowJobs } from './lib/workflowJobs.js';
@@ -55,37 +57,130 @@ describe('shardArgs', () => {
});
});
-it('forwards the scoped matrix slice and all selectors through the real launcher', () => {
+/** A fake `npm` on PATH that prints its arguments and the selection file it was handed. */
+function installFakeNpm(fixture, body) {
+ const npmScript = join(fixture, 'npm.js');
+ writeFileSync(npmScript, body);
+ writeFileSync(join(fixture, 'npm'), `#!${process.execPath}\n${body}`, { mode: 0o755 });
+ writeFileSync(join(fixture, 'npm.cmd'), `@"${process.execPath}" "${npmScript}" %*\r\n`);
+}
+
+const REPORTING_NPM = 'process.stdout.write(JSON.stringify({ args: process.argv.slice(2), '
+ + 'selection: JSON.parse(require("fs").readFileSync(process.env.PORTOS_CI_TEST_SELECTION, "utf8")) }) + "\\n");';
+
+it('forwards the scoped matrix slice as an argument and the selection as a file through the real launcher', () => {
const fixture = mkdtempSync(join(tmpdir(), 'portos-related-shard-'));
try {
- const npmScript = join(fixture, 'npm.js');
- writeFileSync(npmScript, 'process.stdout.write(JSON.stringify(process.argv.slice(2)) + "\\n");');
- writeFileSync(join(fixture, 'npm'), `#!${process.execPath}\n`
- + 'process.stdout.write(JSON.stringify(process.argv.slice(2)) + "\\n");', { mode: 0o755 });
- writeFileSync(join(fixture, 'npm.cmd'), `@"${process.execPath}" "${npmScript}" %*\r\n`);
+ installFakeNpm(fixture, REPORTING_NPM);
// The planner fans both scoped modes out (ci-test-plan.js fansOut), so
// each must hand Vitest its slice rather than run the whole selection twice.
- for (const [mode, script, sources, expected] of [
- ['related', 'test:ci:related', ['server/lib/example.js'], ['./lib/example.js']],
- ['files', 'test:ci', [], []],
+ // Related selection is closure-based: this runner imports
+ // scripts/lib/directInvocation.js, so the runner's own test is related to it.
+ for (const [mode, sources, expected] of [
+ ['related', ['scripts/lib/directInvocation.js'], ['../scripts/run-ci-tests.test.js']],
+ ['files', [], []],
]) {
const result = spawnSync(process.execPath, [join(import.meta.dirname, 'run-ci-tests.js'), 'server'], {
- encoding: 'utf8', timeout: 30000,
+ encoding: 'utf8', timeout: 120000,
env: { ...process.env, PATH: `${fixture}${delimiter}${process.env.PATH}`,
- CI_TEST_MODE: mode, CI_SHARD: '2/3',
+ CI_TEST_MODE: mode, CI_SHARD: '2/3', PORTOS_TEST_MEMO: 'off',
CI_TEST_SOURCES: JSON.stringify(sources),
CI_TEST_FILES: JSON.stringify(['server/lib/example.test.js', 'scripts/example.test.js']),
GITHUB_ENV: '', GITHUB_STEP_SUMMARY: '' },
});
expect(result.error).toBeUndefined();
expect(result.status, result.stderr).toBe(0);
- const args = JSON.parse(result.stdout.split('\n').find(line => line.startsWith('["run"')));
- const selectors = [...expected, './lib/example.test.js', '../scripts/example.test.js', '--shard=2/3'];
- expect(args.slice(0, 5 + selectors.length), mode).toEqual(['run', script, '--prefix', 'server', '--', ...selectors]);
+ const { args, selection } = JSON.parse(result.stdout.split('\n').find(line => line.startsWith('{"args"')));
+ expect(args.slice(0, 6), mode).toEqual(['run', 'test:ci', '--prefix', 'server', '--', '--shard=2/3']);
+ expect(args, mode).not.toContain('--experimental.recordDependencies');
+ expect(selection.exclude, mode).toEqual([]);
+ expect(selection.include, mode).toEqual(expect.arrayContaining([...expected, 'lib/example.test.js', '../scripts/example.test.js']));
+ if (mode === 'files') expect(selection.include).toHaveLength(2);
}
} finally {
rmSync(fixture, { recursive: true, force: true });
}
+}, 150_000);
+
+describe('test memo through the real runner', () => {
+ it('runs a selected file once, then skips it while its inputs are unchanged, and runs it again when they change', () => {
+ const fixture = mkdtempSync(join(tmpdir(), 'portos-memo-e2e-'));
+ const memoDir = join(fixture, 'memo');
+ try {
+ // A fake npm that reports every selected file as passed, so the contract
+ // covers the runner's bookkeeping without a nested Vitest run.
+ installFakeNpm(fixture, [
+ 'const fs = require("fs");',
+ 'const selection = JSON.parse(fs.readFileSync(process.env.PORTOS_CI_TEST_SELECTION, "utf8"));',
+ 'const out = process.argv.find(a => a.startsWith("--outputFile.json=")).slice("--outputFile.json=".length);',
+ `const repo = ${JSON.stringify(join(import.meta.dirname, '..'))};`,
+ 'const testResults = selection.include.map(rel => ({ name: require("path").resolve(repo, "server", rel), status: "passed", assertionResults: [{ status: "passed" }] }));',
+ 'fs.writeFileSync(out, JSON.stringify({ testResults }));',
+ 'process.stdout.write("RAN " + JSON.stringify(selection.include) + "\\n");',
+ ].join('\n'));
+ const test = 'scripts/lib/releaseTitle.test.js';
+ const run = (extraEnv = {}) => spawnSync(process.execPath, [join(import.meta.dirname, 'run-ci-tests.js'), 'server'], {
+ encoding: 'utf8', timeout: 120000,
+ env: { ...process.env, PATH: `${fixture}${delimiter}${process.env.PATH}`,
+ CI_TEST_MODE: 'files', CI_TEST_FILES: JSON.stringify([test]), CI_TEST_SOURCES: '[]',
+ PORTOS_TEST_MEMO_DIR: memoDir, PORTOS_TEST_MEMO_VERBOSE: '1', GITHUB_ENV: '', GITHUB_STEP_SUMMARY: '', ...extraEnv },
+ });
+ const first = run();
+ expect(first.status, first.stderr).toBe(0);
+ expect(first.stdout).toContain('🧠 Test memo: skipped 0 of 1 selected files (closure unchanged); running 1.');
+ expect(first.stdout).toContain('RAN ["../scripts/lib/releaseTitle.test.js"]');
+ // Two entries when this checkout's Vitest cache already holds a depfile for the file (a recorded key), one otherwise.
+ expect(first.stdout).toMatch(/recorded [12] green entries for 1 passed files/);
+
+ const second = run();
+ expect(second.status, second.stderr).toBe(0);
+ expect(second.stdout).toContain('🧠 Test memo: skipped 1 of 1 selected files (closure unchanged); running 0.');
+ expect(second.stdout).toMatch(new RegExp(`skipped ${test.replace(/[./]/g, '\\$&')} \\((?:recorded|floor) key\\)`));
+ expect(second.stdout).not.toContain('RAN ');
+
+ // A gating environment variable is part of the key.
+ const other = run({ PORTOS_TEST_QUIET: process.env.PORTOS_TEST_QUIET === '1' ? '' : '1' });
+ expect(other.stdout).toContain('skipped 0 of 1 selected files');
+ // Off restores the pre-memo behavior: no summary, the file runs.
+ const off = run({ PORTOS_TEST_MEMO: 'off' });
+ expect(off.stdout).not.toContain('🧠');
+ expect(off.stdout).toContain('RAN ');
+ } finally {
+ rmSync(fixture, { recursive: true, force: true });
+ }
+ }, 200_000);
+
+ it('selects the same files as `vitest related` on a fixture graph with a dynamic import', () => {
+ const fixture = mkdtempSync(join(tmpdir(), 'portos-related-parity-'));
+ try {
+ const host = join(fixture, 'host');
+ mkdirSync(host);
+ writeFileSync(join(fixture, 'vitest.config.mjs'), FIXTURE_CONFIG);
+ writeFileSync(join(fixture, 'a.js'), 'export const a = 1;\n');
+ writeFileSync(join(fixture, 'b.js'), "export const load = () => import('./c.js');\n");
+ writeFileSync(join(fixture, 'c.js'), 'export const c = 1;\n');
+ writeFileSync(join(fixture, 'a.test.js'), "import { a } from './a.js';\ntest('a', () => { expect(a).toBe(1); });\n");
+ writeFileSync(join(fixture, 'b.test.js'), "import { load } from './b.js';\ntest('b', async () => { expect((await load()).c).toBe(1); });\n");
+ writeFileSync(join(fixture, 'plain.test.js'), "test('plain', () => { expect(1).toBe(1); });\n");
+ const vitest = join(import.meta.dirname, '..', 'server', 'node_modules', '.bin', 'vitest');
+ const env = { ...process.env, NODE_ENV: 'test', TMPDIR: host, TMP: host, TEMP: host, NODE_DISABLE_COMPILE_CACHE: '1', PORTOS_CI_TEST_SELECTION: '' };
+ delete env.PORTOS_TEST_TEMP_ROOT;
+ const viaVitest = (source) => {
+ const report = join(fixture, `${source}.json`);
+ const result = spawnSync(vitest, ['related', `./${source}`, '--run', '--config', join(fixture, 'vitest.config.mjs'), '--root', fixture,
+ '--reporter=json', `--outputFile=${report}`, '--passWithNoTests'], { cwd: fixture, encoding: 'utf8', env, timeout: 60000 });
+ expect(result.error).toBeUndefined();
+ return JSON.parse(readFileSync(report, 'utf8')).testResults.map(suite => suite.name.replace(/\\/g, '/').split('/').pop()).sort();
+ };
+ const keys = computeTestMemoKeys(['a.test.js', 'b.test.js', 'plain.test.js'], { fs: createDiskFs(fixture, nodeFs, nodePath), workspace: '.', env: {} });
+ for (const source of ['a.js', 'b.js', 'c.js']) {
+ expect(relatedTests(keys, [source]).sort(), source).toEqual(viaVitest(source));
+ }
+ expect(relatedTests(keys, ['c.js'])).toEqual(['b.test.js']);
+ } finally {
+ rmSync(fixture, { recursive: true, force: true });
+ }
+ }, 120_000);
});
it('rejects empty browser coverage except explicitly deferred smoke-only inputs', () => {
@@ -338,19 +433,6 @@ describe('hasOtherTestFailures', () => {
});
});
-describe('relatedInputs', () => {
- it('runs source-related tests and explicit guards in one deduplicated invocation', () => {
- expect(relatedInputs(
- ['./services/auth.js', './services/auth.test.js'],
- ['./services/auth.test.js', '../scripts/repo-scan-guards.test.js'],
- )).toEqual([
- './services/auth.js',
- './services/auth.test.js',
- '../scripts/repo-scan-guards.test.js',
- ]);
- });
-});
-
describe('native crash diagnostics before retry/cancellation (#8152)', () => {
const output = 'Worker exited unexpectedly with exit code 3221226505 during started state while running test file '
+ 'D:/a/PortOS/PortOS/server/services/sprites/walk.test.js\n'
@@ -438,7 +520,7 @@ describe('structured failure diagnostics', () => {
delete env.PORTOS_TEST_TEMP_ROOT;
const result = spawnSync(process.execPath, ['--input-type=module', '-e',
`import { runNpm } from ${JSON.stringify(runnerUrl)};
- const result = await runNpm('server', 'test:ci:related', ${JSON.stringify(args)});
+ const result = await runNpm('server', 'test:ci', ${JSON.stringify(args)});
process.exitCode = result.status;`,
], { encoding: 'utf8', timeout: 30_000, env });
expect(result.error).toBeUndefined();
@@ -476,7 +558,7 @@ describe('structured failure diagnostics', () => {
delete env.PORTOS_TEST_TEMP_ROOT;
const result = spawnSync(process.execPath, ['--input-type=module', '-e',
`import { runNpm } from ${JSON.stringify(runnerUrl)};
- const result = await runNpm('server', 'test:ci:related', ${JSON.stringify(args)});
+ const result = await runNpm('server', 'test:ci', ${JSON.stringify(args)});
process.exitCode = result.status;`,
], { encoding: 'utf8', timeout: 30_000, env });
expect(result.error).toBeUndefined();
@@ -528,7 +610,7 @@ describe('report launcher subprocess lifecycle (#9804)', () => {
try {
const { runNpm } = await import('./run-ci-tests.js');
const controller = new AbortController();
- const pending = runNpm('server', 'test:ci:related', [], {
+ const pending = runNpm('server', 'test:ci', [], {
env: { ...process.env, PORTOS_CI_DIAGNOSTICS_DIR: '' },
signal: controller.signal, cancellationTimeoutMs: 100,
});
@@ -581,7 +663,7 @@ describe('report launcher subprocess lifecycle (#9804)', () => {
import { runNpm } from ${JSON.stringify(runnerUrl)};
const controller = new AbortController();
if (${abortOnSignal}) process.once('SIGUSR2', () => controller.abort());
- const result = await runNpm('server', 'test:ci:related', ${JSON.stringify(args)},
+ const result = await runNpm('server', 'test:ci', ${JSON.stringify(args)},
${abortOnSignal} ? { signal: controller.signal } : {});
process.exitCode = result.status;`], { env, stdio: ['ignore', 'pipe', 'pipe'] });
let output = '';
diff --git a/server/package.json b/server/package.json
index fc9bf17174..bcb54f8c15 100644
--- a/server/package.json
+++ b/server/package.json
@@ -15,7 +15,6 @@
"test:fast": "vitest run --configLoader runner",
"test:watch": "vitest --configLoader runner",
"test:ci": "vitest run --bail=1 --silent=passed-only --reporter=default --reporter=github-actions --configLoader runner",
- "test:ci:related": "vitest related --run --bail=1 --silent=passed-only --reporter=default --reporter=github-actions --configLoader runner",
"test:coverage": "vitest run --coverage --configLoader runner",
"setup:db:test": "node scripts/setup-db-test.mjs",
"test:db": "vitest run --config vitest.config.db.js --configLoader runner",