Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: reuse dependency snapshots during CI planning (#13275)

[._.]/ Adam Eivy2e62eb52eb4a
diff --git a/scripts/ci-test-plan.js b/scripts/ci-test-plan.js
index da1e4d35dc..3a67ea3d3f 100644
--- a/scripts/ci-test-plan.js
+++ b/scripts/ci-test-plan.js
@@ -1526,48 +1526,85 @@ export function forceFullReasonFor({ forceFull, baseRef }) {
return null;
}
-// Start from every browser source, including lazy-loaded pages, then follow
-// relative static imports and re-exports across directory boundaries. Parsing
-// is shared with the structural graph guards, but unresolved edges must widen
-// CI instead of disappearing as they do in that helper's negative graph walk.
-function collectClientDependencies(trackedFiles, cwd) {
- const known = new Set(trackedFiles);
- const pending = trackedFiles.filter((path) => path.startsWith('client/src/')
- && MODULE_SOURCE_RE.test(path) && !isTestFile(path));
- const seen = new Set();
+// A collection owns its snapshot: each source is read and each prerequisite
+// edge is resolved once, even when many browser suites reach the same module.
+// Never retain this across collections: local pregate must see the next edit.
+function createDependencySnapshot(files, cwd) {
+ const known = new Set(files);
+ const sources = new Map();
+ const staticEdges = new Map();
+ const testEdges = new Map();
const extensions = ['.js', '.jsx', '.ts', '.tsx', '.mjs', '.cjs', '.mts', '.cts', '.json'];
- try {
- for (let index = 0; index < pending.length; index++) {
- const file = pending[index];
- if (seen.has(file)) continue;
- seen.add(file);
- if (!MODULE_SOURCE_RE.test(file)) continue;
- for (const specifier of staticImportSpecifiersFromSource(readFileSync(join(cwd, file), 'utf8'))) {
- if (!specifier.startsWith('.')) continue;
- const path = posix.normalize(posix.join(posix.dirname(file), specifier.split(/[?#]/)[0]));
- // Keep every resolvable candidate when extensions collide rather than
- // guessing a bundler-specific precedence and dropping a real consumer.
- const dependencies = [path, ...extensions.map((extension) => `${path}${extension}`),
- ...extensions.map((extension) => `${path}/index${extension}`)].filter((candidate) => known.has(candidate));
- if (dependencies.length === 0) return null;
- pending.push(...dependencies);
+ const readText = (file) => {
+ if (!sources.has(file)) {
+ try {
+ sources.set(file, { text: readFileSync(join(cwd, file), 'utf8') });
+ } catch (error) {
+ sources.set(file, { error });
}
}
- } catch {
- return null;
- }
- return uniqueSorted([...seen].filter((path) => path.startsWith('server/lib/')));
+ const source = sources.get(file);
+ if (source.error) throw source.error;
+ return source.text;
+ };
+ const resolve = (file, specifiers) => {
+ const dependencies = [];
+ let inconclusive = false;
+ for (const specifier of specifiers) {
+ if (!specifier.startsWith('.')) continue;
+ const path = posix.normalize(posix.join(posix.dirname(file), specifier.split(/[?#]/)[0]));
+ // Preserve all candidates when extensions collide, not a guessed bundler precedence.
+ const candidates = [path, ...extensions.map((extension) => `${path}${extension}`),
+ ...extensions.map((extension) => `${path}/index${extension}`)].filter((candidate) => known.has(candidate));
+ if (candidates.length === 0) inconclusive = true;
+ dependencies.push(...candidates);
+ }
+ return { dependencies, inconclusive };
+ };
+ const staticDependencies = (file) => {
+ if (!staticEdges.has(file)) {
+ staticEdges.set(file, resolve(file, staticImportSpecifiersFromSource(readText(file))));
+ }
+ return staticEdges.get(file);
+ };
+ const testDependencies = (file) => {
+ if (!testEdges.has(file)) {
+ const deferred = [];
+ let computed = false;
+ // Nonliterals and extra matches in comments widen prerequisites safely.
+ for (const match of readText(file).matchAll(/\bimport\s*\(\s*([^)]*)\)/g)) {
+ const literal = match[1].trim().match(/^(['"`])([^'"`]*?)\1\s*(?:,.*)?$/s);
+ if (!literal || literal[2].includes('${')) computed = true;
+ else deferred.push(literal[2]);
+ }
+ const statics = staticDependencies(file);
+ const dynamic = resolve(file, deferred);
+ testEdges.set(file, {
+ dependencies: [...statics.dependencies, ...dynamic.dependencies],
+ inconclusive: computed || statics.inconclusive || dynamic.inconclusive,
+ });
+ }
+ return testEdges.get(file);
+ };
+ return { readText, staticDependencies, testDependencies };
+}
+
+// Start from every browser source, including lazy-loaded pages, then follow
+// relative static imports and re-exports across directory boundaries. Unresolved
+// edges must widen CI rather than disappearing from a negative graph walk.
+function collectClientDependencies(files, snapshot) {
+ const entries = files.filter((path) => path.startsWith('client/src/')
+ && MODULE_SOURCE_RE.test(path) && !isTestFile(path));
+ const closure = collectTestDependencies(entries, snapshot.staticDependencies);
+ return closure?.filter((path) => path.startsWith('server/lib/')) ?? null;
}
// Project DB and browser consumers without importing test code or Vitest (the
-// planner runs before npm installation). Include deferred literals; unreadable,
+// planner runs before npm installation). Deferred literals participate; unreadable,
// unresolved and computed imports retain conservative prerequisite coverage.
-// Share parsed edges between suite walks so common dependencies are read once.
-function collectTestDependencies(entries, files, cwd, importCache) {
- const known = new Set(files);
+function collectTestDependencies(entries, dependenciesFor) {
const pending = [...entries];
const seen = new Set();
- const extensions = ['.js', '.jsx', '.ts', '.tsx', '.mjs', '.cjs', '.mts', '.cts', '.json'];
let inconclusive = false;
try {
for (let index = 0; index < pending.length; index++) {
@@ -1575,29 +1612,9 @@ function collectTestDependencies(entries, files, cwd, importCache) {
if (seen.has(file)) continue;
seen.add(file);
if (!MODULE_SOURCE_RE.test(file)) continue;
- if (!importCache.has(file)) {
- const source = readFileSync(join(cwd, file), 'utf8');
- const specifiers = staticImportSpecifiersFromSource(source);
- let computed = false;
- // A nonliteral cannot prove absence of a dependency; extra matches in
- // comments widen safely. Template literals resolve without interpolation.
- for (const match of source.matchAll(/\bimport\s*\(\s*([^)]*)\)/g)) {
- const literal = match[1].trim().match(/^(['"`])([^'"`]*?)\1\s*(?:,.*)?$/s);
- if (!literal || literal[2].includes('${')) computed = true;
- else specifiers.push(literal[2]);
- }
- importCache.set(file, { specifiers, computed });
- }
- const { specifiers, computed } = importCache.get(file);
- if (computed) inconclusive = true;
- for (const specifier of specifiers) {
- if (!specifier.startsWith('.')) continue;
- const path = posix.normalize(posix.join(posix.dirname(file), specifier.split(/[?#]/)[0]));
- const dependencies = [path, ...extensions.map((extension) => `${path}${extension}`),
- ...extensions.map((extension) => `${path}/index${extension}`)].filter((candidate) => known.has(candidate));
- if (dependencies.length === 0) inconclusive = true;
- pending.push(...dependencies);
- }
+ const edges = dependenciesFor(file);
+ if (edges.inconclusive) inconclusive = true;
+ pending.push(...edges.dependencies);
}
} catch {
return null;
@@ -1665,21 +1682,21 @@ export function collectPlanInputs({ baseSha, forceFull = false, changedFiles, cw
return [path, sharedBasenames.has(basenameOf(path)) ? narrowSharedBasenameHits(path, hits, cwd) : hits];
}));
+ const dependencyFiles = hasChangedFilesOverride ? uniqueSorted([...presentFiles, ...untrackedFiles]) : presentFiles;
+ const snapshot = createDependencySnapshot(dependencyFiles, cwd);
const clientDependencies = collectedChangedFiles.some((path) => path.startsWith('server/lib/')
&& isExecutable(path) && !isTestFile(path))
- ? collectClientDependencies(hasChangedFilesOverride ? uniqueSorted([...presentFiles, ...untrackedFiles]) : presentFiles, cwd)
+ ? collectClientDependencies(dependencyFiles, snapshot)
: [];
const hasRelatedServerSource = collectedChangedFiles.some((path) => isServerRunnerFile(path)
&& MODULE_SOURCE_RE.test(path) && !isTestFile(path) && !isStructuralBarrel(path));
- const dependencyFiles = hasChangedFilesOverride ? uniqueSorted([...presentFiles, ...untrackedFiles]) : presentFiles;
- const importCache = new Map();
const dbDependencies = hasRelatedServerSource
- ? collectTestDependencies(resolveDbTestFiles(dependencyFiles), dependencyFiles, cwd, importCache)
+ ? collectTestDependencies(resolveDbTestFiles(dependencyFiles), snapshot.testDependencies)
: [];
const browserDependencies = hasRelatedServerSource
? Object.fromEntries(browserSuitesIn(presentFiles).map((suite) => [suite,
- collectTestDependencies([suite], dependencyFiles, cwd, importCache)]))
+ collectTestDependencies([suite], snapshot.testDependencies)]))
: {};
// Each changed manifest at the diff base vs the tree under test (the working
@@ -1713,6 +1730,7 @@ export function collectPlanInputs({ baseSha, forceFull = false, changedFiles, cw
serverReach = measureRepoImportReach({
cwd,
files: dependencyFiles,
+ readText: snapshot.readText,
sources: collectedChangedFiles.filter((path) => isServerRunnerFile(path)
&& MODULE_SOURCE_RE.test(path) && !isTestFile(path)),
});
diff --git a/scripts/ci-test-plan.test.js b/scripts/ci-test-plan.test.js
index 8e45c0caa5..9fcc4840d4 100644
--- a/scripts/ci-test-plan.test.js
+++ b/scripts/ci-test-plan.test.js
@@ -410,6 +410,78 @@ it('resolves local client imports of present untracked modules and still fails c
}
});
+it('shares source reads within one collection and observes edits and unreadable sources in the next', () => {
+ const root = mkdtempSync(join(tmpdir(), 'portos-ci-snapshot-'));
+ const planner = new URL('./ci-test-plan.js', import.meta.url).href;
+ const git = (...args) => execFileSync('git', args, { cwd: root, encoding: 'utf8' }).trim();
+ const write = (path, body) => {
+ mkdirSync(dirname(join(root, path)), { recursive: true });
+ writeFileSync(join(root, path), body);
+ };
+ const leaf = 'server/lib/first.js';
+ const second = 'server/lib/second.js';
+ const bridge = 'server/lib/bridge.js';
+ const browser = BROWSER_SUITES[0];
+ const db = 'server/routes/decks.db.test.js';
+ try {
+ git('init', '-q');
+ write(leaf, 'export const value = 1;\n');
+ write(second, 'export const value = 2;\n');
+ write(bridge, "export { value } from './first.js';\n");
+ for (const entry of ['client/src/consumer.js', browser, db, 'server/lib/consumer.test.js']) {
+ const relative = posix.relative(posix.dirname(entry), bridge);
+ write(entry, `import '${relative.startsWith('.') ? relative : `./${relative}`}';\n`);
+ }
+ git('add', '--all');
+
+ // A subprocess observes actual disk reads without changing Vitest's own fs.
+ // This catches repeated reads across client, DB, browser and reach scans,
+ // while the same process also proves no snapshot survives the next call.
+ const result = JSON.parse(execFileSync(process.execPath, ['--input-type=module', '-e', `
+ import fs from 'node:fs';
+ import { syncBuiltinESMExports } from 'node:module';
+ import { join } from 'node:path';
+ const original = fs.readFileSync;
+ const reads = new Map();
+ fs.readFileSync = function(path, ...args) {
+ reads.set(String(path), (reads.get(String(path)) ?? 0) + 1);
+ return original.call(this, path, ...args);
+ };
+ syncBuiltinESMExports();
+ const fixtureRoot = ${JSON.stringify(root)};
+ const { collectPlanInputs } = await import(${JSON.stringify(planner)});
+ const collect = () => collectPlanInputs({ baseSha: 'HEAD', changedFiles: [${JSON.stringify(leaf)}], cwd: fixtureRoot });
+ const first = collect();
+ const bridgeReads = reads.get(join(fixtureRoot, ${JSON.stringify(bridge)}));
+ fs.writeFileSync(${JSON.stringify(bridge)}, "export { value } from './second.js';\\n");
+ const edited = collect();
+ fs.unlinkSync(${JSON.stringify(bridge)});
+ fs.mkdirSync(${JSON.stringify(bridge)});
+ const warn = console.warn;
+ console.warn = () => {};
+ const unreadable = collect();
+ console.warn = warn;
+ console.log(JSON.stringify({ first, edited, unreadable, bridgeReads }));
+ `], { cwd: root, encoding: 'utf8' }));
+ expect(result.bridgeReads).toBe(1);
+ for (const [inputs, present, absent] of [[result.first, leaf, second], [result.edited, second, leaf]]) {
+ expect(inputs.clientDependencies).toContain(present);
+ expect(inputs.clientDependencies).not.toContain(absent);
+ expect(inputs.dbDependencies).toContain(present);
+ expect(inputs.dbDependencies).not.toContain(absent);
+ expect(inputs.browserDependencies[browser]).toContain(present);
+ expect(inputs.browserDependencies[browser]).not.toContain(absent);
+ }
+ expect(result.first.serverReach.selected).toBeGreaterThan(0);
+ expect(result.edited.serverReach.selected).toBe(0);
+ expect(result.unreadable.clientDependencies).toBeNull();
+ expect(result.unreadable.dbDependencies).toBeNull();
+ expect(result.unreadable.browserDependencies[browser]).toBeNull();
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
it('requests DB verification for every suite in the real canonical inventory without widening the matrix', () => {
const cwd = fileURLToPath(new URL('../', import.meta.url));
const inputs = collectPlanInputs({ baseSha: 'HEAD', changedFiles: [], cwd });
diff --git a/scripts/lib/importReach.js b/scripts/lib/importReach.js
index 317206b11e..9a1dd1eb8d 100644
--- a/scripts/lib/importReach.js
+++ b/scripts/lib/importReach.js
@@ -155,15 +155,18 @@ export function formatReachSummary(report, { limit = 3 } = {}) {
return `dynamic-aware fan-out ${report.selected} suite(s): ${shown.join(', ')}${more}`;
}
-/** Tracked (plus given untracked) module files and a disk reader rooted at `cwd`. */
-export function measureRepoImportReach({ cwd, files, sources, isTest = serverRunnerTestPredicate() }) {
+/**
+ * Tracked (plus given untracked) module files. A collection may supply its own
+ * `readText(repoRelativePath)` snapshot; otherwise read directly under `cwd`.
+ */
+export function measureRepoImportReach({ cwd, files, sources, isTest = serverRunnerTestPredicate(), readText }) {
const paths = files ?? execFileSync('git', ['ls-files', '-z'], { cwd, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 })
.split('\0').filter(Boolean);
const modules = paths.filter((rel) => MODULE_SOURCE_RE.test(rel) && !rel.includes('node_modules/'));
- const graph = buildImportGraph(modules, (rel) => {
+ const graph = buildImportGraph(modules, readText ?? ((rel) => {
const abs = join(cwd, ...rel.split('/'));
return existsSync(abs) ? readFileSync(abs, 'utf8') : null;
- });
+ }));
return importReachReport(graph, sources.filter((source) => graph.has(source)), isTest);
}