Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

test: key non-literal require() sites and importlib imports in the test memo (#12300)

[._.]/ Adam Eivyfdc502f1c0ee
diff --git a/docs/GITHUB_ACTIONS.md b/docs/GITHUB_ACTIONS.md
index 8e406807d9..750e25b7b3 100644
--- a/docs/GITHUB_ACTIONS.md
+++ b/docs/GITHUB_ACTIONS.md
@@ -946,7 +946,12 @@ cannot resolve (`import.meta.glob`, anything under `.git/`, a non-literal
`readServerSource`/`readClientSource`, a literal directory with more than
400 files); a bare specifier that resolves to no installed package; and, for
the floor key only, a non-literal `import()`/`require()` or an unparseable
-member. A recorded set that reaches outside the repository is not memoizable
+member. Two non-literal `require()` shapes are keyed instead: `require(CONST)`
+with a module-level string is that bare specifier (package version), and
+`require(join(<unknown dir>, 'name.cjs'))` keys every tracked file of that
+basename. A Python `importlib.import_module('x')` follows like `import x`; a
+computed name is not memoizable. `import(variable)` stays outside the
+recorded key by design: only what the green run loaded is keyed. A recorded set that reaches outside the repository is not memoizable
either. Every unknown construct lands in one of those buckets rather than
silently narrowing the key.
diff --git a/scripts/lib/testMemo.js b/scripts/lib/testMemo.js
index 5654de617f..817877e516 100644
--- a/scripts/lib/testMemo.js
+++ b/scripts/lib/testMemo.js
@@ -692,6 +692,7 @@ export function extractFileFacts(src, { fileRel, cwdRel }) {
edges: [], // { spec, kind: 'static'|'dynamic'|'require'|'mock' }
paths: [], // { rel, walk } — repo-relative literal paths
dirs: [], // { rel, walk } — repo-relative directory prefixes whose tree is an input
+ scopedNames: [], // basenames a require() reads under an unknown directory: every tracked file of that name is an input
cwdLiterals: [], // { value, walk, dir } — resolved against cwd, then repo root, then the file
unresolvedEdges: [], // the module graph is not fully known from source
unresolvedInputs: [], // a filesystem input is not fully known from source
@@ -733,6 +734,16 @@ export function extractFileFacts(src, { fileRel, cwdRel }) {
return { kind: 'expr', index: open + 1 };
};
+ /** The literal last segment of `join(<anything>, 'name')`, followed through one binding; else null. */
+ const joinedBasename = (index) => {
+ let at = index;
+ for (let hop = 0; hop < 4 && tokens[at]?.t === 'id' && ctx.bindings.has(tokens[at].v) && tokens[at + 1]?.v !== '('; hop += 1) at = ctx.bindings.get(tokens[at].v);
+ if (tokens[at]?.t === 'id' && tokens[at + 1]?.v === '.' && tokens[at + 2]?.t === 'id') at += 2; // path.join(
+ if (tokens[at]?.t !== 'id' || !PATH_JOIN.has(tokens[at].v) || tokens[at + 1]?.v !== '(') return null;
+ const last = (evaluateArgs(tokens, match, at + 1, ctx, 0) ?? []).at(-1);
+ return last?.k === 'str' && !last.v.endsWith('/') && last.v !== '' ? posix.basename(last.v) : null;
+ };
+
const record = (value, { walk = false, consumer = null, bare = false } = {}) => {
if (value.k === 'home') {
if (!(consumer !== null && NON_READ_CALLEES.has(consumer))) facts.localReads.push('~');
@@ -785,6 +796,12 @@ export function extractFileFacts(src, { fileRel, cwdRel }) {
const { value } = evaluate(tokens, match, arg.index, ctx);
if (value.k === 'path' && !value.partial && !value.isDir) { facts.paths.push({ rel: value.v, walk: true }); continue; }
if (value.k === 'path') { facts.dirs.push({ rel: value.v, walk: true }); continue; }
+ // `require(PACKAGE_CONSTANT)`: a module-level string is a specifier like a literal one.
+ if (value.k === 'str') { facts.edges.push({ spec: value.v, kind }); continue; }
+ // `require(join(<unknown dir>, 'ecosystem.config.cjs'))`: the directory is a parameter or an
+ // install root, the basename is fixed, so every tracked file of that name is an input.
+ const scoped = kind === 'require' ? joinedBasename(arg.index) : null;
+ if (scoped !== null) { facts.scopedNames.push(scoped); continue; }
facts.unresolvedEdges.push(`${token.v}() with a non-literal specifier`);
continue;
}
@@ -916,6 +933,7 @@ export function createTestMemoAnalyzer({ fs, workspaceRel, env = {}, platform =
const facts = new Map();
const trees = new Map();
const packages = new Map();
+ let byName = null;
const fileHash = (rel) => {
if (hashes.has(rel)) return hashes.get(rel);
@@ -948,6 +966,24 @@ export function createTestMemoAnalyzer({ fs, workspaceRel, env = {}, platform =
return result;
};
+ /** Every file named `name` anywhere in the repository (one cached walk, same skips as `tree`). */
+ const filesNamed = (name) => {
+ if (!byName) {
+ byName = new Map();
+ const walk = (dir) => {
+ for (const entry of fs.list(dir)) {
+ if (SKIP_TREE_DIRS.has(entry) || (dir === '' && entry === 'data')) continue;
+ const rel = dir === '' ? entry : `${dir}/${entry}`;
+ const kind = fs.kind(rel);
+ if (kind === 'dir') walk(rel);
+ else if (kind === 'file') byName.set(entry, [...(byName.get(entry) ?? []), rel]);
+ }
+ };
+ walk('');
+ }
+ return (byName.get(name) ?? []).sort();
+ };
+
/** Installed version of the package `spec` reaches from `fromRel`, or null. */
const packageVersion = (fromRel, spec) => {
const name = packageNameOf(spec);
@@ -1006,8 +1042,15 @@ export function createTestMemoAnalyzer({ fs, workspaceRel, env = {}, platform =
const dir = posix.dirname(target);
for (const line of text.split('\n')) {
const m = /^\s*(?:from\s+(\.*[\w.]*)\s+import\b|import\s+([\w.]+(?:\s*,\s*[\w.]+)*))/.exec(line);
- if (!m) continue;
- for (const name of (m[1] !== undefined ? [m[1]] : m[2].split(',')).map((part) => part.trim()).filter(Boolean)) {
+ const names = m ? (m[1] !== undefined ? [m[1]] : m[2].split(',')).map((part) => part.trim()).filter(Boolean) : [];
+ // `importlib.import_module('x')` is a sibling import; a non-literal name may reach any module.
+ if (!/^\s*#/.test(line)) {
+ for (const call of line.matchAll(/\bimport_module\(\s*(?:(['"])([\w.]+)\1|)/g)) {
+ if (call[2] !== undefined) names.push(call[2]);
+ else resolved.unresolvedInputs.push(`python import_module with a non-literal name in ${target}`);
+ }
+ }
+ for (const name of names) {
const relative = name.startsWith('.');
const head = name.replace(/^\.+/, '').split('.')[0];
const candidate = [`${dir}/${head}.py`, `${dir}/${head}/__init__.py`].find((rel) => fs.kind(rel) === 'file');
@@ -1022,6 +1065,7 @@ export function createTestMemoAnalyzer({ fs, workspaceRel, env = {}, platform =
else resolved.inputs.add(target);
};
const localState = (target) => target === 'data' || target.startsWith('data/');
+ for (const name of raw.scopedNames) for (const target of filesNamed(name)) addFile(target);
const addDir = (dirRel, walk) => {
if (localState(dirRel)) { resolved.localReads.push(dirRel); return; }
if (ignoredInput(dirRel)) return;
diff --git a/scripts/lib/testMemo.test.js b/scripts/lib/testMemo.test.js
index af5cb3604c..570471a100 100644
--- a/scripts/lib/testMemo.test.js
+++ b/scripts/lib/testMemo.test.js
@@ -207,6 +207,37 @@ describe('recorded key', () => {
});
});
+describe('non-literal native require() in a product module', () => {
+ const test = ["import { it } from 'vitest';", "import { start } from './boot.js';", "it('b', () => start());"].join('\n');
+ const tree = (boot) => ({
+ ...TREE,
+ 'server/lib/boot.js': boot,
+ 'server/lib/boot.test.js': test,
+ 'ecosystem.config.cjs': 'module.exports = { apps: [] };\n',
+ 'sub/ecosystem.config.cjs': 'module.exports = { apps: [1] };\n',
+ 'server/node_modules/native-driver/package.json': '{"name":"native-driver","version":"1.0.0"}',
+ });
+ const floor = (files) => keyOf(files, 'server/lib/boot.test.js').floor;
+
+ it('keys every tracked file of the basename when the directory is unknown', () => {
+ const boot = ["import { createRequire } from 'node:module';", "import { join } from 'node:path';", 'const require = createRequire(import.meta.url);', "const NAME = 'ecosystem.config.cjs';",
+ 'export const start = (root) => { const configPath = join(root.dir, NAME); return require(configPath); };'].join('\n');
+ const base = floor(tree(boot));
+ expect(base.memoizable).toBe(true);
+ expect(floor({ ...tree(boot), 'ecosystem.config.cjs': 'module.exports = { apps: [2] };\n' }).key).not.toBe(base.key);
+ expect(floor({ ...tree(boot), 'sub/ecosystem.config.cjs': 'module.exports = { apps: [2] };\n' }).key).not.toBe(base.key);
+ // A non-literal basename is still unknown.
+ expect(floor(tree(boot.replace("join(root.dir, NAME)", 'join(root.dir, root.name)')))).toMatchObject({ memoizable: false, reason: expect.stringContaining('non-literal') });
+ });
+
+ it('keys a package-name constant by the installed package version', () => {
+ const boot = ["import { createRequire } from 'node:module';", 'const require = createRequire(import.meta.url);', "const DRIVER = 'native-driver';", 'export const start = () => require(DRIVER);'].join('\n');
+ const base = floor(tree(boot));
+ expect(base.memoizable).toBe(true);
+ expect(floor({ ...tree(boot), 'server/node_modules/native-driver/package.json': '{"name":"native-driver","version":"2.0.0"}' }).key).not.toBe(base.key);
+ });
+});
+
describe('python sidecars, external tools and machine-local state', () => {
it('hashes a python sidecar and its resolvable sibling imports, and keys the python version', () => {
const files = {
@@ -236,6 +267,19 @@ describe('python sidecars, external tools and machine-local state', () => {
expect(key(unknownProgram)).toMatchObject({ memoizable: false, reason: expect.stringContaining('unknown program') });
});
+ it('follows importlib.import_module with a literal name and fails closed on a computed one', () => {
+ const files = {
+ ...TREE,
+ 'scripts/tool.py': "import importlib\nmod = importlib.import_module('_pins')\n",
+ 'scripts/_pins.py': 'PINS = 1\n',
+ 'scripts/tool.test.js': ["import { it } from 'vitest';", "import { spawnSync } from 'node:child_process';", "it('p', () => spawnSync('python3', [new URL('./tool.py', import.meta.url).pathname]));"].join('\n'),
+ };
+ const key = (tree) => keyOf(tree, 'scripts/tool.test.js', { toolVersion: () => 'Python 3.12.1' }).floor;
+ expect(key(files).memoizable).toBe(true);
+ expect(key({ ...files, 'scripts/_pins.py': 'PINS = 2\n' }).key).not.toBe(key(files).key);
+ expect(key({ ...files, 'scripts/tool.py': 'import importlib\nmod = importlib.import_module(name)\n' })).toMatchObject({ memoizable: false, reason: expect.stringContaining('import_module') });
+ });
+
it('fails closed when the test or repository tooling reads the install data tree or the home directory, and ignores a product module doing so', () => {
// The fixture sources spell the data directory from pieces so this file
// does not itself trip lib/testDataIsolation.guards.test.js.
diff --git a/server/services/pm2.js b/server/services/pm2.js
index c268540020..2149c6ff5f 100644
--- a/server/services/pm2.js
+++ b/server/services/pm2.js
@@ -83,6 +83,8 @@ function logJlistRecovery(pm2Home) {
// Resolve PM2 CLI binary path from our local dependency using require.resolve
// to handle hoisted node_modules correctly.
const require = createRequire(import.meta.url);
+const ECOSYSTEM_FILE = 'ecosystem.config.cjs';
+const LEGACY_ECOSYSTEM_FILE = 'ecosystem.config.js';
const PM2_BIN = join(dirname(require.resolve('pm2/package.json')), 'bin', 'pm2');
/**
@@ -812,7 +814,7 @@ export async function getSavedProcessNames(pm2Home = null) {
* @param {string} pm2Home Optional custom PM2_HOME path
*/
export async function startFromEcosystem(cwd, processNames = [], pm2Home = null) {
- const ecosystemFile = ['ecosystem.config.cjs', 'ecosystem.config.js']
+ const ecosystemFile = [ECOSYSTEM_FILE, LEGACY_ECOSYSTEM_FILE]
.find(f => existsSync(`${cwd}/${f}`));
if (!ecosystemFile) {
@@ -834,15 +836,16 @@ export async function startFromEcosystem(cwd, processNames = [], pm2Home = null)
* try to require() .cmd batch files as JavaScript).
*/
async function startFromEcosystemWindows(cwd, ecosystemFile, processNames, pm2Home) {
- const configPath = join(cwd, ecosystemFile);
+ // Each file name is a module-level literal joined at its own require() so the
+ // test memo can key every tracked ecosystem file (scripts/lib/testMemo.js).
+ const configPath = ecosystemFile === ECOSYSTEM_FILE ? join(cwd, ECOSYSTEM_FILE) : join(cwd, LEGACY_ECOSYSTEM_FILE);
// Try to load and patch the config
let config;
try {
- const require = createRequire(configPath);
// Clear module cache to get fresh config on repeated starts
try { delete require.cache[require.resolve(configPath)]; } catch {}
- config = require(configPath);
+ config = ecosystemFile === ECOSYSTEM_FILE ? require(join(cwd, ECOSYSTEM_FILE)) : require(join(cwd, LEGACY_ECOSYSTEM_FILE));
} catch (err) {
// If we can't load the config (syntax error, missing deps, etc.),
// fall back to unpatched start — PM2 may still handle it