Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

fix(gate): never reuse a test listing its own checkout undeclared [C7]

Corey Olson7ed931a41cef
diff --git a/src/affected-tests.js b/src/affected-tests.js
index d006f780..51fbe2aa 100644
--- a/src/affected-tests.js
+++ b/src/affected-tests.js
@@ -346,7 +346,7 @@ function graphAt(root, commit, changedDocs = []) {
*
* @param {string} root
* @param {string} [commit]
- * @returns {{ closures: Map<string, { files: string[] | null, reason: string }>, modules: Map<string, string>, blobs: Map<string, string> }}
+ * @returns {{ closures: Map<string, { files: string[] | null, helpers?: string[], reason: string }>, modules: Map<string, string>, blobs: Map<string, string> }}
*/
export function testClosures(root, commit = 'HEAD') {
const graph = graphAt(root, commit);
@@ -369,7 +369,12 @@ export function testClosures(root, commit = 'HEAD') {
if (!reached.has(dependency)) { reached.add(dependency); queue.push(dependency); }
}
}
- closures.set(test, reason ? { files: null, reason } : { files: [...reached].sort(), reason: '' });
+ // The test and the test/ helpers it reaches without passing through product code: its own test-side reads.
+ const helpers = new Set([test]);
+ for (const path of helpers) {
+ for (const dependency of forward.get(path) ?? []) if (dependency.startsWith('test/')) helpers.add(dependency);
+ }
+ closures.set(test, reason ? { files: null, reason } : { files: [...reached].sort(), helpers: [...helpers].sort(), reason: '' });
}
return { closures, modules: graph.modules, blobs: graph.blobs };
}
diff --git a/src/test-action-cache.js b/src/test-action-cache.js
index c252e255..82a505b8 100644
--- a/src/test-action-cache.js
+++ b/src/test-action-cache.js
@@ -45,7 +45,8 @@ const DECLARED_ENVIRONMENT = ['NODE_OPTIONS', 'PULLBOARD_HOSTED_SHAPES', 'PULLBO
const NEUTRAL_OPTION = /^--(?:test[-_]concurrency|test[-_]reporter(?:[-_]destination)?|test[-_]shard)(?:=|$)/u;
// A run that selects tests inside a file proves only those tests, so it neither reuses nor records.
const FILTER_OPTION = /^--test[-_](?:name|skip)[-_]pattern(?:=|$)|^--test[-_]only$/u;
-// Release notes are prose: src reads them for headlines that tests prove against their own fixture copies.
+// Release notes are prose: src reads them for headlines that tests prove against their own fixture copies. A test that
+// lists and reads them itself has no key at all (listsOwnCheckout).
const PROSE = new Set(['CHANGELOG.md']);
// Whole trees are inputs only by declaration (`@pullboard-test-input`), never because a fixture names `src`.
const TREE_DECLARATIONS = [['test-tree', ['test/']], ['source-tree', ['src/', 'bin/']], ['docs-tree', ['docs/']]];
@@ -114,16 +115,35 @@ function namedFiles(file, source, testSide, index) {
return [...named].filter(path => !MODULE.test(path) && !PROSE.has(path));
}
+/**
+ * True when a module lists the tracked files of its own checkout: a git call naming `ls-files` whose working folder is
+ * this checkout, either by default (no cwd) or through an expression built from import.meta or a ROOT constant. A
+ * listing in a fixture repository (`cwd: fixture.root`, a helper's own git) is not this checkout's.
+ */
+function listsOwnCheckout(source) {
+ for (const match of source.matchAll(/(['"])ls-files\1/gu)) {
+ const start = Math.max(source.lastIndexOf(';', match.index), source.lastIndexOf('\n\n', match.index));
+ const end = source.indexOf(';', match.index);
+ const statement = source.slice(start + 1, end === -1 ? source.length : end);
+ if (!/(['"])git\1/u.test(statement)) continue;
+ const cwd = /\bcwd\s*:\s*([^,}]+)/u.exec(statement)?.[1].trim();
+ if (!cwd || /import\.meta|\bROOT\b/u.test(cwd)) return true;
+ if (/^[A-Za-z_$][\w$]*$/u.test(cwd) && new RegExp(`\\b(?:const|let|var)\\s+${cwd}\\s*=[^;\\n]*(?:import\\.meta|\\bROOT\\b)`, 'u').test(source)) return true;
+ }
+ return false;
+}
+
/**
* What a test's closure reads beyond its imports. A declared tree (`@pullboard-test-input`) is read as data, so every
* file under it counts, JavaScript included: a comment can change a scanner's verdict without changing any import.
* Every module adds the files it names. A test that declares it reads its own checkout and names `ls-files` reads the
* list of tracked paths, so that list counts too; one that names a history command reads commits no tree holds, so it
- * has no key and always runs (fail closed).
+ * has no key and always runs (fail closed). So does a test that lists its own checkout's tracked files without declaring
+ * any tree: what it then reads (CHANGELOG.md included) is known to no key.
*
- * @returns {{ inputs: string[], listing: boolean, history: boolean }}
+ * @returns {{ inputs: string[], listing: boolean, history: boolean, undeclaredListing: boolean }}
*/
-function declaredReads(test, files, modules, index) {
+function declaredReads(test, files, modules, index, helpers = [test]) {
const inputs = new Set();
const header = (modules.get(test) ?? '').split(/\r?\n/u);
let declared = false;
@@ -132,19 +152,26 @@ function declaredReads(test, files, modules, index) {
declared = true;
for (const prefix of prefixes) for (const path of index.under(prefix)) inputs.add(path);
}
- let listing = false, history = false;
+ let listing = false, history = false, undeclaredListing = false;
for (const file of files) {
const testSide = file.startsWith('test/') || file === test;
const memo = `${testSide ? 't' : 's'}:${file}`;
if (!index.named.has(memo)) index.named.set(memo, namedFiles(file, modules.get(file) ?? '', testSide, index));
for (const path of index.named.get(memo)) inputs.add(path);
- if (!declared || !testSide) continue;
+ if (!testSide) continue;
+ if (!declared) {
+ // Only the test and the helpers it reaches directly: a checker that product code imports lists for its caller.
+ if (!helpers.includes(file)) continue;
+ if (!index.listers.has(file)) index.listers.set(file, listsOwnCheckout(modules.get(file) ?? ''));
+ undeclaredListing ||= index.listers.get(file);
+ continue;
+ }
if (!index.literals.has(file)) index.literals.set(file, stringLiterals(modules.get(file) ?? ''));
const literals = index.literals.get(file);
listing ||= literals.has('ls-files');
history ||= [...HISTORY].some(command => literals.has(command));
}
- return { inputs: [...inputs].sort(), listing, history };
+ return { inputs: [...inputs].sort(), listing, history, undeclaredListing };
}
/** Index the tracked tree: every file, every folder, files by their own name, and the files under any folder. */
@@ -166,7 +193,7 @@ function treeIndex(blobs) {
if (!cache.has(prefix)) cache.set(prefix, sorted.filter(path => path.startsWith(prefix)));
return cache.get(prefix);
};
- return { files, folders, byName, under, sorted, named: new Map(), literals: new Map() };
+ return { files, folders, byName, under, sorted, named: new Map(), literals: new Map(), listers: new Map() };
}
/**
@@ -192,8 +219,9 @@ export async function testActionKeys(root, { commit = 'HEAD', options = [], env
const keys = new Map();
for (const [test, closure] of closures) {
if (!closure.files) { keys.set(test, { key: null, reason: closure.reason }); continue; }
- const reads = declaredReads(test, closure.files, modules, index);
+ const reads = declaredReads(test, closure.files, modules, index, closure.helpers);
if (reads.history) { keys.set(test, { key: null, reason: 'it reads the git history of its own checkout' }); continue; }
+ if (reads.undeclaredListing) { keys.set(test, { key: null, reason: 'it lists the tracked files of its own checkout without declaring them' }); continue; }
keys.set(test, { key: digest({ ...shared, test,
closure: closure.files.map(path => [path, blobs.get(path) ?? null]),
inputs: reads.inputs.map(path => [path, blobs.get(path)]),
diff --git a/test/test-action-cache.test.js b/test/test-action-cache.test.js
index 0ab7a16a..8eb188ab 100644
--- a/test/test-action-cache.test.js
+++ b/test/test-action-cache.test.js
@@ -33,6 +33,17 @@ function scan() { appendFileSync(process.env.PULLBOARD_ACTION_CAPTURE, 'd\\n');
test('d [C7]', scan);
`;
+/** A fixture test that lists this checkout's tracked release notes without declaring them, then scans them. */
+const NOTES_TEST = `import { appendFileSync, readFileSync } from 'node:fs';
+import { execFileSync } from 'node:child_process';
+import { join } from 'node:path';
+import { test } from 'node:test';
+const root = join(import.meta.dirname, '..');
+/** Record that this file ran, then scan every listed note. */
+function scanNotes() { appendFileSync(process.env.PULLBOARD_ACTION_CAPTURE, 'e\\n'); const tracked = execFileSync('git', ['ls-files', 'CHANGELOG.md'], { cwd: root, encoding: 'utf8' }).trim().split('\\n'); for (const name of tracked) if (readFileSync(join(root, name), 'utf8').includes('forbidden')) throw new Error('e found a forbidden word'); }
+test('e [C7]', scanNotes);
+`;
+
/** A committed repository with the real runner, three tests and their modules: A imports shared, B and C stand alone. */
function makeFixture(t) {
const root = realpathSync(mkdtempSync(join(tmpdir(), 'pullboard-test-actions-')));
@@ -48,6 +59,8 @@ function makeFixture(t) {
writeFileSync(join(root, 'lib', 'c.js'), "export const value = 'red';\n");
for (const name of ['a', 'b', 'c']) writeFileSync(join(root, 'test', `${name}.test.js`), fixtureTest(name, name));
writeFileSync(join(root, 'test', 'd.test.js'), SCAN_TEST);
+ writeFileSync(join(root, 'test', 'e.test.js'), NOTES_TEST);
+ writeFileSync(join(root, 'CHANGELOG.md'), '# Changelog\n');
writeFileSync(join(root, 'src', 'probe.js'), "// A probe source file.\nexport const probe = 1;\n");
const capture = join(root, 'machine', 'ran.txt');
const env = { ...process.env, PULLBOARD_MACHINE_HOME: join(root, 'machine'), PULLBOARD_ACTION_CAPTURE: capture, PULLBOARD_LANDING_RUN: '0',
@@ -57,7 +70,7 @@ function makeFixture(t) {
/** Run one fixture git command under the private environment. */
const git = (...args) => runFixtureGit(args, { cwd: root, env });
git('init', '-q', '-b', 'main');
- git('add', '.gitignore', 'package.json', 'lib', 'test');
+ git('add', '.gitignore', 'package.json', 'CHANGELOG.md', 'lib', 'test');
git('add', '-f', 'src/probe.js');
git('commit', '-q', '-m', 'fixture');
/** Commit one file's new text. */
@@ -77,19 +90,20 @@ test('an unchanged closure reuses its green result and a changed import reruns i
const fixture = makeFixture(t);
const first = fixture.run();
assert.equal(first.status, 1, `C is red, so the first run fails\n${first.output}`);
- assert.deepEqual(first.ran, ['a', 'b', 'c', 'd'], 'every file runs when nothing is stored');
+ assert.deepEqual(first.ran, ['a', 'b', 'c', 'd', 'e'], 'every file runs when nothing is stored');
- // 1. A module only B imports changes: B runs, A is reused, C runs because a red result is never reused (3).
+ // 1. A module only B imports changes: B runs, A is reused, C runs because a red result is never reused (3). E lists
+ // this checkout's tracked files without declaring them, so it has no key and runs every time (fail closed).
fixture.change('lib/b.js', "export const value = 'green'; // edited\n");
const second = fixture.run();
- assert.deepEqual(second.ran, ['b', 'c'], second.output);
+ assert.deepEqual(second.ran, ['b', 'c', 'e'], second.output);
assert.match(second.output, /^# reused [0-9a-f]{64} from \S+ [0-9a-f]{40} \(test\/a\.test\.js\)$/mu);
assert.doesNotMatch(second.output, /reused .*\(test\/(?:b|c)\.test\.js\)/u);
// 2. A module A imports changes: A runs and B is reused; C still runs.
fixture.change('lib/shared.js', "export const shared = 'green'; // edited\n");
const third = fixture.run();
- assert.deepEqual(third.ran, ['a', 'c'], third.output);
+ assert.deepEqual(third.ran, ['a', 'c', 'e'], third.output);
assert.match(third.output, /^# reused [0-9a-f]{64} from \S+ [0-9a-f]{40} \(test\/b\.test\.js\)$/mu);
assert.equal(readdirSync(join(fixture.store, 'red')).length, 1, 'C has one red key, written once and kept');
@@ -104,22 +118,29 @@ test('an unchanged closure reuses its green result and a changed import reruns i
fixture.change('lib/c.js', "export const value = 'green';\n");
const fourth = fixture.run();
assert.equal(fourth.status, 0, fourth.output);
- assert.deepEqual(fourth.ran, ['a', 'b', 'c'], fourth.output);
+ assert.deepEqual(fourth.ran, ['a', 'b', 'c', 'e'], fourth.output);
assert.match(fourth.output, /^# reused [0-9a-f]{64} from \S+ [0-9a-f]{40} \(test\/d\.test\.js\)$/mu, 'D reads only the source tree, which no change touched');
- // Nothing changed since a green run: no file runs and the suite is proven from reused results alone.
+ // Nothing changed since a green run: only the unkeyed lister runs; every keyed file is reused.
const fifth = fixture.run();
assert.equal(fifth.status, 0, fifth.output);
- assert.deepEqual(fifth.ran, [], fifth.output);
+ assert.deepEqual(fifth.ran, ['e'], fifth.output);
assert.equal(fifth.output.match(/^# reused /gmu)?.length, 4, fifth.output);
- assert.match(fifth.output, /^# all 4 test files reused; none ran$/mu);
+
+ // Cached versus fresh: a forbidden word reaches only CHANGELOG.md. E lists and reads it, so E runs and turns red.
+ fixture.change('CHANGELOG.md', '# Changelog\n\nA forbidden word.\n');
+ const notes = fixture.run();
+ assert.deepEqual(notes.ran, ['e'], notes.output);
+ assert.equal(notes.status, 1, notes.output);
+ assert.match(notes.output, /e found a forbidden word/u);
// Cached versus fresh: only a comment changes, in a source file nothing imports. D reads it as data, so D runs and
// its fresh verdict differs from the green it would otherwise have reused.
fixture.change('src/probe.js', "// A probe source file with a forbidden word.\nexport const probe = 1;\n");
const sixth = fixture.run();
- assert.deepEqual(sixth.ran, ['d'], sixth.output);
- assert.equal(sixth.status, 1, `the fresh scan finds the word the stored green never saw\n${sixth.output}`);
+ assert.deepEqual(sixth.ran, ['d', 'e'], sixth.output);
+ assert.equal(sixth.status, 1, sixth.output);
+ assert.match(sixth.output, /d found a forbidden word/u, 'the fresh scan finds the word the stored green never saw');
});
test('a test that reads its own checkout history is never keyed [C7]', async (t) => {
@@ -136,6 +157,7 @@ test('h [C7]', history);
assert.equal(keys.get('test/h.test.js').key, null, 'commits are not in any tree a key hashes');
assert.match(keys.get('test/h.test.js').reason, /git history/u);
assert.match(keys.get('test/d.test.js').key, /^[0-9a-f]{64}$/u, 'a scanner without history keeps its key');
+ assert.equal(keys.get('test/e.test.js').key, null, 'an undeclared listing of this checkout has no key');
});
test('only passing results under the current key prove the whole suite [C7]', () => {