Skip to content

3D engine that AI agents drive

Fix three test tooling bugs: mutation check scope, timings wipe, timer overflow (#2579)

Nguyen Xuan An4408d3112fc9
diff --git a/AGENTS.md b/AGENTS.md
index 02c1204e..ef4ff233 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -50,7 +50,7 @@ Between one file and the full pass, while you iterate (each prints at most about
- `npm run test:affected -- src/foo.js`: every test file that mentions the file, or a module that re-exports it. With no files it uses what your branch changed.
- `npm run test:fast`: a deterministic 10 percent of the test files, seeded by the branch name (`-- --seed <text> --percent <n>`).
-- `npm run test:mutation-check -- tests/foo.test.js`: reverts your `src/`, `bin/` and `packages/` changes, runs the tests, lists the ones that fail and restores the tree. It exits 1 when every test still passes, which means the test does not pin your fix. Stage unstaged edits outside those folders first, and `git add -N` new files under them.
+- `npm run test:mutation-check -- tests/foo.test.js`: reverts your `src/`, `bin/`, `packages/` and `scripts/` changes, runs the tests, lists the ones that fail and restores the tree. It exits 1 when every test still passes, which means the test does not pin your fix. Stage unstaged edits outside those folders first, and `git add -N` new files under them.
- `npm test -- --test-reporter=./scripts/test-compact-reporter.mjs` gives the full pass the same short output.
## Rules that fail a pull request
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 8d1b4342..7c8a1c4d 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -80,7 +80,7 @@ Between one test file and the full pass there are four commands made for coding
- `npm run test:affected -- <files...>` runs every test file that mentions one of the files, or a module that re-exports it; with no files it uses the files your branch changed.
- `npm run test:fast` runs a deterministic 10 percent of the test files, seeded by the branch name (`-- --seed <text> --percent <n>` changes it). It does not replace `npm test`.
-- `npm run test:mutation-check -- tests/<name>.test.js` reverts your changes under `src/`, `bin/` and `packages/`, runs the tests, prints the ones that fail and restores the tree. It exits 1 when none fail, because then the test does not pin your change. Stage unstaged edits outside those folders first and `git add -N` new files under them.
+- `npm run test:mutation-check -- tests/<name>.test.js` reverts your changes under `src/`, `bin/`, `packages/` and `scripts/`, runs the tests, prints the ones that fail and restores the tree. It exits 1 when none fail, because then the test does not pin your change. Stage unstaged edits outside those folders first and `git add -N` new files under them.
- `npm test -- --test-reporter=./scripts/test-compact-reporter.mjs` gives the full pass the same short output.
The checks it runs, if you want one alone:
diff --git a/FEATURES.md b/FEATURES.md
index 89c2900d..57e64bc9 100644
--- a/FEATURES.md
+++ b/FEATURES.md
@@ -585,7 +585,7 @@ optional `Ready` expression the driver waits for, and `Drive`.
- Drive: none: not a page
### `npm-test-mutation-check`
-- Name: Reverts your src, bin and packages changes, runs a test file and fails when it still passes
+- Name: Reverts your src, bin, packages and scripts changes, runs a test file and fails when it still passes
- Surface: CLI
- Reach: command `npm run test:mutation-check`
- Source: `scripts/test-mutation-check.mjs`
diff --git a/changes/2579.md b/changes/2579.md
new file mode 100644
index 00000000..98248664
--- /dev/null
+++ b/changes/2579.md
@@ -0,0 +1 @@
+- `test:mutation-check` also reverts changes under `scripts/` and reads `--base=<ref>`, `shard-test-files --merge-timings` refuses an empty file list instead of wiping the timings, and the test hang guard refuses a limit above 2147483647 ms (#2579, #2580, #2581)
diff --git a/scripts/shard-test-files.mjs b/scripts/shard-test-files.mjs
index 6c4c7e9b..09e4c57d 100644
--- a/scripts/shard-test-files.mjs
+++ b/scripts/shard-test-files.mjs
@@ -67,7 +67,8 @@ function listTestFiles() {
.sort();
}
-function mergeTimings(paths) {
+export function mergeTimings(paths, destination = timingsPath) {
+ if (paths.length === 0) throw new Error('--merge-timings needs at least one timings file: node scripts/shard-test-files.mjs --merge-timings <files>');
const measurements = new Map();
for (const path of paths) {
for (const [name, milliseconds] of Object.entries(JSON.parse(readFileSync(path, 'utf8')))) {
@@ -77,11 +78,12 @@ function mergeTimings(paths) {
const sorted = Object.fromEntries(
[...measurements.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([name, values]) => [name, Math.round(medianOf(values))])
);
- writeFileSync(timingsPath, `${JSON.stringify(sorted, null, 2)}\n`);
+ writeFileSync(destination, `${JSON.stringify(sorted, null, 2)}\n`);
console.log(`Wrote ${Object.keys(sorted).length} timings to test-timings.json`);
}
function mergeTimingsFromRun(runId) {
+ if (!runId) throw new Error('--from-run needs a run id: node scripts/shard-test-files.mjs --from-run <run id>');
const downloadDirectory = mkdtempSync(join(tmpdir(), 'test-timings-'));
try {
execFileSync('gh', ['run', 'download', runId, '--pattern', 'lcov-shard-*', '-D', downloadDirectory], { stdio: 'inherit' });
diff --git a/scripts/test-hang-guard.mjs b/scripts/test-hang-guard.mjs
index 7d52fb02..a453b01b 100644
--- a/scripts/test-hang-guard.mjs
+++ b/scripts/test-hang-guard.mjs
@@ -16,6 +16,8 @@ import { ChildProcess, spawnSync } from 'node:child_process';
export const DEFAULT_FILE_LIMIT_MS = 660000;
export const DEFAULT_GRACE_AFTER_TIMEOUT_MS = 10000;
+// setTimeout runs a longer delay after 1 ms, which would stop a healthy file at once.
+const LONGEST_TIMER_MS = 2 ** 31 - 1;
const TEST_TIMEOUT_MESSAGE = /test timed out after \d+ms/;
const startedChildren = [];
@@ -106,8 +108,8 @@ function positiveMilliseconds(environment, name, defaultValue) {
const raw = environment[name];
if (raw === undefined || raw === '') return defaultValue;
const parsed = Number(raw);
- if (!Number.isFinite(parsed) || parsed <= 0) {
- throw new Error(`${name} must be a positive number of milliseconds, got "${raw}". Unset it to use ${defaultValue}.`);
+ if (!Number.isFinite(parsed) || parsed <= 0 || parsed > LONGEST_TIMER_MS) {
+ throw new Error(`${name} must be a positive number of milliseconds, at most ${LONGEST_TIMER_MS}, got "${raw}". Unset it to use ${defaultValue}.`);
}
return parsed;
}
diff --git a/scripts/test-mutation-check.mjs b/scripts/test-mutation-check.mjs
index 0f2d3294..752a5cde 100644
--- a/scripts/test-mutation-check.mjs
+++ b/scripts/test-mutation-check.mjs
@@ -8,7 +8,7 @@ import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { isMainModule, repositoryRoot, runTestFiles } from './test-run.mjs';
-export const REVERTED_FOLDERS = ['src', 'bin', 'packages'];
+export const REVERTED_FOLDERS = ['src', 'bin', 'packages', 'scripts'];
function git(arguments_, folder, input) {
const result = spawnSync('git', arguments_, { cwd: folder, encoding: 'utf8', input, maxBuffer: 256 * 1024 * 1024 });
@@ -77,17 +77,26 @@ export function runMutationCheck(testFiles, { folder = repositoryRoot, baseRef =
}
export function parseArguments(arguments_) {
- const baseIndex = arguments_.indexOf('--base');
- const baseRef = baseIndex < 0 ? 'origin/main' : arguments_[baseIndex + 1];
- const valueIndex = baseIndex < 0 ? -1 : baseIndex + 1;
- const testFiles = arguments_.filter((argument, index) => !argument.startsWith('--') && index !== valueIndex);
+ let baseRef = 'origin/main';
+ const testFiles = [];
+ for (let index = 0; index < arguments_.length; index += 1) {
+ const argument = arguments_[index];
+ if (argument === '--base' || argument.startsWith('--base=')) {
+ baseRef = argument === '--base' ? arguments_[(index += 1)] : argument.slice('--base='.length);
+ if (!baseRef) throw new Error('--base needs a ref: --base <ref> or --base=<ref>, for example --base origin/main.');
+ } else if (argument.startsWith('--')) {
+ throw new Error(`Unknown option ${argument}. The only option is --base <ref>.`);
+ } else {
+ testFiles.push(argument);
+ }
+ }
return { baseRef, testFiles };
}
function main(arguments_) {
- const { baseRef, testFiles } = parseArguments(arguments_);
let result;
try {
+ const { baseRef, testFiles } = parseArguments(arguments_);
result = runMutationCheck(testFiles, { baseRef });
} catch (error) {
console.error(error.message);
diff --git a/tests/agent-test-commands.test.js b/tests/agent-test-commands.test.js
index 062635ff..72303287 100644
--- a/tests/agent-test-commands.test.js
+++ b/tests/agent-test-commands.test.js
@@ -157,3 +157,31 @@ test('mutation check arguments keep every test path and read --base', () => {
assert.deepEqual(parseArguments(['tests/a.test.js', 'tests/b.test.js']), { baseRef: 'origin/main', testFiles: ['tests/a.test.js', 'tests/b.test.js'] });
assert.deepEqual(parseArguments(['tests/a.test.js', '--base', 'main']), { baseRef: 'main', testFiles: ['tests/a.test.js'] });
});
+
+test('mutation check reverts a change under scripts/ too', () => {
+ const folder = makeRepository();
+ try {
+ git(folder, 'checkout', '-q', 'main');
+ write(folder, 'scripts/answer.mjs', 'export const answer = 1;\n');
+ write(folder, 'tests/answer.test.js', "import test from 'node:test';\nimport assert from 'node:assert/strict';\nimport { answer } from '../scripts/answer.mjs';\ntest('answer is two', () => assert.equal(answer, 2));\n");
+ git(folder, 'add', '-A');
+ git(folder, 'commit', '-q', '-m', 'script and test');
+ git(folder, 'checkout', '-q', '-b', 'script-fix');
+ write(folder, 'scripts/answer.mjs', 'export const answer = 2;\n');
+ git(folder, 'commit', '-q', '-am', 'fix script');
+ const result = runMutationCheck(['tests/answer.test.js'], { folder, baseRef: 'main', runTests: (files, options) => runTestFiles(files, { ...options, quiet: true }) });
+ assert.deepEqual(result.failingTests.map(failure => failure.name), ['answer is two']);
+ assert.equal(readFileSync(join(folder, 'scripts/answer.mjs'), 'utf8'), 'export const answer = 2;\n');
+ assert.equal(findBlockingChanges('?? scripts/new.mjs\n').length, 1);
+ assert.deepEqual(findBlockingChanges(' M scripts/answer.mjs\n'), []);
+ } finally {
+ rmSync(folder, { recursive: true, force: true });
+ }
+});
+
+test('mutation check arguments accept --base=<ref> and refuse a missing ref or an unknown option', () => {
+ assert.deepEqual(parseArguments(['tests/a.test.js', '--base=main']), { baseRef: 'main', testFiles: ['tests/a.test.js'] });
+ assert.throws(() => parseArguments(['tests/a.test.js', '--base']), /--base needs a ref/);
+ assert.throws(() => parseArguments(['--base=', 'tests/a.test.js']), /--base needs a ref/);
+ assert.throws(() => parseArguments(['tests/a.test.js', '--bsae', 'main']), /Unknown option --bsae/);
+});
diff --git a/tests/shard-test-files.test.js b/tests/shard-test-files.test.js
index f7baa041..85d8f063 100644
--- a/tests/shard-test-files.test.js
+++ b/tests/shard-test-files.test.js
@@ -55,3 +55,23 @@ test('the coverage gate expects as many shards as the workflow matrix has', () =
const expected = Number(workflow.match(/coverage-gate\.mjs [^\n]*--expect-shards (\d+)/)[1]);
assert.equal(expected, matrix.length);
});
+
+test('merging no timings files refuses instead of wiping the timings', async () => {
+ const { mergeTimings } = await import('../scripts/shard-test-files.mjs');
+ const { mkdtempSync, writeFileSync, rmSync } = await import('node:fs');
+ const { tmpdir } = await import('node:os');
+ const { join } = await import('node:path');
+ const folder = mkdtempSync(join(tmpdir(), 'crucible-timings-'));
+ try {
+ const destination = join(folder, 'test-timings.json');
+ writeFileSync(destination, '{"tests/a.test.js":5}\n');
+ assert.throws(() => mergeTimings([], destination), /needs at least one timings file/);
+ assert.equal(readFileSync(destination, 'utf8'), '{"tests/a.test.js":5}\n');
+ const first = join(folder, 'one.json');
+ writeFileSync(first, '{"tests/a.test.js":7,"tests/b.test.js":3}');
+ mergeTimings([first], destination);
+ assert.deepEqual(JSON.parse(readFileSync(destination, 'utf8')), { 'tests/a.test.js': 7, 'tests/b.test.js': 3 });
+ } finally {
+ rmSync(folder, { recursive: true, force: true });
+ }
+});
diff --git a/tests/test-hang-guard.test.js b/tests/test-hang-guard.test.js
index 65a9777e..04ad3fb7 100644
--- a/tests/test-hang-guard.test.js
+++ b/tests/test-hang-guard.test.js
@@ -95,3 +95,9 @@ test('the limits have defaults and reject a value that is not a positive number'
assert.throws(() => fileLimitMilliseconds({ CRUCIBLE_TEST_FILE_LIMIT_MS: '-1' }), /must be a positive number of milliseconds/);
assert.throws(() => graceAfterTimeoutMilliseconds({ CRUCIBLE_TEST_GRACE_AFTER_TIMEOUT_MS: '0' }), /must be a positive number of milliseconds/);
});
+
+test('a limit longer than a timer can hold is refused instead of stopping the file after 1 ms', () => {
+ assert.equal(fileLimitMilliseconds({ CRUCIBLE_TEST_FILE_LIMIT_MS: '2147483647' }), 2147483647);
+ assert.throws(() => fileLimitMilliseconds({ CRUCIBLE_TEST_FILE_LIMIT_MS: '3000000000' }), /at most 2147483647/);
+ assert.throws(() => graceAfterTimeoutMilliseconds({ CRUCIBLE_TEST_GRACE_AFTER_TIMEOUT_MS: '2147483648' }), /at most 2147483647/);
+});