3D engine that AI agents drive
JavaScript 3d3d-engineagentic-aiai-agentsblenderclaude-codegame-enginegaussian-splattinggltfjavascriptmcpmcp-servermodel-context-protocolpath-tracerpath-tracingpbrrenderingthreejs-alternativewebgl2webgpu
Fix three test tooling bugs: mutation check scope, timings wipe, timer overflow (#2579)
Nguyen Xuan An4408d3112fc9
diff --git a/AGENTS.md b/AGENTS.mdindex 02c1204e..ef4ff233 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -50,7 +50,7 @@ Between one file and the full pass, while you iterate (each prints at most about- `npm run test:affected -- src/foo.js`: every test file that mentions the file, or a module that re-exports it. With no files it uses what your branch changed.- `npm run test:fast`: a deterministic 10 percent of the test files, seeded by the branch name (`-- --seed <text> --percent <n>`).-- `npm run test:mutation-check -- tests/foo.test.js`: reverts your `src/`, `bin/` and `packages/` changes, runs the tests, lists the ones that fail and restores the tree. It exits 1 when every test still passes, which means the test does not pin your fix. Stage unstaged edits outside those folders first, and `git add -N` new files under them.+- `npm run test:mutation-check -- tests/foo.test.js`: reverts your `src/`, `bin/`, `packages/` and `scripts/` changes, runs the tests, lists the ones that fail and restores the tree. It exits 1 when every test still passes, which means the test does not pin your fix. Stage unstaged edits outside those folders first, and `git add -N` new files under them.- `npm test -- --test-reporter=./scripts/test-compact-reporter.mjs` gives the full pass the same short output.## Rules that fail a pull requestdiff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex 8d1b4342..7c8a1c4d 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -80,7 +80,7 @@ Between one test file and the full pass there are four commands made for coding- `npm run test:affected -- <files...>` runs every test file that mentions one of the files, or a module that re-exports it; with no files it uses the files your branch changed.- `npm run test:fast` runs a deterministic 10 percent of the test files, seeded by the branch name (`-- --seed <text> --percent <n>` changes it). It does not replace `npm test`.-- `npm run test:mutation-check -- tests/<name>.test.js` reverts your changes under `src/`, `bin/` and `packages/`, runs the tests, prints the ones that fail and restores the tree. It exits 1 when none fail, because then the test does not pin your change. Stage unstaged edits outside those folders first and `git add -N` new files under them.+- `npm run test:mutation-check -- tests/<name>.test.js` reverts your changes under `src/`, `bin/`, `packages/` and `scripts/`, runs the tests, prints the ones that fail and restores the tree. It exits 1 when none fail, because then the test does not pin your change. Stage unstaged edits outside those folders first and `git add -N` new files under them.- `npm test -- --test-reporter=./scripts/test-compact-reporter.mjs` gives the full pass the same short output.The checks it runs, if you want one alone:diff --git a/FEATURES.md b/FEATURES.mdindex 89c2900d..57e64bc9 100644--- a/FEATURES.md+++ b/FEATURES.md@@ -585,7 +585,7 @@ optional `Ready` expression the driver waits for, and `Drive`.- Drive: none: not a page### `npm-test-mutation-check`-- Name: Reverts your src, bin and packages changes, runs a test file and fails when it still passes+- Name: Reverts your src, bin, packages and scripts changes, runs a test file and fails when it still passes- Surface: CLI- Reach: command `npm run test:mutation-check`- Source: `scripts/test-mutation-check.mjs`diff --git a/changes/2579.md b/changes/2579.mdnew file mode 100644index 00000000..98248664--- /dev/null+++ b/changes/2579.md@@ -0,0 +1 @@+- `test:mutation-check` also reverts changes under `scripts/` and reads `--base=<ref>`, `shard-test-files --merge-timings` refuses an empty file list instead of wiping the timings, and the test hang guard refuses a limit above 2147483647 ms (#2579, #2580, #2581)diff --git a/scripts/shard-test-files.mjs b/scripts/shard-test-files.mjsindex 6c4c7e9b..09e4c57d 100644--- a/scripts/shard-test-files.mjs+++ b/scripts/shard-test-files.mjs@@ -67,7 +67,8 @@ function listTestFiles() {.sort();}-function mergeTimings(paths) {+export function mergeTimings(paths, destination = timingsPath) {+ if (paths.length === 0) throw new Error('--merge-timings needs at least one timings file: node scripts/shard-test-files.mjs --merge-timings <files>');const measurements = new Map();for (const path of paths) {for (const [name, milliseconds] of Object.entries(JSON.parse(readFileSync(path, 'utf8')))) {@@ -77,11 +78,12 @@ function mergeTimings(paths) {const sorted = Object.fromEntries([...measurements.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([name, values]) => [name, Math.round(medianOf(values))]));- writeFileSync(timingsPath, `${JSON.stringify(sorted, null, 2)}\n`);+ writeFileSync(destination, `${JSON.stringify(sorted, null, 2)}\n`);console.log(`Wrote ${Object.keys(sorted).length} timings to test-timings.json`);}function mergeTimingsFromRun(runId) {+ if (!runId) throw new Error('--from-run needs a run id: node scripts/shard-test-files.mjs --from-run <run id>');const downloadDirectory = mkdtempSync(join(tmpdir(), 'test-timings-'));try {execFileSync('gh', ['run', 'download', runId, '--pattern', 'lcov-shard-*', '-D', downloadDirectory], { stdio: 'inherit' });diff --git a/scripts/test-hang-guard.mjs b/scripts/test-hang-guard.mjsindex 7d52fb02..a453b01b 100644--- a/scripts/test-hang-guard.mjs+++ b/scripts/test-hang-guard.mjs@@ -16,6 +16,8 @@ import { ChildProcess, spawnSync } from 'node:child_process';export const DEFAULT_FILE_LIMIT_MS = 660000;export const DEFAULT_GRACE_AFTER_TIMEOUT_MS = 10000;+// setTimeout runs a longer delay after 1 ms, which would stop a healthy file at once.+const LONGEST_TIMER_MS = 2 ** 31 - 1;const TEST_TIMEOUT_MESSAGE = /test timed out after \d+ms/;const startedChildren = [];@@ -106,8 +108,8 @@ function positiveMilliseconds(environment, name, defaultValue) {const raw = environment[name];if (raw === undefined || raw === '') return defaultValue;const parsed = Number(raw);- if (!Number.isFinite(parsed) || parsed <= 0) {- throw new Error(`${name} must be a positive number of milliseconds, got "${raw}". Unset it to use ${defaultValue}.`);+ if (!Number.isFinite(parsed) || parsed <= 0 || parsed > LONGEST_TIMER_MS) {+ throw new Error(`${name} must be a positive number of milliseconds, at most ${LONGEST_TIMER_MS}, got "${raw}". Unset it to use ${defaultValue}.`);}return parsed;}diff --git a/scripts/test-mutation-check.mjs b/scripts/test-mutation-check.mjsindex 0f2d3294..752a5cde 100644--- a/scripts/test-mutation-check.mjs+++ b/scripts/test-mutation-check.mjs@@ -8,7 +8,7 @@ import { existsSync, mkdirSync, writeFileSync } from 'node:fs';import { join } from 'node:path';import { isMainModule, repositoryRoot, runTestFiles } from './test-run.mjs';-export const REVERTED_FOLDERS = ['src', 'bin', 'packages'];+export const REVERTED_FOLDERS = ['src', 'bin', 'packages', 'scripts'];function git(arguments_, folder, input) {const result = spawnSync('git', arguments_, { cwd: folder, encoding: 'utf8', input, maxBuffer: 256 * 1024 * 1024 });@@ -77,17 +77,26 @@ export function runMutationCheck(testFiles, { folder = repositoryRoot, baseRef =}export function parseArguments(arguments_) {- const baseIndex = arguments_.indexOf('--base');- const baseRef = baseIndex < 0 ? 'origin/main' : arguments_[baseIndex + 1];- const valueIndex = baseIndex < 0 ? -1 : baseIndex + 1;- const testFiles = arguments_.filter((argument, index) => !argument.startsWith('--') && index !== valueIndex);+ let baseRef = 'origin/main';+ const testFiles = [];+ for (let index = 0; index < arguments_.length; index += 1) {+ const argument = arguments_[index];+ if (argument === '--base' || argument.startsWith('--base=')) {+ baseRef = argument === '--base' ? arguments_[(index += 1)] : argument.slice('--base='.length);+ if (!baseRef) throw new Error('--base needs a ref: --base <ref> or --base=<ref>, for example --base origin/main.');+ } else if (argument.startsWith('--')) {+ throw new Error(`Unknown option ${argument}. The only option is --base <ref>.`);+ } else {+ testFiles.push(argument);+ }+ }return { baseRef, testFiles };}function main(arguments_) {- const { baseRef, testFiles } = parseArguments(arguments_);let result;try {+ const { baseRef, testFiles } = parseArguments(arguments_);result = runMutationCheck(testFiles, { baseRef });} catch (error) {console.error(error.message);diff --git a/tests/agent-test-commands.test.js b/tests/agent-test-commands.test.jsindex 062635ff..72303287 100644--- a/tests/agent-test-commands.test.js+++ b/tests/agent-test-commands.test.js@@ -157,3 +157,31 @@ test('mutation check arguments keep every test path and read --base', () => {assert.deepEqual(parseArguments(['tests/a.test.js', 'tests/b.test.js']), { baseRef: 'origin/main', testFiles: ['tests/a.test.js', 'tests/b.test.js'] });assert.deepEqual(parseArguments(['tests/a.test.js', '--base', 'main']), { baseRef: 'main', testFiles: ['tests/a.test.js'] });});++test('mutation check reverts a change under scripts/ too', () => {+ const folder = makeRepository();+ try {+ git(folder, 'checkout', '-q', 'main');+ write(folder, 'scripts/answer.mjs', 'export const answer = 1;\n');+ write(folder, 'tests/answer.test.js', "import test from 'node:test';\nimport assert from 'node:assert/strict';\nimport { answer } from '../scripts/answer.mjs';\ntest('answer is two', () => assert.equal(answer, 2));\n");+ git(folder, 'add', '-A');+ git(folder, 'commit', '-q', '-m', 'script and test');+ git(folder, 'checkout', '-q', '-b', 'script-fix');+ write(folder, 'scripts/answer.mjs', 'export const answer = 2;\n');+ git(folder, 'commit', '-q', '-am', 'fix script');+ const result = runMutationCheck(['tests/answer.test.js'], { folder, baseRef: 'main', runTests: (files, options) => runTestFiles(files, { ...options, quiet: true }) });+ assert.deepEqual(result.failingTests.map(failure => failure.name), ['answer is two']);+ assert.equal(readFileSync(join(folder, 'scripts/answer.mjs'), 'utf8'), 'export const answer = 2;\n');+ assert.equal(findBlockingChanges('?? scripts/new.mjs\n').length, 1);+ assert.deepEqual(findBlockingChanges(' M scripts/answer.mjs\n'), []);+ } finally {+ rmSync(folder, { recursive: true, force: true });+ }+});++test('mutation check arguments accept --base=<ref> and refuse a missing ref or an unknown option', () => {+ assert.deepEqual(parseArguments(['tests/a.test.js', '--base=main']), { baseRef: 'main', testFiles: ['tests/a.test.js'] });+ assert.throws(() => parseArguments(['tests/a.test.js', '--base']), /--base needs a ref/);+ assert.throws(() => parseArguments(['--base=', 'tests/a.test.js']), /--base needs a ref/);+ assert.throws(() => parseArguments(['tests/a.test.js', '--bsae', 'main']), /Unknown option --bsae/);+});diff --git a/tests/shard-test-files.test.js b/tests/shard-test-files.test.jsindex f7baa041..85d8f063 100644--- a/tests/shard-test-files.test.js+++ b/tests/shard-test-files.test.js@@ -55,3 +55,23 @@ test('the coverage gate expects as many shards as the workflow matrix has', () =const expected = Number(workflow.match(/coverage-gate\.mjs [^\n]*--expect-shards (\d+)/)[1]);assert.equal(expected, matrix.length);});++test('merging no timings files refuses instead of wiping the timings', async () => {+ const { mergeTimings } = await import('../scripts/shard-test-files.mjs');+ const { mkdtempSync, writeFileSync, rmSync } = await import('node:fs');+ const { tmpdir } = await import('node:os');+ const { join } = await import('node:path');+ const folder = mkdtempSync(join(tmpdir(), 'crucible-timings-'));+ try {+ const destination = join(folder, 'test-timings.json');+ writeFileSync(destination, '{"tests/a.test.js":5}\n');+ assert.throws(() => mergeTimings([], destination), /needs at least one timings file/);+ assert.equal(readFileSync(destination, 'utf8'), '{"tests/a.test.js":5}\n');+ const first = join(folder, 'one.json');+ writeFileSync(first, '{"tests/a.test.js":7,"tests/b.test.js":3}');+ mergeTimings([first], destination);+ assert.deepEqual(JSON.parse(readFileSync(destination, 'utf8')), { 'tests/a.test.js': 7, 'tests/b.test.js': 3 });+ } finally {+ rmSync(folder, { recursive: true, force: true });+ }+});diff --git a/tests/test-hang-guard.test.js b/tests/test-hang-guard.test.jsindex 65a9777e..04ad3fb7 100644--- a/tests/test-hang-guard.test.js+++ b/tests/test-hang-guard.test.js@@ -95,3 +95,9 @@ test('the limits have defaults and reject a value that is not a positive number'assert.throws(() => fileLimitMilliseconds({ CRUCIBLE_TEST_FILE_LIMIT_MS: '-1' }), /must be a positive number of milliseconds/);assert.throws(() => graceAfterTimeoutMilliseconds({ CRUCIBLE_TEST_GRACE_AFTER_TIMEOUT_MS: '0' }), /must be a positive number of milliseconds/);});++test('a limit longer than a timer can hold is refused instead of stopping the file after 1 ms', () => {+ assert.equal(fileLimitMilliseconds({ CRUCIBLE_TEST_FILE_LIMIT_MS: '2147483647' }), 2147483647);+ assert.throws(() => fileLimitMilliseconds({ CRUCIBLE_TEST_FILE_LIMIT_MS: '3000000000' }), /at most 2147483647/);+ assert.throws(() => graceAfterTimeoutMilliseconds({ CRUCIBLE_TEST_GRACE_AFTER_TIMEOUT_MS: '2147483648' }), /at most 2147483647/);+});