3D engine that AI agents drive
Give branch coverage room for run-to-run noise in the gate (issue #563)
Nguyen Xuan Anf9a1b7abd373
diff --git a/changes/563.md b/changes/563.mdindex ee3b62b1..8c468ccf 100644--- a/changes/563.md+++ b/changes/563.md@@ -1 +1,3 @@Run the unit tests in eight shards so a CI run stays under 2.5 minutes (#563).++The coverage gate now allows a 1.5 point drop on branches (lines and functions stay at 0.5), because branch totals move by up to about 0.8 points between runs of the same commit; `tests/coverage-gate.test.js` covers both allowances.diff --git a/docs/ci.md b/docs/ci.mdindex 4f3d9d7a..b22dc31d 100644--- a/docs/ci.md+++ b/docs/ci.md@@ -19,7 +19,7 @@ All jobs start at once.## Coverage-Coverage comes from Node's built-in `--experimental-test-coverage`. The `coverage` job fails when line, branch or function coverage falls more than 0.5 points below `coverage-baseline.json`. Raise the baseline by editing that file; lower it only on purpose and say why in the pull request. To reproduce locally: `npm run test:coverage -- --test-reporter=lcov --test-reporter-destination=coverage/lcov.info`, then `npm run coverage:gate -- coverage`.+Coverage comes from Node's built-in `--experimental-test-coverage`. The `coverage` job fails when line or function coverage falls more than 0.5 points, or branch coverage more than 1.5 points, below `coverage-baseline.json` (branch totals move by up to about 0.8 points between runs of the same commit). Raise the baseline by editing that file; lower it only on purpose and say why in the pull request. To reproduce locally: `npm run test:coverage -- --test-reporter=lcov --test-reporter-destination=coverage/lcov.info`, then `npm run coverage:gate -- coverage`.## Add a job or a smokediff --git a/scripts/coverage-gate.mjs b/scripts/coverage-gate.mjsindex 14006d9d..3445c63b 100644--- a/scripts/coverage-gate.mjs+++ b/scripts/coverage-gate.mjs@@ -6,7 +6,9 @@ import { appendFileSync, readdirSync, readFileSync, statSync, writeFileSync } frimport { join, resolve } from 'node:path';import { fileURLToPath } from 'node:url';-const ALLOWED_DROP_POINTS = 0.5;+// Branch totals move by up to about 0.8 points between runs of the same commit (merged V8 block ranges across shards),+// so branches get more room than lines and functions, which stay within a few hundredths.+const ALLOWED_DROP_POINTS = Object.freeze({ lines: 0.5, branches: 1.5, functions: 0.5 });const METRICS = ['lines', 'branches', 'functions'];function collectLcovPaths(inputs) {@@ -94,7 +96,7 @@ export function compareWithBaseline(percentages, baseline) {metric,actual: percentages[metric],baseline: baseline[metric],- ok: percentages[metric] >= baseline[metric] - ALLOWED_DROP_POINTS,+ ok: percentages[metric] >= baseline[metric] - ALLOWED_DROP_POINTS[metric],}));}@@ -130,7 +132,7 @@ function main() {const results = compareWithBaseline(percentages, baseline);const table = [- `Coverage from ${paths.length} lcov file(s), ${files.size} source files. A drop of more than ${ALLOWED_DROP_POINTS} points below the baseline fails.`,+ `Coverage from ${paths.length} lcov file(s), ${files.size} source files. A drop below the baseline of more than ${ALLOWED_DROP_POINTS.lines} points (lines, functions) or ${ALLOWED_DROP_POINTS.branches} points (branches) fails.`,'','| Metric | Covered | Total | Actual % | Baseline % | Result |','| --- | ---: | ---: | ---: | ---: | --- |',diff --git a/tests/coverage-gate.test.js b/tests/coverage-gate.test.jsnew file mode 100644index 00000000..9e9231d5--- /dev/null+++ b/tests/coverage-gate.test.js@@ -0,0 +1,17 @@+import test from 'node:test';+import assert from 'node:assert/strict';+import { compareWithBaseline } from '../scripts/coverage-gate.mjs';++const baseline = { lines: 90, branches: 80, functions: 90 };++function verdicts(percentages) {+ return Object.fromEntries(compareWithBaseline(percentages, baseline).map(result => [result.metric, result.ok]));+}++test('branch coverage may wander by run-to-run noise without failing the gate', () => {+ assert.deepEqual(verdicts({ lines: 90, branches: 78.6, functions: 90 }), { lines: true, branches: true, functions: true });+});++test('a real drop fails each metric past its own allowance', () => {+ assert.deepEqual(verdicts({ lines: 89.4, branches: 78.4, functions: 89.4 }), { lines: false, branches: false, functions: false });+});