Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Give branch coverage room for run-to-run noise in the gate (issue #563)

Nguyen Xuan Anf9a1b7abd373
diff --git a/changes/563.md b/changes/563.md
index ee3b62b1..8c468ccf 100644
--- a/changes/563.md
+++ b/changes/563.md
@@ -1 +1,3 @@
Run the unit tests in eight shards so a CI run stays under 2.5 minutes (#563).
+
+The coverage gate now allows a 1.5 point drop on branches (lines and functions stay at 0.5), because branch totals move by up to about 0.8 points between runs of the same commit; `tests/coverage-gate.test.js` covers both allowances.
diff --git a/docs/ci.md b/docs/ci.md
index 4f3d9d7a..b22dc31d 100644
--- a/docs/ci.md
+++ b/docs/ci.md
@@ -19,7 +19,7 @@ All jobs start at once.
## Coverage
-Coverage comes from Node's built-in `--experimental-test-coverage`. The `coverage` job fails when line, branch or function coverage falls more than 0.5 points below `coverage-baseline.json`. Raise the baseline by editing that file; lower it only on purpose and say why in the pull request. To reproduce locally: `npm run test:coverage -- --test-reporter=lcov --test-reporter-destination=coverage/lcov.info`, then `npm run coverage:gate -- coverage`.
+Coverage comes from Node's built-in `--experimental-test-coverage`. The `coverage` job fails when line or function coverage falls more than 0.5 points, or branch coverage more than 1.5 points, below `coverage-baseline.json` (branch totals move by up to about 0.8 points between runs of the same commit). Raise the baseline by editing that file; lower it only on purpose and say why in the pull request. To reproduce locally: `npm run test:coverage -- --test-reporter=lcov --test-reporter-destination=coverage/lcov.info`, then `npm run coverage:gate -- coverage`.
## Add a job or a smoke
diff --git a/scripts/coverage-gate.mjs b/scripts/coverage-gate.mjs
index 14006d9d..3445c63b 100644
--- a/scripts/coverage-gate.mjs
+++ b/scripts/coverage-gate.mjs
@@ -6,7 +6,9 @@ import { appendFileSync, readdirSync, readFileSync, statSync, writeFileSync } fr
import { join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
-const ALLOWED_DROP_POINTS = 0.5;
+// Branch totals move by up to about 0.8 points between runs of the same commit (merged V8 block ranges across shards),
+// so branches get more room than lines and functions, which stay within a few hundredths.
+const ALLOWED_DROP_POINTS = Object.freeze({ lines: 0.5, branches: 1.5, functions: 0.5 });
const METRICS = ['lines', 'branches', 'functions'];
function collectLcovPaths(inputs) {
@@ -94,7 +96,7 @@ export function compareWithBaseline(percentages, baseline) {
metric,
actual: percentages[metric],
baseline: baseline[metric],
- ok: percentages[metric] >= baseline[metric] - ALLOWED_DROP_POINTS,
+ ok: percentages[metric] >= baseline[metric] - ALLOWED_DROP_POINTS[metric],
}));
}
@@ -130,7 +132,7 @@ function main() {
const results = compareWithBaseline(percentages, baseline);
const table = [
- `Coverage from ${paths.length} lcov file(s), ${files.size} source files. A drop of more than ${ALLOWED_DROP_POINTS} points below the baseline fails.`,
+ `Coverage from ${paths.length} lcov file(s), ${files.size} source files. A drop below the baseline of more than ${ALLOWED_DROP_POINTS.lines} points (lines, functions) or ${ALLOWED_DROP_POINTS.branches} points (branches) fails.`,
'',
'| Metric | Covered | Total | Actual % | Baseline % | Result |',
'| --- | ---: | ---: | ---: | ---: | --- |',
diff --git a/tests/coverage-gate.test.js b/tests/coverage-gate.test.js
new file mode 100644
index 00000000..9e9231d5
--- /dev/null
+++ b/tests/coverage-gate.test.js
@@ -0,0 +1,17 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { compareWithBaseline } from '../scripts/coverage-gate.mjs';
+
+const baseline = { lines: 90, branches: 80, functions: 90 };
+
+function verdicts(percentages) {
+ return Object.fromEntries(compareWithBaseline(percentages, baseline).map(result => [result.metric, result.ok]));
+}
+
+test('branch coverage may wander by run-to-run noise without failing the gate', () => {
+ assert.deepEqual(verdicts({ lines: 90, branches: 78.6, functions: 90 }), { lines: true, branches: true, functions: true });
+});
+
+test('a real drop fails each metric past its own allowance', () => {
+ assert.deepEqual(verdicts({ lines: 89.4, branches: 78.4, functions: 89.4 }), { lines: false, branches: false, functions: false });
+});