☕️ Classic, reliable, trusted test framework for Node.js and the browser
ci: replace Codecov with merged coverage artifacts (#6350)
Mostafa Nazari8eca642addbe
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.mdindex 8baf1517..be923b42 100644--- a/.github/CONTRIBUTING.md+++ b/.github/CONTRIBUTING.md@@ -106,8 +106,7 @@ Feel free to open PRs for changes you think are trivial. If a maintainer believe- This will run both Node.js-based and browser-based tests.- Ultimately, your pull request will be built on our continuous integration servers ([GitHub Actions](https://github.com/mochajs/mocha/actions?query=workflow%3A%22Tests%22)).The first step to ensuring these checks pass is to test on your own machine.- - When tests are run in CI, a coverage check is sent to [Codecov](https://app.codecov.io/gh/mochajs/mocha). You'll need to [add the Codecov GitHub app](https://app.codecov.io/login) to upload these results from your fork. This is recommended but not necessary to open a PR.- **A drop in code coverage % is considered a failed check**.+ - When tests are run in CI, the coverage each test job produces is combined into a single report, downloadable from the workflow run as the `coverage` artifact.1. Commit your changes.- Use a brief message on the first line, referencing a relevant issue (e.g. `closes #12345`).- Add detail in subsequent lines.@@ -167,8 +166,6 @@ Join us in [our Discord](https://discord.gg/KeDn2uXhER)!Mocha uses several external services for project infrastructure:-- **[Codecov](https://app.codecov.io/gh/mochajs/mocha)**: Code coverage tracking. PRs must not decrease coverage. Contributors should add the Codecov GitHub app to upload coverage from forks.-- **[Discord](https://discord.gg/KeDn2uXhER)**: Official community chat for support, discussions, and connecting with maintainers and users.- **[EasyCLA](https://openjsf.org/cla)**: Contributor License Agreement system via Linux Foundation. All code contributions require a signed CLA. A bot prompts contributors on their first PR.diff --git a/.github/workflows/format.yml b/.github/workflows/format.ymlindex e414cfd2..151d9924 100644--- a/.github/workflows/format.yml+++ b/.github/workflows/format.yml@@ -12,8 +12,6 @@ on:permissions:contents: read- id-token: write # for Codecov OIDC (required by npm-script.yml)- issues: writejobs:format:diff --git a/.github/workflows/mocha.yml b/.github/workflows/mocha.ymlindex 1a214367..5f021d35 100644--- a/.github/workflows/mocha.yml+++ b/.github/workflows/mocha.yml@@ -16,8 +16,6 @@ on:permissions:contents: read- id-token: write # for Codecov OIDC- issues: writejobs:lint:@@ -53,9 +51,7 @@ jobs:test-node-all:name: Test ${{ matrix.test-part }} in all environmentspermissions:- id-token: write # for Codecov OIDC- contents: read # for Codecov OIDC- issues: write+ contents: read# TODO: Restore "mocha-github-actions-reporter" style reporting without relying on third party module# https://github.com/mochajs/mocha/issues/5183uses: ./.github/workflows/npm-script.yml@@ -82,6 +78,37 @@ jobs:npm-script: test-node:${{ matrix.test-part }}coverage: ${{ matrix.coverage }}+ coverage:+ name: Merge coverage+ runs-on: ubuntu-latest+ timeout-minutes: 10+ permissions:+ contents: read+ if: ${{ !cancelled() && needs.test-node-all.result != 'skipped' }}+ needs: test-node-all+ steps:+ - uses: actions/checkout@v7+ with:+ persist-credentials: false+ - uses: actions/setup-node@v7+ with:+ node-version: "26"+ - uses: actions/download-artifact@v8+ with:+ path: coverage-reports+ pattern: coverage-*+ - name: Merge coverage reports+ shell: bash+ run: |+ echo '```text' >> "$GITHUB_STEP_SUMMARY"+ node scripts/merge-coverage.js coverage-reports coverage/lcov.info | tee -a "$GITHUB_STEP_SUMMARY"+ echo '```' >> "$GITHUB_STEP_SUMMARY"+ - uses: actions/upload-artifact@v7+ with:+ name: coverage+ path: coverage/lcov.info+ overwrite: true+test-browser:name: Browser testsruns-on: ubuntu-latestdiff --git a/.github/workflows/npm-script.yml b/.github/workflows/npm-script.ymlindex ed7ebcfc..7060ae30 100644--- a/.github/workflows/npm-script.yml+++ b/.github/workflows/npm-script.yml@@ -26,8 +26,6 @@ on:permissions:contents: read- id-token: write- issues: writejobs:resolve-inputs:@@ -35,6 +33,7 @@ jobs:outputs:browsers: ${{ steps.split-browsers.outputs.splitted }}nodeVersions: ${{ steps.split-node-versions.outputs.splitted }}+ npmScript: ${{ steps.sanitize-npm-script.outputs.value }}os: ${{ steps.split-os.outputs.splitted }}steps:- id: split-browsers@@ -43,6 +42,12 @@ jobs:- id: split-node-versionsif: inputs.node-versionsrun: echo "splitted=$(echo '${{ inputs.node-versions }}' | jq -R -c 'split(",")')" >> $GITHUB_OUTPUT+ # Artifact names cannot contain `:` (colon) which every test script uses.+ # Replace `:` with `-` (dash) in artifact names+ - id: sanitize-npm-script+ env:+ NPM_SCRIPT: ${{ inputs.npm-script }}+ run: printf "value=%s\n" "${NPM_SCRIPT//:/-}" >> "$GITHUB_OUTPUT"- id: split-osif: inputs.osrun: echo "splitted=$(echo '${{ inputs.os }}' | jq -R -c 'split(",")')" >> $GITHUB_OUTPUT@@ -77,68 +82,11 @@ jobs:- name: Generate coverage reportif: ${{ !cancelled() && inputs.coverage && (steps.npm-script.conclusion == 'success' || steps.npm-script.conclusion == 'failure') }}run: npm run test-coverage-generate- # https://github.com/marketplace/actions/codecov- # https://app.codecov.io/gh/mochajs/mocha- - id: upload-coverage- name: Upload coverage reports+ - name: Upload coverage reportif: ${{ !cancelled() && inputs.coverage && (steps.npm-script.conclusion == 'success' || steps.npm-script.conclusion == 'failure') }}- continue-on-error: true- uses: codecov/codecov-action@v7- with:- fail_ci_if_error: true- use_oidc: true- - name: Track Codecov failures- if: ${{ !cancelled() && inputs.coverage && (steps.npm-script.conclusion == 'success' || steps.npm-script.conclusion == 'failure') && steps.upload-coverage.outcome == 'failure' }}- uses: actions/github-script@v9+ uses: actions/upload-artifact@v7with:- script: |- const owner = context.repo.owner;- const repo = context.repo.repo;- const issueTitle = 'Codecov upload failed';- const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`;- const issueBody = [- 'Codecov failed to upload coverage reports for this repository.',- '',- `- Workflow run: ${runUrl}`,- `- Commit: ${context.sha}`,- `- Ref: ${context.ref}`,- '',- 'This issue is automatically updated when the upload fails again in CI.',- ].join('\n');-- const { data: issues } = await github.rest.issues.listForRepo({- owner,- repo,- state: 'all',- per_page: 100,- });-- const existingIssue = issues.find(issue => issue.title === issueTitle && !issue.pull_request);-- if (existingIssue) {- if (existingIssue.state === 'closed') {- await github.rest.issues.update({- owner,- repo,- issue_number: existingIssue.number,- state: 'open',- });- }-- await github.rest.issues.createComment({- owner,- repo,- issue_number: existingIssue.number,- body: `Codecov upload failed again.\n\n${issueBody}`,- });-- return;- }-- await github.rest.issues.create({- owner,- repo,- title: issueTitle,- body: issueBody,- labels: ['status: in triage', 'area: repository tooling'],- });+ name: coverage-${{ needs.resolve-inputs.outputs.npmScript }}-${{ matrix.os }}-node-${{ matrix.node_version }}+ path: coverage/lcov.info+ retention-days: 1+ overwrite:truediff --git a/AGENTS.md b/AGENTS.mdindex ea7209f4..3e7332bb 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -108,7 +108,7 @@ Reusable execution details in `.github/workflows/npm-script.yml`:- Uses `npm ci --ignore-scripts`- Default Node 22 unless overridden- `NODE_OPTIONS=--trace-warnings`-- Optional coverage upload via Codecov+- Optional coverage report upload, combined by the `coverage` job in `mocha.yml`## Transient validation issues and environment driftdiff --git a/README.md b/README.mdindex ac7a33dd..0fa3a419 100644--- a/README.md+++ b/README.md@@ -9,7 +9,6 @@<a href="https://www.npmjs.com/package/mocha"><img src="https://img.shields.io/npm/v/mocha.svg" alt="npm version 12.0.3"></a> <!-- x-release-please-version --><a href="https://github.com/mochajs/mocha"><img src="https://img.shields.io/node/v/mocha.svg" alt="Node Version"></a>[](https://github.com/mochajs/mocha/actions/workflows/mocha.yml)-<a href="https://codecov.io/gh/mochajs/mocha"><img src="https://codecov.io/gh/mochajs/mocha/branch/main/graph/badge.svg" alt="Codecov Coverage Status"></a></div>diff --git a/scripts/merge-coverage.js b/scripts/merge-coverage.jsnew file mode 100644index 00000000..ce54c621--- /dev/null+++ b/scripts/merge-coverage.js@@ -0,0 +1,213 @@+import { mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";+import { dirname, join, relative, sep } from "node:path";++// Reports are written per file with / on Linux and \ on Windows.+const toPosix = (path) => path.split("\\").join("/");++// lcov marks a branch that was never reached with "-" rather than 0.+const toHits = (value) => (value === "-" ? 0 : Number(value));++function findReports(dir) {+ return readdirSync(dir, { recursive: true, withFileTypes: true })+ .filter((entry) => entry.isFile() && entry.name.endsWith(".info"))+ .map((entry) => join(entry.parentPath, entry.name))+ .sort();+}++function parseReport(file) {+ const sources = new Map();+ let current = null;+ let functions = [];+ let functionHits = 0;++ const linesSeen = new Map();+ const functionKey = (line) => {+ const seen = linesSeen.get(line) ?? 0;+ linesSeen.set(line, seen + 1);+ return `${line}:${seen}`;+ };++ for (const raw of readFileSync(file, "utf8").split("\n")) {+ const line = raw.trim();+ const value = line.slice(line.indexOf(":") + 1);++ if (line.startsWith("SF:")) {+ const path = toPosix(value);+ if (!sources.has(path)) {+ sources.set(path, {+ branches: new Map(),+ functions: new Map(),+ lines: new Map(),+ });+ }+ current = sources.get(path);+ functions = [];+ functionHits = 0;+ linesSeen.clear();+ } else if (!current) {+ continue;+ } else if (line.startsWith("FN:")) {+ const [start, ...name] = value.split(",");+ functions.push({ key: functionKey(start), name: name.join(","), start });+ } else if (line.startsWith("FNDA:")) {+ const [hits, ...name] = value.split(",");+ const fn = functions[functionHits++];+ if (!fn) {+ throw new Error(+ `${file}: more FNDA than FN records, cannot pair "${name.join(",")}"`,+ );+ }+ fn.hits = toHits(hits);+ } else if (line.startsWith("DA:")) {+ const [start, hits] = value.split(",");+ current.lines.set(start, (current.lines.get(start) ?? 0) + toHits(hits));+ } else if (line.startsWith("BRDA:")) {+ const [start, block, branch, taken] = value.split(",");+ const key = `${start},${block},${branch}`;+ current.branches.set(+ key,+ (current.branches.get(key) ?? 0) + toHits(taken),+ );+ } else if (line === "end_of_record") {+ for (const fn of functions) {+ const merged = current.functions.get(fn.key);+ if (merged) {+ merged.hits += fn.hits ?? 0;+ } else {+ current.functions.set(fn.key, {+ hits: fn.hits ?? 0,+ name: fn.name,+ start: fn.start,+ });+ }+ }+ current = null;+ }+ }++ return sources;+}++function mergeInto(into, from) {+ for (const [path, source] of from) {+ const target = into.get(path);+ if (!target) {+ into.set(path, source);+ continue;+ }++ for (const [line, hits] of source.lines) {+ target.lines.set(line, (target.lines.get(line) ?? 0) + hits);+ }+ for (const [key, taken] of source.branches) {+ target.branches.set(key, (target.branches.get(key) ?? 0) + taken);+ }+ for (const [key, fn] of source.functions) {+ const merged = target.functions.get(key);+ if (merged) {+ merged.hits += fn.hits;+ } else {+ target.functions.set(key, fn);+ }+ }+ }+}++function formatReport(sources) {+ const byNumber = (a, b) => Number(a[0]) - Number(b[0]);+ const out = [];++ for (const path of [...sources.keys()].sort()) {+ const { branches, functions, lines } = sources.get(path);+ const fns = [...functions.values()].sort(+ (a, b) => Number(a.start) - Number(b.start),+ );++ out.push("TN:", `SF:${path}`);+ for (const fn of fns) out.push(`FN:${fn.start},${fn.name}`);+ out.push(+ `FNF:${fns.length}`,+ `FNH:${fns.filter((fn) => fn.hits > 0).length}`,+ );+ for (const fn of fns) out.push(`FNDA:${fn.hits},${fn.name}`);++ const daLines = [...lines.entries()].sort(byNumber);+ for (const [line, hits] of daLines) out.push(`DA:${line},${hits}`);+ out.push(+ `LF:${daLines.length}`,+ `LH:${daLines.filter(([, hits]) => hits > 0).length}`,+ );++ const brdaLines = [...branches.entries()].sort((a, b) =>+ a[0].localeCompare(b[0], "en", { numeric: true }),+ );+ for (const [key, taken] of brdaLines) out.push(`BRDA:${key},${taken}`);+ out.push(+ `BRF:${brdaLines.length}`,+ `BRH:${brdaLines.filter(([, taken]) => taken > 0).length}`,+ "end_of_record",+ );+ }++ return out.join("\n") + "\n";+}++function summarize(sources) {+ const totals = { branches: [0, 0], functions: [0, 0], lines: [0, 0] };++ for (const source of sources.values()) {+ for (const [name, values] of [+ ["branches", source.branches.values()],+ ["functions", [...source.functions.values()].map((fn) => fn.hits)],+ ["lines", source.lines.values()],+ ]) {+ for (const hits of values) {+ totals[name][1]++;+ if (hits > 0) totals[name][0]++;+ }+ }+ }++ return totals;+}++const [inputDir, outputFile] = process.argv.slice(2);+if (!inputDir || !outputFile) {+ console.error(+ "Usage: node scripts/merge-coverage.js <input-dir> <output-file>",+ );+ process.exit(1);+}++const reports = findReports(inputDir);+if (reports.length === 0) {+ console.error(`No coverage reports found in ${inputDir}`);+ process.exit(1);+}++const merged = new Map();+for (const report of reports) {+ const sources = parseReport(report);+ if (sources.size === 0) {+ console.warn(`Empty coverage report: ${relative(inputDir, report)}`);+ }+ mergeInto(merged, sources);+}++if (merged.size === 0) {+ console.error(`Coverage reports in ${inputDir} contain no coverage data`);+ process.exit(1);+}++mkdirSync(dirname(outputFile), { recursive: true });+writeFileSync(outputFile, formatReport(merged));++const totals = summarize(merged);+console.log(+ `Merged ${reports.length} coverage reports covering ${merged.size} files ` ++ `into ${toPosix(outputFile.split(sep).join("/"))}`,+);+for (const [name, [covered, total]] of Object.entries(totals)) {+ const pct = total === 0 ? 100 : (covered / total) * 100;+ console.log(` ${name.padEnd(9)} ${pct.toFixed(2)}% (${covered}/${total})`);+}diff --git a/test/node-unit/merge-coverage.spec.cjs b/test/node-unit/merge-coverage.spec.cjsnew file mode 100644index 00000000..7d376655--- /dev/null+++ b/test/node-unit/merge-coverage.spec.cjs@@ -0,0 +1,116 @@+"use strict";++const { spawnSync } = require("node:child_process");+const fs = require("node:fs");+const os = require("node:os");+const path = require("node:path");++const script = path.join(__dirname, "..", "..", "scripts", "merge-coverage.js");++const reports = {+ "a.info": [+ "TN:",+ "SF:lib/a.js",+ "FN:5,foo",+ "FNF:1",+ "FNH:1",+ "FNDA:1,foo",+ "DA:5,1",+ "DA:6,0",+ "LF:2",+ "LH:1",+ "BRDA:5,0,0,1",+ "BRDA:5,0,1,0",+ "BRF:2",+ "BRH:1",+ "end_of_record",+ ].join("\n"),+ "b.info": [+ "TN:",+ "SF:lib\\a.js",+ "FN:5,foo$2",+ "FNF:1",+ "FNH:1",+ "FNDA:2,foo$2",+ "DA:5,2",+ "DA:6,0",+ "LF:2",+ "LH:1",+ "BRDA:5,0,0,2",+ "BRDA:5,0,1,-",+ "BRF:2",+ "BRH:1",+ "end_of_record",+ ].join("\n"),+};++describe("merge-coverage", function () {+ this.timeout(10000);++ let tmpDir;++ const merge = (inputDir) => {+ const output = path.join(tmpDir, "merged.info");+ const result = spawnSync(process.execPath, [script, inputDir, output], {+ encoding: "utf8",+ });+ return { ...result, output };+ };++ beforeEach(function () {+ tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "mocha-merge-coverage-"));+ });++ afterEach(function () {+ fs.rmSync(tmpDir, { force: true, recursive: true });+ });++ it("should combine reports of the same file from every job", function () {+ for (const [name, contents] of Object.entries(reports)) {+ const dir = path.join(tmpDir, `coverage-${path.parse(name).name}`);+ fs.mkdirSync(dir);+ fs.writeFileSync(path.join(dir, name), contents);+ }++ const { output, status } = merge(tmpDir);++ expect(status, "to be", 0);+ expect(fs.readFileSync(output, "utf8").trim().split("\n"), "to equal", [+ "TN:",+ "SF:lib/a.js",+ "FN:5,foo",+ "FNF:1",+ "FNH:1",+ "FNDA:3,foo",+ "DA:5,3",+ "DA:6,0",+ "LF:2",+ "LH:1",+ "BRDA:5,0,0,3",+ "BRDA:5,0,1,0",+ "BRF:2",+ "BRH:1",+ "end_of_record",+ ]);+ });++ it("should fail when no reports were collected", function () {+ const { output, status, stderr } = merge(tmpDir);++ expect(status, "to be", 1);+ expect(stderr, "to contain", "No coverage reports found");+ expect(fs.existsSync(output), "to be false");+ });++ it("should fail when the reports contain no coverage data", function () {+ const dir = path.join(tmpDir, "coverage-empty");+ fs.mkdirSync(dir);+ fs.writeFileSync(path.join(dir, "lcov.info"), "");++ const { output, status, stderr } = merge(tmpDir);++ expect(status, "to be", 1);+ expect(stderr, "to contain", "contain no coverage data");+ expect(fs.existsSync(output), "to be false");+ });+});