Skip to content

armgit / mocha

Public Imported from GitHub
0

☕️ Classic, reliable, trusted test framework for Node.js and the browser

ci: replace Codecov with merged coverage artifacts (#6350)

Mostafa Nazari8eca642addbe
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
index 8baf1517..be923b42 100644
--- a/.github/CONTRIBUTING.md
+++ b/.github/CONTRIBUTING.md
@@ -106,8 +106,7 @@ Feel free to open PRs for changes you think are trivial. If a maintainer believe
- This will run both Node.js-based and browser-based tests.
- Ultimately, your pull request will be built on our continuous integration servers ([GitHub Actions](https://github.com/mochajs/mocha/actions?query=workflow%3A%22Tests%22)).
The first step to ensuring these checks pass is to test on your own machine.
- - When tests are run in CI, a coverage check is sent to [Codecov](https://app.codecov.io/gh/mochajs/mocha). You'll need to [add the Codecov GitHub app](https://app.codecov.io/login) to upload these results from your fork. This is recommended but not necessary to open a PR.
- **A drop in code coverage % is considered a failed check**.
+ - When tests are run in CI, the coverage each test job produces is combined into a single report, downloadable from the workflow run as the `coverage` artifact.
1. Commit your changes.
- Use a brief message on the first line, referencing a relevant issue (e.g. `closes #12345`).
- Add detail in subsequent lines.
@@ -167,8 +166,6 @@ Join us in [our Discord](https://discord.gg/KeDn2uXhER)!
Mocha uses several external services for project infrastructure:
-- **[Codecov](https://app.codecov.io/gh/mochajs/mocha)**: Code coverage tracking. PRs must not decrease coverage. Contributors should add the Codecov GitHub app to upload coverage from forks.
-
- **[Discord](https://discord.gg/KeDn2uXhER)**: Official community chat for support, discussions, and connecting with maintainers and users.
- **[EasyCLA](https://openjsf.org/cla)**: Contributor License Agreement system via Linux Foundation. All code contributions require a signed CLA. A bot prompts contributors on their first PR.
diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml
index e414cfd2..151d9924 100644
--- a/.github/workflows/format.yml
+++ b/.github/workflows/format.yml
@@ -12,8 +12,6 @@ on:
permissions:
contents: read
- id-token: write # for Codecov OIDC (required by npm-script.yml)
- issues: write
jobs:
format:
diff --git a/.github/workflows/mocha.yml b/.github/workflows/mocha.yml
index 1a214367..5f021d35 100644
--- a/.github/workflows/mocha.yml
+++ b/.github/workflows/mocha.yml
@@ -16,8 +16,6 @@ on:
permissions:
contents: read
- id-token: write # for Codecov OIDC
- issues: write
jobs:
lint:
@@ -53,9 +51,7 @@ jobs:
test-node-all:
name: Test ${{ matrix.test-part }} in all environments
permissions:
- id-token: write # for Codecov OIDC
- contents: read # for Codecov OIDC
- issues: write
+ contents: read
# TODO: Restore "mocha-github-actions-reporter" style reporting without relying on third party module
# https://github.com/mochajs/mocha/issues/5183
uses: ./.github/workflows/npm-script.yml
@@ -82,6 +78,37 @@ jobs:
npm-script: test-node:${{ matrix.test-part }}
coverage: ${{ matrix.coverage }}
+ coverage:
+ name: Merge coverage
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ permissions:
+ contents: read
+ if: ${{ !cancelled() && needs.test-node-all.result != 'skipped' }}
+ needs: test-node-all
+ steps:
+ - uses: actions/checkout@v7
+ with:
+ persist-credentials: false
+ - uses: actions/setup-node@v7
+ with:
+ node-version: "26"
+ - uses: actions/download-artifact@v8
+ with:
+ path: coverage-reports
+ pattern: coverage-*
+ - name: Merge coverage reports
+ shell: bash
+ run: |
+ echo '```text' >> "$GITHUB_STEP_SUMMARY"
+ node scripts/merge-coverage.js coverage-reports coverage/lcov.info | tee -a "$GITHUB_STEP_SUMMARY"
+ echo '```' >> "$GITHUB_STEP_SUMMARY"
+ - uses: actions/upload-artifact@v7
+ with:
+ name: coverage
+ path: coverage/lcov.info
+ overwrite: true
+
test-browser:
name: Browser tests
runs-on: ubuntu-latest
diff --git a/.github/workflows/npm-script.yml b/.github/workflows/npm-script.yml
index ed7ebcfc..7060ae30 100644
--- a/.github/workflows/npm-script.yml
+++ b/.github/workflows/npm-script.yml
@@ -26,8 +26,6 @@ on:
permissions:
contents: read
- id-token: write
- issues: write
jobs:
resolve-inputs:
@@ -35,6 +33,7 @@ jobs:
outputs:
browsers: ${{ steps.split-browsers.outputs.splitted }}
nodeVersions: ${{ steps.split-node-versions.outputs.splitted }}
+ npmScript: ${{ steps.sanitize-npm-script.outputs.value }}
os: ${{ steps.split-os.outputs.splitted }}
steps:
- id: split-browsers
@@ -43,6 +42,12 @@ jobs:
- id: split-node-versions
if: inputs.node-versions
run: echo "splitted=$(echo '${{ inputs.node-versions }}' | jq -R -c 'split(",")')" >> $GITHUB_OUTPUT
+ # Artifact names cannot contain `:` (colon) which every test script uses.
+ # Replace `:` with `-` (dash) in artifact names
+ - id: sanitize-npm-script
+ env:
+ NPM_SCRIPT: ${{ inputs.npm-script }}
+ run: printf "value=%s\n" "${NPM_SCRIPT//:/-}" >> "$GITHUB_OUTPUT"
- id: split-os
if: inputs.os
run: echo "splitted=$(echo '${{ inputs.os }}' | jq -R -c 'split(",")')" >> $GITHUB_OUTPUT
@@ -77,68 +82,11 @@ jobs:
- name: Generate coverage report
if: ${{ !cancelled() && inputs.coverage && (steps.npm-script.conclusion == 'success' || steps.npm-script.conclusion == 'failure') }}
run: npm run test-coverage-generate
- # https://github.com/marketplace/actions/codecov
- # https://app.codecov.io/gh/mochajs/mocha
- - id: upload-coverage
- name: Upload coverage reports
+ - name: Upload coverage report
if: ${{ !cancelled() && inputs.coverage && (steps.npm-script.conclusion == 'success' || steps.npm-script.conclusion == 'failure') }}
- continue-on-error: true
- uses: codecov/codecov-action@v7
- with:
- fail_ci_if_error: true
- use_oidc: true
- - name: Track Codecov failures
- if: ${{ !cancelled() && inputs.coverage && (steps.npm-script.conclusion == 'success' || steps.npm-script.conclusion == 'failure') && steps.upload-coverage.outcome == 'failure' }}
- uses: actions/github-script@v9
+ uses: actions/upload-artifact@v7
with:
- script: |
- const owner = context.repo.owner;
- const repo = context.repo.repo;
- const issueTitle = 'Codecov upload failed';
- const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`;
- const issueBody = [
- 'Codecov failed to upload coverage reports for this repository.',
- '',
- `- Workflow run: ${runUrl}`,
- `- Commit: ${context.sha}`,
- `- Ref: ${context.ref}`,
- '',
- 'This issue is automatically updated when the upload fails again in CI.',
- ].join('\n');
-
- const { data: issues } = await github.rest.issues.listForRepo({
- owner,
- repo,
- state: 'all',
- per_page: 100,
- });
-
- const existingIssue = issues.find(issue => issue.title === issueTitle && !issue.pull_request);
-
- if (existingIssue) {
- if (existingIssue.state === 'closed') {
- await github.rest.issues.update({
- owner,
- repo,
- issue_number: existingIssue.number,
- state: 'open',
- });
- }
-
- await github.rest.issues.createComment({
- owner,
- repo,
- issue_number: existingIssue.number,
- body: `Codecov upload failed again.\n\n${issueBody}`,
- });
-
- return;
- }
-
- await github.rest.issues.create({
- owner,
- repo,
- title: issueTitle,
- body: issueBody,
- labels: ['status: in triage', 'area: repository tooling'],
- });
+ name: coverage-${{ needs.resolve-inputs.outputs.npmScript }}-${{ matrix.os }}-node-${{ matrix.node_version }}
+ path: coverage/lcov.info
+ retention-days: 1
+ overwrite:true
diff --git a/AGENTS.md b/AGENTS.md
index ea7209f4..3e7332bb 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -108,7 +108,7 @@ Reusable execution details in `.github/workflows/npm-script.yml`:
- Uses `npm ci --ignore-scripts`
- Default Node 22 unless overridden
- `NODE_OPTIONS=--trace-warnings`
-- Optional coverage upload via Codecov
+- Optional coverage report upload, combined by the `coverage` job in `mocha.yml`
## Transient validation issues and environment drift
diff --git a/README.md b/README.md
index ac7a33dd..0fa3a419 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,6 @@
<a href="https://www.npmjs.com/package/mocha"><img src="https://img.shields.io/npm/v/mocha.svg" alt="npm version 12.0.3"></a> <!-- x-release-please-version -->
<a href="https://github.com/mochajs/mocha"><img src="https://img.shields.io/node/v/mocha.svg" alt="Node Version"></a>
[![GitHub Actions Build Status](https://github.com/mochajs/mocha/actions/workflows/mocha.yml/badge.svg)](https://github.com/mochajs/mocha/actions/workflows/mocha.yml)
-<a href="https://codecov.io/gh/mochajs/mocha"><img src="https://codecov.io/gh/mochajs/mocha/branch/main/graph/badge.svg" alt="Codecov Coverage Status"></a>
</div>
diff --git a/scripts/merge-coverage.js b/scripts/merge-coverage.js
new file mode 100644
index 00000000..ce54c621
--- /dev/null
+++ b/scripts/merge-coverage.js
@@ -0,0 +1,213 @@
+import { mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join, relative, sep } from "node:path";
+
+// Reports are written per file with / on Linux and \ on Windows.
+const toPosix = (path) => path.split("\\").join("/");
+
+// lcov marks a branch that was never reached with "-" rather than 0.
+const toHits = (value) => (value === "-" ? 0 : Number(value));
+
+function findReports(dir) {
+ return readdirSync(dir, { recursive: true, withFileTypes: true })
+ .filter((entry) => entry.isFile() && entry.name.endsWith(".info"))
+ .map((entry) => join(entry.parentPath, entry.name))
+ .sort();
+}
+
+function parseReport(file) {
+ const sources = new Map();
+ let current = null;
+ let functions = [];
+ let functionHits = 0;
+
+ const linesSeen = new Map();
+ const functionKey = (line) => {
+ const seen = linesSeen.get(line) ?? 0;
+ linesSeen.set(line, seen + 1);
+ return `${line}:${seen}`;
+ };
+
+ for (const raw of readFileSync(file, "utf8").split("\n")) {
+ const line = raw.trim();
+ const value = line.slice(line.indexOf(":") + 1);
+
+ if (line.startsWith("SF:")) {
+ const path = toPosix(value);
+ if (!sources.has(path)) {
+ sources.set(path, {
+ branches: new Map(),
+ functions: new Map(),
+ lines: new Map(),
+ });
+ }
+ current = sources.get(path);
+ functions = [];
+ functionHits = 0;
+ linesSeen.clear();
+ } else if (!current) {
+ continue;
+ } else if (line.startsWith("FN:")) {
+ const [start, ...name] = value.split(",");
+ functions.push({ key: functionKey(start), name: name.join(","), start });
+ } else if (line.startsWith("FNDA:")) {
+ const [hits, ...name] = value.split(",");
+ const fn = functions[functionHits++];
+ if (!fn) {
+ throw new Error(
+ `${file}: more FNDA than FN records, cannot pair "${name.join(",")}"`,
+ );
+ }
+ fn.hits = toHits(hits);
+ } else if (line.startsWith("DA:")) {
+ const [start, hits] = value.split(",");
+ current.lines.set(start, (current.lines.get(start) ?? 0) + toHits(hits));
+ } else if (line.startsWith("BRDA:")) {
+ const [start, block, branch, taken] = value.split(",");
+ const key = `${start},${block},${branch}`;
+ current.branches.set(
+ key,
+ (current.branches.get(key) ?? 0) + toHits(taken),
+ );
+ } else if (line === "end_of_record") {
+ for (const fn of functions) {
+ const merged = current.functions.get(fn.key);
+ if (merged) {
+ merged.hits += fn.hits ?? 0;
+ } else {
+ current.functions.set(fn.key, {
+ hits: fn.hits ?? 0,
+ name: fn.name,
+ start: fn.start,
+ });
+ }
+ }
+ current = null;
+ }
+ }
+
+ return sources;
+}
+
+function mergeInto(into, from) {
+ for (const [path, source] of from) {
+ const target = into.get(path);
+ if (!target) {
+ into.set(path, source);
+ continue;
+ }
+
+ for (const [line, hits] of source.lines) {
+ target.lines.set(line, (target.lines.get(line) ?? 0) + hits);
+ }
+ for (const [key, taken] of source.branches) {
+ target.branches.set(key, (target.branches.get(key) ?? 0) + taken);
+ }
+ for (const [key, fn] of source.functions) {
+ const merged = target.functions.get(key);
+ if (merged) {
+ merged.hits += fn.hits;
+ } else {
+ target.functions.set(key, fn);
+ }
+ }
+ }
+}
+
+function formatReport(sources) {
+ const byNumber = (a, b) => Number(a[0]) - Number(b[0]);
+ const out = [];
+
+ for (const path of [...sources.keys()].sort()) {
+ const { branches, functions, lines } = sources.get(path);
+ const fns = [...functions.values()].sort(
+ (a, b) => Number(a.start) - Number(b.start),
+ );
+
+ out.push("TN:", `SF:${path}`);
+ for (const fn of fns) out.push(`FN:${fn.start},${fn.name}`);
+ out.push(
+ `FNF:${fns.length}`,
+ `FNH:${fns.filter((fn) => fn.hits > 0).length}`,
+ );
+ for (const fn of fns) out.push(`FNDA:${fn.hits},${fn.name}`);
+
+ const daLines = [...lines.entries()].sort(byNumber);
+ for (const [line, hits] of daLines) out.push(`DA:${line},${hits}`);
+ out.push(
+ `LF:${daLines.length}`,
+ `LH:${daLines.filter(([, hits]) => hits > 0).length}`,
+ );
+
+ const brdaLines = [...branches.entries()].sort((a, b) =>
+ a[0].localeCompare(b[0], "en", { numeric: true }),
+ );
+ for (const [key, taken] of brdaLines) out.push(`BRDA:${key},${taken}`);
+ out.push(
+ `BRF:${brdaLines.length}`,
+ `BRH:${brdaLines.filter(([, taken]) => taken > 0).length}`,
+ "end_of_record",
+ );
+ }
+
+ return out.join("\n") + "\n";
+}
+
+function summarize(sources) {
+ const totals = { branches: [0, 0], functions: [0, 0], lines: [0, 0] };
+
+ for (const source of sources.values()) {
+ for (const [name, values] of [
+ ["branches", source.branches.values()],
+ ["functions", [...source.functions.values()].map((fn) => fn.hits)],
+ ["lines", source.lines.values()],
+ ]) {
+ for (const hits of values) {
+ totals[name][1]++;
+ if (hits > 0) totals[name][0]++;
+ }
+ }
+ }
+
+ return totals;
+}
+
+const [inputDir, outputFile] = process.argv.slice(2);
+if (!inputDir || !outputFile) {
+ console.error(
+ "Usage: node scripts/merge-coverage.js <input-dir> <output-file>",
+ );
+ process.exit(1);
+}
+
+const reports = findReports(inputDir);
+if (reports.length === 0) {
+ console.error(`No coverage reports found in ${inputDir}`);
+ process.exit(1);
+}
+
+const merged = new Map();
+for (const report of reports) {
+ const sources = parseReport(report);
+ if (sources.size === 0) {
+ console.warn(`Empty coverage report: ${relative(inputDir, report)}`);
+ }
+ mergeInto(merged, sources);
+}
+
+if (merged.size === 0) {
+ console.error(`Coverage reports in ${inputDir} contain no coverage data`);
+ process.exit(1);
+}
+
+mkdirSync(dirname(outputFile), { recursive: true });
+writeFileSync(outputFile, formatReport(merged));
+
+const totals = summarize(merged);
+console.log(
+ `Merged ${reports.length} coverage reports covering ${merged.size} files ` +
+ `into ${toPosix(outputFile.split(sep).join("/"))}`,
+);
+for (const [name, [covered, total]] of Object.entries(totals)) {
+ const pct = total === 0 ? 100 : (covered / total) * 100;
+ console.log(` ${name.padEnd(9)} ${pct.toFixed(2)}% (${covered}/${total})`);
+}
diff --git a/test/node-unit/merge-coverage.spec.cjs b/test/node-unit/merge-coverage.spec.cjs
new file mode 100644
index 00000000..7d376655
--- /dev/null
+++ b/test/node-unit/merge-coverage.spec.cjs
@@ -0,0 +1,116 @@
+"use strict";
+
+const { spawnSync } = require("node:child_process");
+const fs = require("node:fs");
+const os = require("node:os");
+const path = require("node:path");
+
+const script = path.join(__dirname, "..", "..", "scripts", "merge-coverage.js");
+
+const reports = {
+ "a.info": [
+ "TN:",
+ "SF:lib/a.js",
+ "FN:5,foo",
+ "FNF:1",
+ "FNH:1",
+ "FNDA:1,foo",
+ "DA:5,1",
+ "DA:6,0",
+ "LF:2",
+ "LH:1",
+ "BRDA:5,0,0,1",
+ "BRDA:5,0,1,0",
+ "BRF:2",
+ "BRH:1",
+ "end_of_record",
+ ].join("\n"),
+ "b.info": [
+ "TN:",
+ "SF:lib\\a.js",
+ "FN:5,foo$2",
+ "FNF:1",
+ "FNH:1",
+ "FNDA:2,foo$2",
+ "DA:5,2",
+ "DA:6,0",
+ "LF:2",
+ "LH:1",
+ "BRDA:5,0,0,2",
+ "BRDA:5,0,1,-",
+ "BRF:2",
+ "BRH:1",
+ "end_of_record",
+ ].join("\n"),
+};
+
+describe("merge-coverage", function () {
+ this.timeout(10000);
+
+ let tmpDir;
+
+ const merge = (inputDir) => {
+ const output = path.join(tmpDir, "merged.info");
+ const result = spawnSync(process.execPath, [script, inputDir, output], {
+ encoding: "utf8",
+ });
+ return { ...result, output };
+ };
+
+ beforeEach(function () {
+ tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "mocha-merge-coverage-"));
+ });
+
+ afterEach(function () {
+ fs.rmSync(tmpDir, { force: true, recursive: true });
+ });
+
+ it("should combine reports of the same file from every job", function () {
+ for (const [name, contents] of Object.entries(reports)) {
+ const dir = path.join(tmpDir, `coverage-${path.parse(name).name}`);
+ fs.mkdirSync(dir);
+ fs.writeFileSync(path.join(dir, name), contents);
+ }
+
+ const { output, status } = merge(tmpDir);
+
+ expect(status, "to be", 0);
+ expect(fs.readFileSync(output, "utf8").trim().split("\n"), "to equal", [
+ "TN:",
+ "SF:lib/a.js",
+ "FN:5,foo",
+ "FNF:1",
+ "FNH:1",
+ "FNDA:3,foo",
+ "DA:5,3",
+ "DA:6,0",
+ "LF:2",
+ "LH:1",
+ "BRDA:5,0,0,3",
+ "BRDA:5,0,1,0",
+ "BRF:2",
+ "BRH:1",
+ "end_of_record",
+ ]);
+ });
+
+ it("should fail when no reports were collected", function () {
+ const { output, status, stderr } = merge(tmpDir);
+
+ expect(status, "to be", 1);
+ expect(stderr, "to contain", "No coverage reports found");
+ expect(fs.existsSync(output), "to be false");
+ });
+
+ it("should fail when the reports contain no coverage data", function () {
+ const dir = path.join(tmpDir, "coverage-empty");
+ fs.mkdirSync(dir);
+ fs.writeFileSync(path.join(dir, "lcov.info"), "");
+
+ const { output, status, stderr } = merge(tmpDir);
+
+ expect(status, "to be", 1);
+ expect(stderr, "to contain", "contain no coverage data");
+ expect(fs.existsSync(output), "to be false");
+ });
+});