The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Dev 2343 record coverage per test and track routes (#78195)
Fraser616ee101b749
diff --git a/.github/workflows/e2e-coverage-manifest.yml b/.github/workflows/e2e-coverage-manifest.ymlindex 8cda7c4221b..7655f570610 100644--- a/.github/workflows/e2e-coverage-manifest.yml+++ b/.github/workflows/e2e-coverage-manifest.yml@@ -48,6 +48,18 @@ jobs:name: instrumented-uberjarpath: target/uberjar/metabase.jarretention-days: 1+ # Regenerate the OpenAPI spec from this SHA's defendpoint definitions so+ # route normalization in the merge job matches against the endpoints+ # that actually served this run, not the checked-in file (which is only+ # refreshed by the label-gated openapi-check workflow and drifts).+ - name: Generate OpenAPI route table+ run: bun run generate-openapi+ - name: Upload OpenAPI route table+ uses: actions/upload-artifact@v7+ with:+ name: openapi-spec+ path: resources/openapi/openapi.json+ retention-days: 1# Fixed-N sharding — coverage doesn't need the PR matrix-builder's tag/edition# groups, just an even split of the regular spec set. The shard count lives in@@ -130,6 +142,15 @@ jobs:- name: Unzip previous manifestif: steps.prev.outputs.found == 'true'run: unzip -o prev-manifest.zip -d prev-manifest+ # Overwrites the checked-out resources/openapi/openapi.json with the one+ # generated from this run's SHA. If the artifact is somehow missing, the+ # checked-in file is still there and the builder falls back to it.+ - name: Download OpenAPI route table+ continue-on-error: true+ uses: actions/download-artifact@v8+ with:+ name: openapi-spec+ path: resources/openapi# GITHUB_SHA (the master commit this nightly built from) is stamped into# the manifest as `builtAt`, so a PR can later pick the manifest closest# to its own point in history. See resolve-coverage-manifest.mjs.@@ -137,9 +158,25 @@ jobs:env:PREV_MANIFEST: ${{ steps.prev.outputs.found == 'true' && 'prev-manifest/spec-file-manifest.json' || '' }}run: node e2e/coverage/build-coverage-manifest.mjs- - name: Upload manifest+ # Manifest routes are stored table-independent; consumers match them+ # against an endpoint spec at selection time. Ship this run's generated+ # spec alongside so they can choose it over their checkout's copy.+ # Copied into e2e/coverage so the artifact stays flat — the backfill+ # step expects the manifests at the archive root.+ - name: Include OpenAPI spec in artifact+ run: |+ if [ -f resources/openapi/openapi.json ]; then+ cp resources/openapi/openapi.json e2e/coverage/openapi.json+ fi+ # All files share one artifact so the per-test manifest always travels+ # with the spec manifest it was built from (backfill and PR-side+ # resolution read them as siblings).+ - name: Upload manifestsuses: actions/upload-artifact@v7with:name: spec-file-manifest- path: e2e/coverage/spec-file-manifest.json+ path: |+ e2e/coverage/spec-file-manifest.json+ e2e/coverage/spec-test-manifest.json+ e2e/coverage/openapi.jsonretention-days: 30diff --git a/.gitignore b/.gitignoreindex fa38fb324c4..62931eeacbf 100644--- a/.gitignore+++ b/.gitignore@@ -62,6 +62,8 @@ cypress.env.json/e2e/coverage-manifest/e2e/coverage/analysis/e2e/coverage/spec-file-manifest.json+/e2e/coverage/spec-test-manifest.json+/e2e/coverage/openapi.json/e2e/support/.nyc_output/e2e/support/coverage/e2e/support/e2ediff --git a/e2e/coverage/baseline.mjs b/e2e/coverage/baseline.mjsindex c4795cd9f6b..53fbdc29f1e 100644--- a/e2e/coverage/baseline.mjs+++ b/e2e/coverage/baseline.mjs@@ -5,41 +5,88 @@* Booting Metabase executes a large fraction of the FE bundle on every spec* (routing, store, shared components, app shell). Raw "statement executed"* coverage is therefore dominated by boot noise and is near-useless for- * deciding which specs a change affects. We instead keep only files a spec- * exercised MORE than a boot-only baseline run.+ * deciding which specs a change affects. We instead keep only files+ * where a spec executed a function the boot-only baseline run never reached.*/import path from "node:path";-// A spec exercised `file` beyond boot iff some function in it fired more times-// than it did in the baseline run. A function that fired the same-// number of times in both is treated as boot-only and ignored. Relies on-// function indices being identical between baseline and spec, which holds-// because both come from the same instrumented nightly build.+// A spec exercised `file` beyond boot iff it fired some function the baseline run never fired.+// Counts are not comparable: boot code re-fires on every page load,+// so any spec with more than one visit beats the single-boot baseline on every boot file.+// Relies on function indices being identical between baseline and spec,+// which holds because both come from the same instrumented nightly build.export function fileExceedsBaseline(specFileCov, baselineFileCov) {const sf = specFileCov.f || {};const bf = baselineFileCov?.f || {};for (const [idx, count] of Object.entries(sf)) {- if (count > (bf[idx] || 0)) {+ if (count > 0 && !(bf[idx] > 0)) {return true;}}return false;}-// Repo-relative paths of files a spec exercised beyond baseline.-//-// Paths are relativized because Istanbul keys are absolute to the machine that-// produced them (the nightly CI runner) and must resolve against a PR-// checkout's repo root at selection time. Files outside the repo are dropped.-export function discriminatingFiles(coverage, baselineCov, repoRoot) {- return Object.entries(coverage)- .filter(([file, fileCov]) =>- fileExceedsBaseline(fileCov, baselineCov?.[file]),- )- .map(([file]) =>+// Istanbul keys are absolute to the machine that produced them (the nightly+// CI runner) and must resolve against a PR checkout's repo root at selection+// time. Files outside the repo are dropped.+function toRepoRelative(files, repoRoot) {+ return files+ .map((file) =>path.isAbsolute(file) ? path.relative(repoRoot, file) : file,).filter((rel) => !rel.startsWith("..")).sort();}++// Repo-relative paths of files a spec exercised beyond baseline.+export function discriminatingFiles(coverage, baselineCov, repoRoot) {+ return toRepoRelative(+ Object.entries(coverage)+ .filter(([file, fileCov]) =>+ fileExceedsBaseline(fileCov, baselineCov?.[file]),+ )+ .map(([file]) => file),+ repoRoot,+ );+}++// Per-test variant of discriminatingFiles. `testDeltas` and `baselineDeltas`+// are sparse {file: {fnIdx: firedCount}} maps as recorded by the+// recordTestCapture task — counter deltas for a single test, not cumulative+// totals. A file survives when the test fired some function the baseline+// spec's single test never fired.+export function discriminatingFilesForTest(+ testDeltas,+ baselineDeltas,+ repoRoot,+) {+ return toRepoRelative(+ Object.entries(testDeltas || {})+ .filter(([file, deltas]) =>+ Object.entries(deltas).some(+ ([idx, count]) => count > 0 && !(baselineDeltas?.[file]?.[idx] > 0),+ ),+ )+ .map(([file]) => file),+ repoRoot,+ );+}++// Merges the baseline spec's per-test entries into a single per-visit noise map.+// The baseline spec has one test. Retried attempts merge by max,+// so a function fired in any attempt counts as boot-fired.+export function baselinePerTestDeltas(baselineEntry) {+ const merged = {};+ for (const test of baselineEntry?.tests || []) {+ for (const [file, deltas] of Object.entries(test.f || {})) {+ const fileMax = (merged[file] ??= {});+ for (const [idx, count] of Object.entries(deltas)) {+ if (count > (fileMax[idx] || 0)) {+ fileMax[idx] = count;+ }+ }+ }+ }+ return merged;+}diff --git a/e2e/coverage/build-coverage-manifest.mjs b/e2e/coverage/build-coverage-manifest.mjsindex cd6e2d715bc..f8063cf8a20 100644--- a/e2e/coverage/build-coverage-manifest.mjs+++ b/e2e/coverage/build-coverage-manifest.mjs@@ -1,28 +1,40 @@/*** Aggregates per-spec raw coverage written by the after:spec hook in- * e2e/support/config.js into a single spec -> files manifest used for- * selective e2e runs.+ * e2e/support/config.js into the manifests used for selective e2e runs:*- * For each spec, includes only files whose function counters strictly- * exceeded the baseline spec's.- * This strips boot noise. Files where every function fired the same count in baseline- * and spec are treated as "loaded but not exercised by this spec."+ * - spec-file-manifest.json: { builtAt, specs: {spec: [files]},+ * routes: {spec: [normalized API routes]},+ * pages: {spec: [normalized FE page paths]} }+ * - spec-test-manifest.json: { builtAt, specs: {spec: {testTitle:+ * {files, routes, pages}}} } — the per-test breakdown of the same data.*- * The manifest stores files, not modules, the test planner can map to modules later.+ * Capture is unfiltered (all requests, origins kept for third-party ones);+ * the `routes` dimension keeps only internal /api traffic, and `pages` keeps+ * document navigations. Everything captured stays in the raw per-spec+ * artifacts, so widening policy here needs no capture change.+ *+ * For files, includes only those where the spec fired a function+ * the baseline spec never fired (per test: the baseline spec's single test).+ * This strips boot noise. Files where the spec only re-fired+ * baseline functions are treated as "loaded but not exercised by this spec."+ *+ * The manifests store files and route shapes, not modules; the test planner+ * can map those to modules later.** `builtAt` records the commit the instrumented run was built from, so an old* PR can pick the manifest closest to its own point in history rather than a* newer one whose file tree has since drifted.** Backfill: a spec missing from today's coverage (its shard failed or flaked)- * keeps its entry from the previous manifest+ * keeps its entry from the previous manifests** Run after a full instrumented e2e pass:* INSTRUMENT_COVERAGE=true bun run build-release:js* <run cypress, including the coverage-baseline spec>* node e2e/coverage/build-coverage-manifest.mjs*- * Output: e2e/coverage/spec-file-manifest.json+ * Output: e2e/coverage/spec-file-manifest.json and+ * e2e/coverage/spec-test-manifest.json*/import { execSync } from "node:child_process";@@ -35,18 +47,41 @@ import {listSpecFiles,} from "../../.github/scripts/e2e-spec-globs.mjs";-import { discriminatingFiles } from "./baseline.mjs";+import {+ baselinePerTestDeltas,+ discriminatingFiles,+ discriminatingFilesForTest,+} from "./baseline.mjs";+import {+ apiRoutes,+ loadRouteTable,+ matchRoute,+ normalizePages,+ normalizeRoute,+ normalizeRoutes,+} from "./routes.mjs";const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)),"../..",);+// Authoritative endpoint shapes, generated from the backend's defendpoint+// definitions. Used only to LOG captured routes that match no endpoint (a+// drift/capture-bug signal) — stored shapes stay table-independent so+// consumers can match them against their own checkout's spec version.+// Missing file just means no drift log.+const OPENAPI_FILE = path.join(REPO_ROOT, "resources/openapi/openapi.json");+const PER_SPEC_DIR = path.join(REPO_ROOT, "e2e/coverage-manifest-raw");const OUTPUT_FILE = path.join(REPO_ROOT,"e2e/coverage/spec-file-manifest.json",);+const OUTPUT_TEST_FILE = path.join(+ REPO_ROOT,+ "e2e/coverage/spec-test-manifest.json",+);function readEntry(file) {try {@@ -59,19 +94,47 @@ function readEntry(file) {}}-// The previous nightly's manifest (downloaded by the merge job) used to backfill-// specs missing from today's run. Missing/unparseable means no backfill.-function loadPrevSpecs() {+// The previous nightly's manifests (downloaded by the merge job) used to+// backfill specs missing from today's run. Missing/unparseable means no+// backfill. The per-test manifest travels in the same artifact next to the+// main one; its absence (e.g. the previous nightly predates per-test capture)+// just means no per-test backfill.+function loadPrevManifests() {const file = process.env.PREV_MANIFEST;if (!file || !fs.existsSync(file)) {return null;}+ let main;try {- const { specs } = JSON.parse(fs.readFileSync(file, "utf8"));- return specs && typeof specs === "object" ? specs : null;+ main = JSON.parse(fs.readFileSync(file, "utf8"));} catch {return null;}+ if (!main?.specs || typeof main.specs !== "object") {+ return null;+ }++ let tests = {};+ try {+ const testFile = path.join(+ path.dirname(file),+ path.basename(OUTPUT_TEST_FILE),+ );+ if (fs.existsSync(testFile)) {+ const parsed = JSON.parse(fs.readFileSync(testFile, "utf8"));+ tests =+ parsed?.specs && typeof parsed.specs === "object" ? parsed.specs : {};+ }+ } catch {+ tests = {};+ }++ return {+ specs: main.specs,+ routes: main.routes ?? {},+ pages: main.pages ?? {},+ tests,+ };}// The commit this manifest describes. CI passes the instrumented build's SHA;@@ -101,7 +164,9 @@ function main() {continue;}const entry = readEntry(path.join(PER_SPEC_DIR, file));- if (entry) {+ // The raw dir also holds fnmap-*.json function-metadata files (for+ // offline analysis, not the manifests); only spec entries belong here.+ if (entry?.spec) {entries[entry.spec] = entry;}}@@ -114,8 +179,21 @@ function main() {process.exit(1);}+ const baselineTestDeltas = baselinePerTestDeltas(baseline);++ const routeTable = loadRouteTable(OPENAPI_FILE);+ if (!routeTable) {+ console.warn(`No OpenAPI spec at ${OPENAPI_FILE}; skipping drift check.`);+ }+const specs = {};+ const specRoutes = {};+ const specPages = {};+ const specTests = {};+ // Concrete (pre-normalization) API routes captured today, for the drift log.+ const concreteApiRoutes = new Set();let totalFiles = 0;+ let totalTests = 0;for (const [spec, entry] of Object.entries(entries)) {if (spec === BASELINE_SPEC) {continue;@@ -127,33 +205,128 @@ function main() {);specs[spec] = files;totalFiles += files.length;++ // Per-test breakdown. Retried attempts share a title; their files,+ // routes, and pages union together.+ const tests = {};+ const routes = new Set();+ const pages = new Set();+ for (const attempt of entry.tests ?? []) {+ const test = (tests[attempt.title] ??= {+ files: new Set(),+ routes: new Set(),+ pages: new Set(),+ });+ for (const file of discriminatingFilesForTest(+ attempt.f,+ baselineTestDeltas,+ REPO_ROOT,+ )) {+ test.files.add(file);+ }+ const attemptApiRoutes = apiRoutes(attempt.routes);+ for (const route of attemptApiRoutes) {+ concreteApiRoutes.add(route);+ }+ for (const route of normalizeRoutes(attemptApiRoutes)) {+ test.routes.add(route);+ routes.add(route);+ }+ for (const page of normalizePages(attempt.pages)) {+ test.pages.add(page);+ pages.add(page);+ }+ }+ specRoutes[spec] = [...routes].sort();+ specPages[spec] = [...pages].sort();+ specTests[spec] = Object.fromEntries(+ Object.entries(tests).map(([title, test]) => [+ title,+ {+ files: [...test.files].sort(),+ routes: [...test.routes].sort(),+ pages: [...test.pages].sort(),+ },+ ]),+ );+ totalTests += Object.keys(tests).length;}// Carry over still-existing specs that didn't run today (failed/flaked shard).- const prev = loadPrevSpecs();+ const prev = loadPrevManifests();let backfilled = 0;if (prev) {// Specs that still exist, so backfill never resurrects deleted ones.const current = new Set(listSpecFiles(REPO_ROOT));- for (const [spec, files] of Object.entries(prev)) {+ for (const [spec, files] of Object.entries(prev.specs)) {if (!(spec in specs) && current.has(spec)) {specs[spec] = files;+ specRoutes[spec] = prev.routes[spec] ?? [];+ specPages[spec] = prev.pages[spec] ?? [];+ specTests[spec] = prev.tests[spec] ?? {};totalFiles += files.length;+ totalTests += Object.keys(specTests[spec]).length;backfilled += 1;}}}- const manifest = { builtAt: builtAtSha(), specs };- fs.writeFileSync(OUTPUT_FILE, JSON.stringify(manifest, null, 2) + "\n");+ const builtAt = builtAtSha();+ fs.writeFileSync(+ OUTPUT_FILE,+ JSON.stringify(+ { builtAt, specs, routes: specRoutes, pages: specPages },+ null,+ 2,+ ) + "\n",+ );+ // Compact on purpose — with per-test file lists populated this is ~200 MB+ // pretty-printed; nobody reads it by eye.+ fs.writeFileSync(+ OUTPUT_TEST_FILE,+ JSON.stringify({ builtAt, specs: specTests }) + "\n",+ );const specCount = Object.keys(specs).length;const bytes = fs.statSync(OUTPUT_FILE).size;+ const testBytes = fs.statSync(OUTPUT_TEST_FILE).size;console.log(`Wrote ${OUTPUT_FILE} (${specCount} specs, ${backfilled} backfilled, ` +`${totalFiles} file edges, ${(bytes / 1e6).toFixed(1)} MB, ` +- `builtAt ${manifest.builtAt.slice(0, 12)}).`,+ `builtAt ${builtAt.slice(0, 12)}).`,);+ console.log(+ `Wrote ${OUTPUT_TEST_FILE} (${totalTests} tests, ` ++ `${(testBytes / 1e6).toFixed(1)} MB).`,+ );++ // Drift log: captured API routes that resolve to no endpoint definition —+ // either an endpoint defined outside defendpoint or a capture bug. Checked+ // against today's concrete captures (backfilled entries were checked the+ // day they ran) and reported in normalized form for readability.+ if (routeTable) {+ const unmatched = new Set();+ for (const route of concreteApiRoutes) {+ const separator = route.indexOf(" ");+ const method = route.slice(0, separator);+ const pathname = route.slice(separator + 1);+ if (!matchRoute(routeTable, method, pathname)) {+ unmatched.add(normalizeRoute(route));+ }+ }+ const sorted = [...unmatched].sort();+ console.log(+ `${concreteApiRoutes.size} distinct captured API routes; ` ++ `${sorted.length} shapes match no endpoint definition.`,+ );+ const MAX_UNMATCHED_LISTED = 100;+ for (const route of sorted.slice(0, MAX_UNMATCHED_LISTED)) {+ console.log(` unmatched: ${route}`);+ }+ if (sorted.length > MAX_UNMATCHED_LISTED) {+ console.log(` ... and ${sorted.length - MAX_UNMATCHED_LISTED} more`);+ }+ }}main();diff --git a/e2e/coverage/manifest.unit.spec.js b/e2e/coverage/manifest.unit.spec.jsnew file mode 100644index 00000000000..56f3a15a663--- /dev/null+++ b/e2e/coverage/manifest.unit.spec.js@@ -0,0 +1,270 @@+import {+ baselinePerTestDeltas,+ discriminatingFiles,+ discriminatingFilesForTest,+ fileExceedsBaseline,+} from "./baseline.mjs";+import {+ apiRoutes,+ buildRouteTable,+ matchRoute,+ normalizePages,+ normalizeRoute,+ normalizeRoutes,+} from "./routes.mjs";++const REPO_ROOT = "/repo";++describe("normalizeRoute", () => {+ it("replaces numeric segments with :id", () => {+ expect(normalizeRoute("GET /api/card/173/query")).toBe(+ "GET /api/card/:id/query",+ );+ });++ it("replaces UUID segments with :uuid", () => {+ expect(+ normalizeRoute(+ "GET /api/public/dashboard/f0e545b0-5f2e-45a7-91b6-371041fdcd48",+ ),+ ).toBe("GET /api/public/dashboard/:uuid");+ });++ it("replaces NanoID entity-id segments with :entity-id", () => {+ expect(normalizeRoute("GET /api/card/vXpKSQIDGNfeAmS1VYzN9")).toBe(+ "GET /api/card/:entity-id",+ );+ });++ it("keeps 21-char lowercase literal segments", () => {+ expect(normalizeRoute("GET /api/notification_channels")).toBe(+ "GET /api/notification_channels",+ );+ });++ it("keeps literal segments and passes through malformed values", () => {+ expect(normalizeRoute("POST /api/dataset")).toBe("POST /api/dataset");+ expect(normalizeRoute("not-a-route")).toBe("not-a-route");+ });+});++describe("normalizeRoutes", () => {+ it("normalizes, dedupes, and sorts", () => {+ expect(+ normalizeRoutes([+ "GET /api/card/2",+ "GET /api/card/1",+ "POST /api/dataset",+ ]),+ ).toEqual(["GET /api/card/:id", "POST /api/dataset"]);+ });++ it("tolerates missing input", () => {+ expect(normalizeRoutes(undefined)).toEqual([]);+ });+});++describe("apiRoutes", () => {+ it("keeps only internal /api requests", () => {+ expect(+ apiRoutes([+ "GET /api/card/1",+ "GET /app/assets/vendor-abc123.js",+ "GET /dashboard/1-orders",+ "POST https://snowplow-micro:9090/com.snowplowanalytics/tp2",+ "not-a-route",+ ]),+ ).toEqual(["GET /api/card/1"]);+ expect(apiRoutes(undefined)).toEqual([]);+ });+});++describe("normalizePages", () => {+ it("normalizes ids, slugs, uuids, and tokens; dedupes and sorts", () => {+ expect(+ normalizePages([+ "/dashboard/1-orders-dashboard",+ "/dashboard/2",+ "/question/173-quarterly-revenue",+ "/public/dashboard/f0e545b0-5f2e-45a7-91b6-371041fdcd48",+ "/embed/dashboard/fake-header.fake-payload.fake-signature",+ "/admin/settings",+ "/admin/settings",+ ]),+ ).toEqual([+ "/admin/settings",+ "/dashboard/:id",+ "/embed/dashboard/:token",+ "/public/dashboard/:uuid",+ "/question/:id",+ ]);+ expect(normalizePages(undefined)).toEqual([]);+ });+});++describe("OpenAPI route table", () => {+ const table = buildRouteTable({+ paths: {+ "/api/card/{id}": { get: {}, put: {} },+ "/api/card/{id}/query": { post: {} },+ "/api/card/pivot": { post: {} },+ "/api/setting/{key}": { get: {}, parameters: [] },+ "/api/dataset": { post: {} },+ },+ });++ it("matches concrete paths to their shape", () => {+ expect(matchRoute(table, "GET", "/api/card/173")).toBe("/api/card/{id}");+ expect(matchRoute(table, "POST", "/api/card/173/query")).toBe(+ "/api/card/{id}/query",+ );+ });++ it("prefers literal segments over params, like a router", () => {+ expect(matchRoute(table, "POST", "/api/card/pivot")).toBe(+ "/api/card/pivot",+ );+ });++ it("matches non-numeric params the regex fallback cannot", () => {+ expect(matchRoute(table, "GET", "/api/setting/site-name")).toBe(+ "/api/setting/{key}",+ );+ });++ it("respects the method and returns null for unknown routes", () => {+ expect(matchRoute(table, "DELETE", "/api/card/173")).toBeNull();+ expect(matchRoute(table, "GET", "/api/unknown")).toBeNull();+ });++ it("ignores trailing slashes on captured paths", () => {+ expect(matchRoute(table, "GET", "/api/card/173/")).toBe("/api/card/{id}");+ });++ it("matches stored manifest shapes structurally, for consumers", () => {+ expect(matchRoute(table, "GET", "/api/card/:id")).toBe("/api/card/{id}");+ expect(matchRoute(table, "POST", "/api/card/:id/query")).toBe(+ "/api/card/{id}/query",+ );+ expect(matchRoute(table, "GET", "/api/setting/site-name")).toBe(+ "/api/setting/{key}",+ );+ });++ it("buildRouteTable returns null for unusable specs", () => {+ expect(buildRouteTable(null)).toBeNull();+ expect(buildRouteTable({ paths: {} })).toBeNull();+ });+});++describe("fileExceedsBaseline", () => {+ it("is true when some function fired that baseline never fired", () => {+ expect(+ fileExceedsBaseline({ f: { 0: 2, 1: 1 } }, { f: { 0: 2, 1: 0 } }),+ ).toBe(true);+ });++ it("is false when the spec only re-fired baseline functions", () => {+ // Boot code re-fires on every page load,+ // so higher counts on baseline-fired functions are still boot noise.+ expect(fileExceedsBaseline({ f: { 0: 24 } }, { f: { 0: 2 } })).toBe(false);+ });++ it("is false when every function matches baseline", () => {+ expect(fileExceedsBaseline({ f: { 0: 2 } }, { f: { 0: 2 } })).toBe(false);+ });++ it("treats a file absent from baseline as exceeding", () => {+ expect(fileExceedsBaseline({ f: { 0: 1 } }, undefined)).toBe(true);+ });++ it("ignores functions the spec did not fire", () => {+ expect(fileExceedsBaseline({ f: { 0: 2, 1: 0 } }, { f: { 0: 2 } })).toBe(+ false,+ );+ });+});++describe("discriminatingFiles", () => {+ it("relativizes surviving files and drops ones outside the repo", () => {+ const coverage = {+ "/repo/frontend/src/b.js": { f: { 0: 5 } },+ "/repo/frontend/src/a.js": { f: { 0: 1 } },+ "/elsewhere/c.js": { f: { 0: 1 } },+ };+ const baseline = {+ "/repo/frontend/src/a.js": { f: { 0: 1 } },+ };+ expect(discriminatingFiles(coverage, baseline, REPO_ROOT)).toEqual([+ "frontend/src/b.js",+ ]);+ });+});++describe("discriminatingFilesForTest", () => {+ const baselineDeltas = {+ "/repo/frontend/src/boot.js": { 0: 1, 1: 1 },+ };++ it("drops files whose deltas match single-visit boot noise", () => {+ const testDeltas = {+ "/repo/frontend/src/boot.js": { 0: 1, 1: 1 },+ "/repo/frontend/src/feature.js": { 3: 2 },+ };+ expect(+ discriminatingFilesForTest(testDeltas, baselineDeltas, REPO_ROOT),+ ).toEqual(["frontend/src/feature.js"]);+ });++ it("drops boot files that only re-fired baseline functions", () => {+ // Extra visits re-fire boot code at higher counts, which is still boot noise.+ const testDeltas = {+ "/repo/frontend/src/boot.js": { 0: 3, 1: 1 },+ };+ expect(+ discriminatingFilesForTest(testDeltas, baselineDeltas, REPO_ROOT),+ ).toEqual([]);+ });++ it("keeps boot files when a function fired that baseline never fired", () => {+ const testDeltas = {+ "/repo/frontend/src/boot.js": { 0: 1, 2: 1 },+ };+ expect(+ discriminatingFilesForTest(testDeltas, baselineDeltas, REPO_ROOT),+ ).toEqual(["frontend/src/boot.js"]);+ });++ it("handles missing deltas and files outside the repo", () => {+ expect(+ discriminatingFilesForTest(undefined, baselineDeltas, REPO_ROOT),+ ).toEqual([]);+ expect(+ discriminatingFilesForTest(+ { "/elsewhere/x.js": { 0: 1 } },+ baselineDeltas,+ REPO_ROOT,+ ),+ ).toEqual([]);+ });+});++describe("baselinePerTestDeltas", () => {+ it("merges attempts by max per function", () => {+ const baselineEntry = {+ tests: [+ { title: "t", f: { "/repo/a.js": { 0: 1, 1: 2 } }, routes: [] },+ { title: "t", f: { "/repo/a.js": { 0: 3 }, "/repo/b.js": { 0: 1 } } },+ ],+ };+ expect(baselinePerTestDeltas(baselineEntry)).toEqual({+ "/repo/a.js": { 0: 3, 1: 2 },+ "/repo/b.js": { 0: 1 },+ });+ });++ it("returns an empty map for entries without per-test data", () => {+ expect(baselinePerTestDeltas({ coverage: {} })).toEqual({});+ expect(baselinePerTestDeltas(undefined)).toEqual({});+ });+});diff --git a/e2e/coverage/routes.mjs b/e2e/coverage/routes.mjsnew file mode 100644index 00000000000..a923b1e0bf2--- /dev/null+++ b/e2e/coverage/routes.mjs@@ -0,0 +1,188 @@+/**+ * Route and page normalization for the coverage manifests.+ *+ * Capture (e2e/support/per-test-capture.js) records every concrete request+ * ("GET /api/card/173/query", "GET /app/assets/vendor-abc.js", third-party+ * URLs with origin kept) and every document navigation ("/dashboard/1-orders").+ * The manifests want shapes so entries dedupe across runs and can be matched+ * against backend endpoint definitions — filtering and normalization policy+ * lives here, offline, so it can change without re-running a nightly.+ *+ * Stored shapes are regex-normalized (":id"/":uuid"/":entity-id"/":token")+ * and deliberately independent of the OpenAPI route table. Matching against+ * endpoint definitions happens at consumption time: the manifest is built+ * from last night's SHA while a consumer compares against its own checkout's+ * endpoints, so baking one table's spelling in would break on any route+ * whose shape changed in between. A consumer matches structurally with+ * matchRoute() below against whichever spec version fits its checkout+ * (resources/openapi/openapi.json, generated from the backend's defendpoint+ * definitions by `bun run generate-openapi`; the nightly also ships its+ * freshly generated copy alongside the manifests).+ *+ * The nightly build still loads the table for one thing: logging captured+ * API routes that correspond to no endpoint definition — an endpoint defined+ * outside defendpoint, or a capture bug.+ */++import fs from "node:fs";++// Numeric ids, with or without a slug tail: "173", "1-orders-dashboard".+const NUMERIC_SEGMENT = /^\d+(-[^/]*)?$/;+const UUID_SEGMENT =+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;+// NanoID entity ids: 21 chars of [A-Za-z0-9_-]. Real API path segments are+// lowercase words, so requiring a digit or uppercase letter avoids swallowing+// a literal segment that happens to be 21 chars long.+const ENTITY_ID_SEGMENT = /^(?=.*[A-Z0-9])[A-Za-z0-9_-]{21}$/;+// Signed embedding tokens in /embed/* page paths: header.payload.signature.+const JWT_SEGMENT = /^[\w-]+\.[\w-]+\.[\w-]+$/;++const HTTP_METHODS = new Set([+ "get",+ "post",+ "put",+ "patch",+ "delete",+ "head",+ "options",+]);++const PARAM_SEGMENT = /^\{.+\}$/;++function splitPath(pathname) {+ return pathname.replace(/\/+$/, "").split("/");+}++/**+ * Builds a route matcher from a parsed OpenAPI spec. Returns null when the+ * spec has no usable paths.+ */+export function buildRouteTable(spec) {+ if (!spec?.paths || typeof spec.paths !== "object") {+ return null;+ }++ // Indexed by "METHOD segment-count" — only routes with the same method and+ // arity can match a captured path.+ const index = new Map();+ for (const [path, operations] of Object.entries(spec.paths)) {+ const segments = splitPath(path);+ for (const method of Object.keys(operations)) {+ if (!HTTP_METHODS.has(method)) {+ continue;+ }+ const key = `${method.toUpperCase()} ${segments.length}`;+ let candidates = index.get(key);+ if (!candidates) {+ index.set(key, (candidates = []));+ }+ candidates.push({ shape: path, segments });+ }+ }+ return index.size > 0 ? index : null;+}++/**+ * Loads the route table from an OpenAPI spec file. Returns null when the file+ * is missing or unparseable, in which case normalization is pure regex+ * fallback.+ */+export function loadRouteTable(openapiFile) {+ try {+ return buildRouteTable(JSON.parse(fs.readFileSync(openapiFile, "utf8")));+ } catch {+ return null;+ }+}++// Matches a pathname against the table the way a router would: literal+// segments must match exactly, "{param}" segments match anything, and the+// candidate with the most literal matches wins ("/api/card/pivot" beats+// "/api/card/{id}" for POST /api/card/pivot). Ties break lexicographically+// for determinism. Returns the matched shape or null. Works for concrete+// paths ("/api/card/173") and for stored manifest shapes ("/api/card/:id") —+// a ":param" segment simply matches the table's "{param}" wildcard.+export function matchRoute(table, method, pathname) {+ const segments = splitPath(pathname);+ const candidates =+ table.get(`${String(method).toUpperCase()} ${segments.length}`) || [];++ let best = null;+ let bestLiterals = -1;+ for (const candidate of candidates) {+ let literals = 0;+ let matched = true;+ for (let i = 0; i < segments.length; i += 1) {+ if (candidate.segments[i] === segments[i]) {+ literals += 1;+ } else if (!PARAM_SEGMENT.test(candidate.segments[i])) {+ matched = false;+ break;+ }+ }+ if (+ matched &&+ (literals > bestLiterals ||+ (literals === bestLiterals && candidate.shape < best.shape))+ ) {+ best = candidate;+ bestLiterals = literals;+ }+ }+ return best ? best.shape : null;+}++function normalizeSegment(segment) {+ if (NUMERIC_SEGMENT.test(segment)) {+ return ":id";+ }+ if (UUID_SEGMENT.test(segment)) {+ return ":uuid";+ }+ if (ENTITY_ID_SEGMENT.test(segment)) {+ return ":entity-id";+ }+ if (JWT_SEGMENT.test(segment)) {+ return ":token";+ }+ return segment;+}++// "GET /api/card/173/query" -> "GET /api/card/:id/query". Unparseable values+// pass through untouched rather than dropping data.+export function normalizeRoute(route) {+ const separator = route.indexOf(" ");+ if (separator === -1) {+ return route;+ }+ const method = route.slice(0, separator);+ const pathname = route.slice(separator + 1);+ return `${method} ${pathname.split("/").map(normalizeSegment).join("/")}`;+}++// Normalized, deduped, sorted.+export function normalizeRoutes(routes) {+ return [...new Set((routes || []).map(normalizeRoute))].sort();+}++// Capture records everything; the manifests' `routes` dimension is only+// internal backend API traffic. Internal requests are bare paths (third-party+// ones keep their origin), so "starts with /api/" selects exactly those.+export function apiRoutes(routes) {+ return (routes || []).filter((route) => {+ const separator = route.indexOf(" ");+ return separator !== -1 && route.slice(separator + 1).startsWith("/api/");+ });+}++// Document navigation paths -> deduped FE route shapes+// ("/dashboard/1-orders" -> "/dashboard/:id").+export function normalizePages(pages) {+ return [+ ...new Set(+ (pages || []).map((page) =>+ page.split("/").map(normalizeSegment).join("/"),+ ),+ ),+ ].sort();+}diff --git a/e2e/support/config.js b/e2e/support/config.jsindex aaadc62990c..7f47b89edb7 100644--- a/e2e/support/config.js+++ b/e2e/support/config.js@@ -52,23 +52,94 @@ const COVERAGE_MANIFEST_RAW_DIR = path.resolve();const NYC_OUTPUT_FILE = path.resolve(__dirname, ".nyc_output/out.json");+// Function metadata (name + line per Istanbul function index), accumulated+// across the specs this process runs and shipped with the raw shard artifact.+// The f-counter indices in the per-spec/per-test entries are only meaningful+// against the exact instrumented bundle that produced them; this file makes+// the artifact self-describing for offline analysis (test-overlap heat maps)+// without rebuilding that bundle. Named uniquely per process so shard+// artifacts can merge into one directory without clobbering each other —+// consumers shallow-merge all fnmap-*.json (same file => identical entries).+const FNMAP_FILE = path.join(+ COVERAGE_MANIFEST_RAW_DIR,+ `fnmap-${require("node:crypto").randomUUID()}.json`,+);+const isEnterprise = process.env["MB_EDITION"] === "ee";const isCI = !!process.env.CI;const snowplowMicroUrl = process.env["MB_SNOWPLOW_URL"];-// Persists raw __coverage__ counters per spec. The manifest builder reads-// these later, applies baseline subtraction, and maps surviving files to-// modules. We delete .nyc_output/out.json between specs so each entry-// reflects only that spec's execution — @cypress/code-coverage otherwise-// accumulates.+// Per-test capture state, fed by the recordTestCapture task that the+// support-file afterEach calls (e2e/support/per-test-capture.js). Each entry+// is one test attempt: { title, f: {file: {fnIdx: firedCount}}, routes,+// pages }. The function counts arrive already per-test — the support file+// reads them from the app windows' Istanbul counters and zeroes those after+// each flush.+let perTestEntries = [];++const perTestCaptureTasks = {+ recordTestCapture({ title, f, routes, pages }) {+ perTestEntries.push({ title, f, routes, pages });+ return null;+ },++ resetTestCapture() {+ perTestEntries = [];+ return null;+ },+};++// Records name + line for every instrumented function in files this process+// hasn't seen yet. The metadata is identical for a given file across specs+// (same bundle), so first sighting wins.+function appendFnMap(coverage) {+ let fnMap = {};+ try {+ fnMap = JSON.parse(fs.readFileSync(FNMAP_FILE, "utf8"));+ } catch {+ // First spec of the run.+ }+ let changed = false;+ for (const [file, fileCov] of Object.entries(coverage)) {+ if (fnMap[file] || !fileCov.fnMap) {+ continue;+ }+ const entry = {};+ for (const [idx, fn] of Object.entries(fileCov.fnMap)) {+ entry[idx] = {+ name: fn.name,+ line: fn.decl?.start?.line ?? fn.loc?.start?.line ?? null,+ };+ }+ fnMap[file] = entry;+ changed = true;+ }+ if (changed) {+ fs.writeFileSync(FNMAP_FILE, JSON.stringify(fnMap));+ }+}++// Persists raw __coverage__ counters per spec, plus the per-test breakdown+// (function deltas and API routes). The manifest builder reads these later,+// applies baseline subtraction, and maps surviving files to modules. We+// delete .nyc_output/out.json between specs so each entry reflects only that+// spec's execution — @cypress/code-coverage otherwise accumulates.function writeSpecCoverageEntry(spec) {+ // Consume the per-test state up front so a missing/corrupt out.json can't+ // leak one spec's tests into the next spec's entry.+ const tests = perTestEntries;+ perTestEntries = [];+if (!fs.existsSync(NYC_OUTPUT_FILE)) {return;}const coverage = JSON.parse(fs.readFileSync(NYC_OUTPUT_FILE, "utf8"));+ fs.mkdirSync(COVERAGE_MANIFEST_RAW_DIR, { recursive: true });+ appendFnMap(coverage);+// The manifest builder only needs per-file function counters to compute the// baseline greater-delta. Drop statement/branch maps and counters, and drop// files where no function was invoked. Cuts each entry from ~25MB to <200KB.@@ -84,7 +155,7 @@ function writeSpecCoverageEntry(spec) {const entryName = spec.relative.replace(/[\\/]/g, "__") + ".json";fs.writeFileSync(path.join(COVERAGE_MANIFEST_RAW_DIR, entryName),- JSON.stringify({ spec: spec.relative, coverage: trimmed }),+ JSON.stringify({ spec: spec.relative, coverage: trimmed, tests }),);fs.unlinkSync(NYC_OUTPUT_FILE);@@ -217,6 +288,7 @@ const defaultConfig = {startCustomVizDevServer,stopCustomVizDevServer,buildDataApp,+ ...perTestCaptureTasks,});/********************************************************************diff --git a/e2e/support/cypress.js b/e2e/support/cypress.jsindex f0a7aea723a..fcd8089436c 100644--- a/e2e/support/cypress.js+++ b/e2e/support/cypress.js@@ -9,6 +9,11 @@ import { configure } from "@testing-library/cypress";import "cypress-real-events/support";import addContext from "mochawesome/addContext";import "./commands";+// Must stay imported after "@cypress/code-coverage/support": its afterEach+// zeroes the window coverage counters after collecting each test's fires,+// which must happen after the plugin's own afterEach has sent them to its+// accumulator.+import "./per-test-capture";const isCI = Cypress.expose("CI");const isNetworkThrottlingEnabled = Cypress.expose("ENABLE_NETWORK_THROTTLING");@@ -20,6 +25,10 @@ const isFailFastEnabled = Cypress.expose("FAIL_FAST");if (Cypress.expose("codeCoverageTasksRegistered") === true) {before(() => {cy.task("resetCoverage", { isInteractive: true }, { log: false });+ // Companion reset for the per-test capture state (per-test-capture.js);+ // headless runs also reset it in after:spec, but that event doesn't fire+ // between interactive re-runs.+ cy.task("resetTestCapture", null, { log: false });});}diff --git a/e2e/support/per-test-capture.js b/e2e/support/per-test-capture.jsnew file mode 100644index 00000000000..4521bf46dec--- /dev/null+++ b/e2e/support/per-test-capture.js@@ -0,0 +1,252 @@+/**+ * Per-test usage capture for the nightly instrumented e2e runs.+ *+ * For every test (attempt) this records:+ * - every HTTP request it made: app traffic (fetch/XHR/assets/documents)+ * via a pass-through middleware intercept plus fetch/XHR wrappers+ * installed on every app window, and setup traffic issued through+ * cy.request (helpers like H.createQuestion never hit cy.intercept+ * because they go through the Cypress server, not the browser). Capture+ * is deliberately unfiltered — same-origin requests record as paths,+ * third-party ones keep their origin — so filtering policy (e.g. "only+ * backend API endpoints") lives in the offline manifest builder, where+ * it can change without re-running a nightly.+ * - the pages it navigated to: document loads of the app window+ * (window:before:load, which fires regardless of which hook triggered+ * the visit) and framed documents (via the intercept's resourceType).+ * - which instrumented functions fired, read browser-side from each app+ * window's Istanbul counters (window.__coverage__). The counters are+ * zeroed after every flush, so each test reports only its own fires —+ * they cannot be diffed node-side against @cypress/code-coverage's+ * .nyc_output/out.json because the plugin only writes that file once per+ * spec (combineCoverage merges in memory; coverageReport persists).+ *+ * Two app-traffic capture paths on purpose: the intercept sees traffic the+ * window wrappers can't (child iframes in embedding specs), while the+ * wrappers see traffic the intercept can't — suite-level before() hooks run+ * ahead of every beforeEach and intercepts reset between tests, so no+ * intercept can be live during them, but window:before:load fires for every+ * app window regardless of which hook visited it. The overlap dedupes at+ * flush.+ *+ * The afterEach flush below MUST run after @cypress/code-coverage's own+ * afterEach, which sends the window's cumulative counters to the plugin's+ * accumulator — zeroing before that send would drop the test's fires from+ * the spec-level totals. That holds because this module is imported after+ * "@cypress/code-coverage/support" in e2e/support/cypress.js and root-level+ * hooks run in registration order. (A side effect of per-test zeroing is+ * that the plugin's summed spec totals become accurate instead of+ * re-counting each window's cumulative counters every test.)+ *+ * Known attribution gap, acceptable for manifest purposes: requests from+ * hooks attribute to the surrounding test:before:run/afterEach window, so+ * suite-level before() traffic lands on the suite's first test.+ */++const isInstrumented = Cypress.expose("coverage") === true;++const HTTP_METHODS = new Set([+ "GET",+ "POST",+ "PUT",+ "PATCH",+ "DELETE",+ "HEAD",+ "OPTIONS",+]);++// Relative URLs resolve against this sentinel so they are distinguishable+// from absolute URLs pointing at third-party hosts (webhook testers, snowplow+// micro, ...), which record with their origin kept.+const RELATIVE_ORIGIN = "http://relative.invalid";++let routeBuffer = [];+let pageBuffer = [];++// References to the __coverage__ objects of app windows that may still gain+// counts. Istanbul registers every instrumented chunk into one object per+// window, so holding the reference sees lazily-loaded files too. A reload+// creates a fresh object (tracked by the window:load handler below); the old+// one keeps any counts fired earlier in the same test until the flush.+let coverageObjects = [];++function trackCoverage(win) {+ const coverage = win.__coverage__;+ if (coverage && !coverageObjects.includes(coverage)) {+ coverageObjects.push(coverage);+ }+}++// Sums the per-file function counters across all tracked windows, zeroing+// every counter (functions, statements, branches) as it goes so the next+// flush reports only what fired after this one. Dead windows' objects are+// zeroed and pruned — only the current window can still gain counts.+function collectAndZeroFunctionCounts(currentWin) {+ trackCoverage(currentWin);+ const f = {};+ for (const coverage of coverageObjects) {+ for (const [file, fileCov] of Object.entries(coverage)) {+ const fired = fileCov.f || {};+ for (const [idx, count] of Object.entries(fired)) {+ if (count > 0) {+ const fileTotals = (f[file] ??= {});+ fileTotals[idx] = (fileTotals[idx] || 0) + count;+ fired[idx] = 0;+ }+ }+ for (const idx of Object.keys(fileCov.s || {})) {+ fileCov.s[idx] = 0;+ }+ for (const counts of Object.values(fileCov.b || {})) {+ counts.fill(0);+ }+ }+ }+ coverageObjects = coverageObjects.filter(+ (coverage) => coverage === currentWin.__coverage__,+ );+ return f;+}++function isInternalOrigin(origin) {+ if (origin === RELATIVE_ORIGIN) {+ return true;+ }+ const baseUrl = Cypress.config("baseUrl");+ try {+ return baseUrl != null && origin === new URL(baseUrl).origin;+ } catch {+ return false;+ }+}++function recordRoute(method, url) {+ if (typeof url !== "string") {+ return;+ }+ let parsed;+ try {+ parsed = new URL(url, RELATIVE_ORIGIN);+ } catch {+ return;+ }+ if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {+ return;+ }+ // Internal requests record as bare paths; third-party ones keep the origin+ // so the offline builder can tell them apart.+ const target = isInternalOrigin(parsed.origin)+ ? parsed.pathname+ : `${parsed.origin}${parsed.pathname}`;+ routeBuffer.push(`${String(method).toUpperCase()} ${target}`);+}++// Document navigations of the app's own pages (query/hash stripped).+function recordPage(url) {+ let parsed;+ try {+ parsed = new URL(url);+ } catch {+ return; // about:blank and friends+ }+ if (isInternalOrigin(parsed.origin)) {+ pageBuffer.push(parsed.pathname);+ }+}++// Mirrors how Cypress itself disambiguates cy.request(url), cy.request(url,+// body), cy.request(method, url[, body]) and cy.request(options).+function recordRequestArgs(args) {+ const [first, second] = args;+ if (first != null && typeof first === "object") {+ recordRoute(first.method || "GET", first.url);+ } else if (+ typeof second === "string" &&+ HTTP_METHODS.has(String(first).toUpperCase())+ ) {+ recordRoute(first, second);+ } else if (typeof first === "string") {+ recordRoute("GET", first);+ }+}++// Records a fetch() call. `input` is fetch's first argument (string, URL, or+// Request); per the fetch spec an explicit init.method overrides a Request's.+function recordFetchArgs(win, input, init) {+ if (input instanceof win.Request) {+ recordRoute(init?.method || input.method || "GET", input.url);+ } else {+ recordRoute(init?.method || "GET", String(input));+ }+}++if (isInstrumented) {+ // Fires once per attempt, before any of the attempt's hooks.+ Cypress.on("test:before:run", () => {+ routeBuffer = [];+ pageBuffer = [];+ });++ // Every app page load, including ones triggered inside suite-level+ // before() hooks. At load time all synchronously-executed instrumented+ // chunks have registered, so __coverage__ exists.+ Cypress.on("window:load", trackCoverage);++ Cypress.on("window:before:load", (win) => {+ recordPage(win.location.href);++ const originalFetch = win.fetch;+ win.fetch = function (input, init) {+ try {+ recordFetchArgs(win, input, init);+ } catch {+ // Recording must never break the app's request.+ }+ return originalFetch.apply(this, arguments);+ };++ const originalOpen = win.XMLHttpRequest.prototype.open;+ win.XMLHttpRequest.prototype.open = function (method, url) {+ try {+ recordRoute(method, String(url));+ } catch {+ // Recording must never break the app's request.+ }+ return originalOpen.apply(this, arguments);+ };+ });++ beforeEach(() => {+ // middleware: true observes and passes through, so this coexists with the+ // specs' own cy.intercept stubs/waits without changing any behavior.+ cy.intercept({ pathname: "/**", middleware: true }, (req) => {+ recordRoute(req.method, req.url);+ // Framed documents (embedding specs) never hit window:before:load,+ // which only fires for the top app window.+ if (req.resourceType === "document") {+ recordPage(req.url);+ }+ });+ });++ Cypress.Commands.overwrite("request", (originalFn, ...args) => {+ recordRequestArgs(args);+ return originalFn(...args);+ });++ afterEach(function () {+ const title = this.currentTest.fullTitle();+ const routes = [...new Set(routeBuffer)].sort();+ const pages = [...new Set(pageBuffer)].sort();+ routeBuffer = [];+ pageBuffer = [];+ cy.window({ log: false }).then((win) => {+ const f = collectAndZeroFunctionCounts(win);+ return cy.task(+ "recordTestCapture",+ { title, f, routes, pages },+ { log: false },+ );+ });+ });+}