Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

record E2E statistics (#82496)

Chris Vandoodewaard6fdc4439da4f
diff --git a/.github/scripts/upload-e2e-test-timings.ts b/.github/scripts/upload-e2e-test-timings.ts
new file mode 100644
index 00000000000..377cce57288
--- /dev/null
+++ b/.github/scripts/upload-e2e-test-timings.ts
@@ -0,0 +1,73 @@
+/**
+ * Collect some stats about our e2e tests:
+ * - total number of spec files
+ * - total time to run all specs
+ * - average spec runtime
+ * - max spec runtime
+ *
+ */
+
+import { readFileSync } from "node:fs";
+
+import { importStats, StatsRow } from "./stats-import";
+
+const TABLE = "e2e_test_timings";
+
+export interface Timings {
+ durations?: { spec?: string; duration?: unknown }[];
+}
+
+export const buildTestTimingsRow = (
+ timestamp: string,
+ timings: Timings,
+): StatsRow => {
+ const durations = timings.durations ?? [];
+ const specCount = durations.length;
+
+ if (specCount === 0) {
+ throw new Error("No durations found.");
+ }
+
+ let totalRunTime = 0;
+ let maxSpecRuntime = 0;
+ for (const { duration } of durations) {
+ if (typeof duration !== "number") {
+ continue;
+ }
+
+ totalRunTime += duration;
+ maxSpecRuntime = duration > maxSpecRuntime ? duration : maxSpecRuntime;
+ }
+
+ const averageSpecRuntime = Math.round(totalRunTime / specCount);
+
+ return {
+ date: timestamp,
+ spec_count: specCount,
+ total_runtime: totalRunTime,
+ avg_spec_runtime: averageSpecRuntime,
+ max_spec_runtime: maxSpecRuntime,
+ };
+};
+
+export async function main() {
+ const path = process.env.TIMINGS_PATH || "e2e/support/timings.json";
+
+ try {
+ const timings = JSON.parse(readFileSync(path, "utf-8")) as Timings;
+ const row = buildTestTimingsRow(new Date().toISOString(), timings);
+ console.log(row);
+
+ await importStats({ table: TABLE, rows: [row] });
+ console.log(`Successfully uploaded e2e test timings to ${TABLE}.`);
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ console.log(
+ `::warning::Uploading e2e test timings from ${path} failed; leaving run green: ${message}`,
+ );
+ }
+}
+
+if ((import.meta as ImportMeta & { main?: boolean }).main) {
+ main();
+}
diff --git a/.github/scripts/upload-e2e-test-timings.unit.spec.ts b/.github/scripts/upload-e2e-test-timings.unit.spec.ts
new file mode 100644
index 00000000000..40f9757fc0a
--- /dev/null
+++ b/.github/scripts/upload-e2e-test-timings.unit.spec.ts
@@ -0,0 +1,61 @@
+import { buildTestTimingsRow } from "./upload-e2e-test-timings";
+
+const timestamp = "2026-06-30T00:00:00.000Z";
+
+describe("buildTestTimingsRow", () => {
+ it("summarizes spec durations", () => {
+ const row = buildTestTimingsRow(timestamp, {
+ durations: [
+ { spec: "a.cy.spec.js", duration: 1000 },
+ { spec: "b.cy.spec.js", duration: 3000 },
+ { spec: "c.cy.spec.js", duration: 2000 },
+ ],
+ });
+
+ expect(row).toEqual({
+ date: timestamp,
+ spec_count: 3,
+ total_runtime: 6000,
+ avg_spec_runtime: 2000,
+ max_spec_runtime: 3000,
+ });
+ });
+
+ it("rounds the average runtime", () => {
+ const row = buildTestTimingsRow(timestamp, {
+ durations: [
+ { spec: "a.cy.spec.js", duration: 1000 },
+ { spec: "b.cy.spec.js", duration: 1001 },
+ ],
+ });
+
+ expect(row.avg_spec_runtime).toBe(1001);
+ });
+
+ it("counts specs with a non-numeric duration but ignores their value", () => {
+ const row = buildTestTimingsRow(timestamp, {
+ durations: [
+ { spec: "a.cy.spec.js", duration: 1000 },
+ { spec: "b.cy.spec.js", duration: null },
+ { spec: "c.cy.spec.js" },
+ ],
+ });
+
+ expect(row).toEqual({
+ date: timestamp,
+ spec_count: 3,
+ total_runtime: 1000,
+ avg_spec_runtime: 333,
+ max_spec_runtime: 1000,
+ });
+ });
+
+ it("throws when there are no durations", () => {
+ expect(() => buildTestTimingsRow(timestamp, { durations: [] })).toThrow(
+ "No durations found.",
+ );
+ expect(() => buildTestTimingsRow(timestamp, {})).toThrow(
+ "No durations found.",
+ );
+ });
+});
diff --git a/.github/workflows/report.update-e2e-timings.yml b/.github/workflows/report.update-e2e-timings.yml
index 6a65d4f008f..b5f739e47c9 100644
--- a/.github/workflows/report.update-e2e-timings.yml
+++ b/.github/workflows/report.update-e2e-timings.yml
@@ -56,6 +56,16 @@ jobs:
core.setOutput('artifact-count', result.artifactCount);
+ - uses: ./.github/actions/prepare-node-bun
+
+ - name: Upload e2e test timings to Stats
+ if: ${{ github.repository == 'metabase/metabase' }}
+ env:
+ TIMINGS_PATH: e2e/support/timings.json
+ ENG_STATS_URL: ${{ secrets.ENG_STATS_URL }}
+ API_KEY: ${{ secrets.ENG_STATS_API_KEY }}
+ run: bun .github/scripts/upload-e2e-test-timings.ts
+
- name: Configure git and commit changes
id: commit
run: |
@@ -110,7 +120,7 @@ jobs:
--body "$BODY" \
--label ".CI & Tests" \
--label "no-backport" \
- --reviewer metabase/devexp)
+ --reviewer metabase/devex)
PR_NUMBER=${URL##*/}