Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Dynamic E2E chunks (#85586)

Nemanja Glumacef57d2349557
diff --git a/.github/file-paths.yaml b/.github/file-paths.yaml
index 995e1e62898..595940c9dc6 100644
--- a/.github/file-paths.yaml
+++ b/.github/file-paths.yaml
@@ -293,7 +293,7 @@ e2e_infra: &e2e_infra
- "e2e/**/!(*.cy.spec.@(js|jsx|ts|tsx))"
- "frontend/build/**"
- "patches/**"
- - ".github/scripts/build-e2e-matrix.js"
+ - ".github/scripts/build-e2e-matrix.ts"
- ".github/file-paths.yaml"
e2e_all: &e2e_all
diff --git a/.github/scripts/build-e2e-matrix.js b/.github/scripts/build-e2e-matrix.js
deleted file mode 100755
index ac8535eb58b..00000000000
--- a/.github/scripts/build-e2e-matrix.js
+++ /dev/null
@@ -1,53 +0,0 @@
-// This script is used in .github/workflows/e2e-matrix-builder.yml
-// its aim is to split the e2e test matrix into multiple jobs
-// grouping some specific tests together, other tests are split into chunks
-
-const SPECS_PER_JOB = 5;
-const DEFAULT_SPEC_PATTERN = "./e2e/test/scenarios/**/*.cy.spec.*";
-
-const taggedJobs = [
- {
- name: "oss-subset",
- edition: "oss",
- tags: "@OSS @prerelease+-@EE",
- specs: DEFAULT_SPEC_PATTERN,
- },
- { name: "mongo", tags: "@mongo", specs: DEFAULT_SPEC_PATTERN },
- { name: "python", tags: "@python", specs: DEFAULT_SPEC_PATTERN },
-];
-
-function buildMatrix(options, specFiles, maxJobs) {
- const { java, defaultRunner } = options;
- if (!Number.isInteger(maxJobs) || maxJobs <= taggedJobs.length) {
- throw new Error(
- `E2E job limit must be an integer greater than ${taggedJobs.length} to allow at least one regular job alongside the tagged jobs`,
- );
- }
-
- if (specFiles?.length === 0) {
- return { config: [], regularChunks: 0 };
- }
-
- const maxRegularJobs = maxJobs - taggedJobs.length;
- let regularChunks = maxRegularJobs;
- if (specFiles !== null) {
- const jobsForSelectedSpecs = Math.ceil(specFiles.length / SPECS_PER_JOB);
- regularChunks = Math.min(maxRegularJobs, jobsForSelectedSpecs);
- }
-
- const regularJobs = Array.from({ length: regularChunks }, (_, index) => ({
- name: `e2e-group-${String(index + 1).padStart(2, "0")}`,
- }));
-
- // Regular jobs exclude the OSS, Mongo, and Python tags, so these jobs must also run for selected specs.
- const config = [...regularJobs, ...taggedJobs].map((test) => ({
- "java-version": java,
- runner: defaultRunner,
- edition: "ee",
- ...test,
- }));
-
- return { config, regularChunks };
-}
-
-module.exports = { buildMatrix };
diff --git a/.github/scripts/build-e2e-matrix.ts b/.github/scripts/build-e2e-matrix.ts
new file mode 100644
index 00000000000..4f7c5112a78
--- /dev/null
+++ b/.github/scripts/build-e2e-matrix.ts
@@ -0,0 +1,186 @@
+// This script is used in .github/workflows/e2e-matrix-builder.yml
+// its aim is to split the e2e test matrix into multiple jobs
+// grouping some specific tests together, other tests are split into chunks
+
+import { appendFileSync, readFileSync } from "node:fs";
+import path from "node:path";
+
+import { readSpecPaths } from "../../e2e/runner/read-spec-paths";
+
+const DEFAULT_SPEC_PATTERN = "./e2e/test/scenarios/**/*.cy.spec.*";
+const TIMINGS_FILE = "e2e/support/timings.json";
+// Spec paths in timings.json are relative to this directory.
+const TIMINGS_DIR = path.posix.dirname(TIMINGS_FILE);
+const MS_PER_MINUTE = 60_000;
+
+type SpecTiming = { spec: string; duration: number };
+export type Timings = { durations: SpecTiming[] };
+
+type MatrixJob = {
+ name: string;
+ edition?: "ee" | "oss";
+ tags?: string;
+ specs?: string;
+};
+
+type MatrixEntry = MatrixJob & {
+ "java-version": number;
+ runner: string;
+ edition: "ee" | "oss";
+};
+
+const taggedJobs: MatrixJob[] = [
+ {
+ name: "oss-subset",
+ edition: "oss",
+ tags: "@OSS @prerelease+-@EE",
+ specs: DEFAULT_SPEC_PATTERN,
+ },
+ { name: "mongo", tags: "@mongo", specs: DEFAULT_SPEC_PATTERN },
+ { name: "python", tags: "@python", specs: DEFAULT_SPEC_PATTERN },
+];
+
+function sum(values: number[]): number {
+ return values.reduce((total, value) => total + value, 0);
+}
+
+// Specs without a recorded duration count as the average duration, as in cypress-split.
+function getTotalTestMinutes(
+ timings: Timings,
+ specFiles: string[] | null,
+): number {
+ const durations = new Map(
+ timings.durations.map(({ spec, duration }) => [
+ path.posix.join(TIMINGS_DIR, spec),
+ duration,
+ ]),
+ );
+ const recordedDurations = [...durations.values()];
+ const averageDuration =
+ recordedDurations.length > 0
+ ? sum(recordedDurations) / recordedDurations.length
+ : 0;
+ const selectedDurations =
+ specFiles === null
+ ? recordedDurations
+ : specFiles.map(
+ (spec) =>
+ durations.get(path.posix.normalize(spec)) ?? averageDuration,
+ );
+
+ return sum(selectedDurations) / MS_PER_MINUTE;
+}
+
+export function buildMatrix({
+ java,
+ defaultRunner,
+ specFiles,
+ maxJobs,
+ timings,
+ targetRunnerMinutes,
+}: {
+ java: number;
+ defaultRunner: string;
+ /** null runs the full suite. */
+ specFiles: string[] | null;
+ maxJobs: number;
+ timings: Timings;
+ targetRunnerMinutes: number;
+}): { config: MatrixEntry[]; regularChunks: number } {
+ if (!Number.isInteger(maxJobs) || maxJobs <= taggedJobs.length) {
+ throw new Error(
+ `E2E job limit must be an integer greater than ${taggedJobs.length} to allow at least one regular job alongside the tagged jobs`,
+ );
+ }
+ if (!(targetRunnerMinutes > 0)) {
+ throw new Error("E2E target runner minutes must be a positive number");
+ }
+
+ if (specFiles?.length === 0) {
+ return { config: [], regularChunks: 0 };
+ }
+
+ const maxRegularJobs = maxJobs - taggedJobs.length;
+ const neededJobs = Math.ceil(
+ getTotalTestMinutes(timings, specFiles) / targetRunnerMinutes,
+ );
+ const regularChunks = Math.min(maxRegularJobs, Math.max(1, neededJobs));
+
+ const regularJobs: MatrixJob[] = Array.from(
+ { length: regularChunks },
+ (_, index) => ({
+ name: `e2e-group-${String(index + 1).padStart(2, "0")}`,
+ }),
+ );
+
+ // Regular jobs exclude the OSS, Mongo, and Python tags, so these jobs must also run for selected specs.
+ const config = [...regularJobs, ...taggedJobs].map((test) => ({
+ "java-version": java,
+ runner: defaultRunner,
+ edition: "ee" as const,
+ ...test,
+ }));
+
+ return { config, regularChunks };
+}
+
+function isSpecTiming(value: unknown): value is SpecTiming {
+ return (
+ typeof value === "object" &&
+ value !== null &&
+ "spec" in value &&
+ typeof value.spec === "string" &&
+ "duration" in value &&
+ typeof value.duration === "number"
+ );
+}
+
+function readTimings(file: string): Timings {
+ const timings: unknown = JSON.parse(readFileSync(file, "utf8"));
+ if (
+ typeof timings !== "object" ||
+ timings === null ||
+ !("durations" in timings) ||
+ !Array.isArray(timings.durations) ||
+ !timings.durations.every(isSpecTiming)
+ ) {
+ throw new Error(`Invalid e2e timings in ${file}`);
+ }
+ return { durations: timings.durations };
+}
+
+if (require.main === module) {
+ const {
+ GITHUB_OUTPUT,
+ JAVA_VERSION,
+ DEFAULT_RUNNER,
+ E2E_SPEC_PATHS_FILE,
+ E2E_MAX_JOBS,
+ E2E_TARGET_RUNNER_MINUTES,
+ } = process.env;
+ if (!GITHUB_OUTPUT || !DEFAULT_RUNNER) {
+ throw new Error("GITHUB_OUTPUT and DEFAULT_RUNNER are required");
+ }
+
+ const { config, regularChunks } = buildMatrix({
+ java: Number(JAVA_VERSION),
+ defaultRunner: DEFAULT_RUNNER,
+ specFiles: readSpecPaths(E2E_SPEC_PATHS_FILE),
+ maxJobs: Number(E2E_MAX_JOBS),
+ timings: readTimings(TIMINGS_FILE),
+ targetRunnerMinutes: Number(E2E_TARGET_RUNNER_MINUTES),
+ });
+
+ appendFileSync(
+ GITHUB_OUTPUT,
+ [
+ `regularChunks=${JSON.stringify(regularChunks)}`,
+ `config=${JSON.stringify({ include: config })}`,
+ `hasTests=${config.length > 0}`,
+ "",
+ ].join("\n"),
+ );
+
+ console.log({ regularChunks });
+ console.log("matrix config", config);
+}
diff --git a/.github/scripts/build-e2e-matrix.unit.spec.js b/.github/scripts/build-e2e-matrix.unit.spec.js
deleted file mode 100644
index aa7fcb989f2..00000000000
--- a/.github/scripts/build-e2e-matrix.unit.spec.js
+++ /dev/null
@@ -1,74 +0,0 @@
-const { buildMatrix } = require("./build-e2e-matrix");
-
-const options = { java: 25, defaultRunner: "runner" };
-const specs = (count) =>
- Array.from({ length: count }, (_, i) => `e2e/test/scenarios/${i}.cy.spec.ts`);
-
-describe("buildMatrix", () => {
- it.each([0, 3, 3.5])("should reject an invalid job budget: %s", (chunks) => {
- expect(() => buildMatrix(options, specs(1), chunks)).toThrow();
- });
-
- it("should use the full job budget when there is no plan", () => {
- const result = buildMatrix(options, null, 50);
-
- expect(result.regularChunks).toBe(47);
- expect(result.config).toHaveLength(50);
- expect(result.config[0]).toMatchObject({
- name: "e2e-group-01",
- edition: "ee",
- runner: "runner",
- });
- expect(result.config[46].name).toBe("e2e-group-47");
- });
-
- it("should create no jobs for an empty plan", () => {
- expect(buildMatrix(options, [], 50)).toEqual({
- config: [],
- regularChunks: 0,
- });
- });
-
- it.each([
- [1, 1],
- [5, 1],
- [6, 2],
- [12, 3],
- [424, 47],
- ])(
- "should allocate %i selected specs to %i regular jobs",
- (count, regularChunks) => {
- const result = buildMatrix(options, specs(count), 50);
-
- expect(result.regularChunks).toBe(regularChunks);
- expect(result.config).toHaveLength(regularChunks + 3);
- },
- );
-
- it.each([null, specs(1), specs(424)])(
- "should retain every tagged variant",
- (plan) => {
- const result = buildMatrix(options, plan, 50);
-
- expect(result.config.slice(-3)).toMatchObject([
- { name: "oss-subset", edition: "oss", tags: "@OSS @prerelease+-@EE" },
- { name: "mongo", edition: "ee", tags: "@mongo" },
- { name: "python", edition: "ee", tags: "@python" },
- ]);
- },
- );
-
- it("should leave regular jobs to cypress-split", () => {
- const result = buildMatrix(options, specs(12), 50);
-
- const regularJobs = result.config.slice(0, result.regularChunks);
- expect(regularJobs.map((job) => job.name)).toEqual([
- "e2e-group-01",
- "e2e-group-02",
- "e2e-group-03",
- ]);
- for (const job of regularJobs) {
- expect(job.specs).toBeUndefined();
- }
- });
-});
diff --git a/.github/scripts/build-e2e-matrix.unit.spec.ts b/.github/scripts/build-e2e-matrix.unit.spec.ts
new file mode 100644
index 00000000000..50e2eea70c5
--- /dev/null
+++ b/.github/scripts/build-e2e-matrix.unit.spec.ts
@@ -0,0 +1,156 @@
+import { type Timings, buildMatrix } from "./build-e2e-matrix";
+
+const MINUTE_MS = 60_000;
+
+const specs = (count: number) =>
+ Array.from({ length: count }, (_, i) => `e2e/test/scenarios/${i}.cy.spec.ts`);
+
+// Gives every spec from specs(count) the same duration, in the timings.json path format.
+const timingsFor = (count: number, minutesPerSpec: number): Timings => ({
+ durations: Array.from({ length: count }, (_, i) => ({
+ spec: `../test/scenarios/${i}.cy.spec.ts`,
+ duration: minutesPerSpec * MINUTE_MS,
+ })),
+});
+
+type SetupOpts = {
+ specFiles?: string[] | null;
+ maxJobs?: number;
+ timings?: Timings;
+ targetRunnerMinutes?: number;
+};
+
+const setup = ({
+ specFiles = null,
+ maxJobs = 50,
+ timings = timingsFor(500, 1),
+ targetRunnerMinutes = 8,
+}: SetupOpts = {}) =>
+ buildMatrix({
+ java: 25,
+ defaultRunner: "runner",
+ specFiles,
+ maxJobs,
+ timings,
+ targetRunnerMinutes,
+ });
+
+describe("buildMatrix", () => {
+ it.each([0, 3, 3.5])("should reject an invalid job budget: %s", (maxJobs) => {
+ expect(() => setup({ specFiles: specs(1), maxJobs })).toThrow();
+ });
+
+ it.each([0, -1, NaN])(
+ "should reject an invalid target runner duration: %s",
+ (targetRunnerMinutes) => {
+ expect(() => setup({ targetRunnerMinutes })).toThrow();
+ },
+ );
+
+ it("should size the full suite from all recorded timings when there is no plan", () => {
+ const result = setup({ timings: timingsFor(80, 1) });
+
+ expect(result.regularChunks).toBe(10);
+ expect(result.config).toHaveLength(13);
+ expect(result.config[0]).toMatchObject({
+ name: "e2e-group-01",
+ edition: "ee",
+ runner: "runner",
+ });
+ expect(result.config[9].name).toBe("e2e-group-10");
+ });
+
+ it("should cap the full suite at the job budget", () => {
+ const result = setup({ timings: timingsFor(500, 1) });
+
+ expect(result.regularChunks).toBe(47);
+ expect(result.config).toHaveLength(50);
+ });
+
+ it("should create no jobs for an empty plan", () => {
+ expect(setup({ specFiles: [] })).toEqual({
+ config: [],
+ regularChunks: 0,
+ });
+ });
+
+ it.each([
+ [1, 1],
+ [8, 1],
+ [9, 2],
+ [24, 3],
+ [500, 47],
+ ])(
+ "should allocate %i selected one-minute specs to %i regular jobs",
+ (count, regularChunks) => {
+ const result = setup({ specFiles: specs(count) });
+
+ expect(result.regularChunks).toBe(regularChunks);
+ expect(result.config).toHaveLength(regularChunks + 3);
+ },
+ );
+
+ it("should apply the target runner duration", () => {
+ const result = setup({ specFiles: specs(24), targetRunnerMinutes: 4 });
+
+ expect(result.regularChunks).toBe(6);
+ });
+
+ it("should count selected specs without recorded timings as the average duration", () => {
+ // Recorded durations of 1 and 3 minutes average to 2 minutes.
+ const timings: Timings = {
+ durations: [
+ { spec: "../test/scenarios/short.cy.spec.ts", duration: MINUTE_MS },
+ { spec: "../test/scenarios/long.cy.spec.ts", duration: 3 * MINUTE_MS },
+ ],
+ };
+ const newSpecs = Array.from(
+ { length: 5 },
+ (_, i) => `e2e/test/scenarios/new-${i}.cy.spec.ts`,
+ );
+
+ const result = setup({
+ specFiles: ["e2e/test/scenarios/short.cy.spec.ts", ...newSpecs],
+ timings,
+ targetRunnerMinutes: 1,
+ });
+
+ expect(result.regularChunks).toBe(11);
+ });
+
+ it("should keep one regular job when there are no recorded timings", () => {
+ const result = setup({
+ specFiles: ["e2e/test/scenarios/new.cy.spec.ts"],
+ timings: { durations: [] },
+ });
+
+ expect(result.regularChunks).toBe(1);
+ });
+
+ it.each([null, specs(1), specs(500)])(
+ "should retain every tagged variant",
+ (specFiles) => {
+ const result = setup({ specFiles });
+
+ expect(result.config.slice(-3)).toMatchObject([
+ { name: "oss-subset", edition: "oss", tags: "@OSS @prerelease+-@EE" },
+ { name: "mongo", edition: "ee", tags: "@mongo" },
+ { name: "python", edition: "ee", tags: "@python" },
+ ]);
+ },
+ );
+
+ it("should leave regular jobs to cypress-split", () => {
+ const result = setup({ specFiles: specs(24) });
+
+ const regularJobs = result.config.slice(0, result.regularChunks);
+ expect(regularJobs.map((job) => job.name)).toEqual([
+ "e2e-group-01",
+ "e2e-group-02",
+ "e2e-group-03",
+ ]);
+ for (const job of regularJobs) {
+ expect(job.specs).toBeUndefined();
+ }
+ });
+});
diff --git a/.github/scripts/file-paths.unit.spec.ts b/.github/scripts/file-paths.unit.spec.ts
index 07867edc528..39a86fa8d8a 100644
--- a/.github/scripts/file-paths.unit.spec.ts
+++ b/.github/scripts/file-paths.unit.spec.ts
@@ -261,7 +261,7 @@ describe("file-paths.yaml", () => {
".github/workflows/e2e-test.yml",
".github/workflows/run-tests.yml",
".github/actions/prepare-cypress/action.yml",
- ".github/scripts/build-e2e-matrix.js",
+ ".github/scripts/build-e2e-matrix.ts",
".github/file-paths.yaml",
"frontend/build/shared/esbuild/side-effect-free-modules-plugin.js",
"rspack.main.config.js",
diff --git a/.github/workflows/e2e-matrix-builder.yml b/.github/workflows/e2e-matrix-builder.yml
index 926ea9c8018..e80f82d8dae 100644
--- a/.github/workflows/e2e-matrix-builder.yml
+++ b/.github/workflows/e2e-matrix-builder.yml
@@ -40,9 +40,9 @@ jobs:
/.nvmrc
/package.json
e2e/runner/read-spec-paths.js
+ e2e/support/timings.json
sparse-checkout-cone-mode: false
- name: Prepare Node + Bun environment
- if: ${{ inputs.test-plan-artifact-id != '' }}
uses: ./.github/actions/prepare-node-bun
- name: Prepare E2E spec selection
id: selection
@@ -52,28 +52,11 @@ jobs:
suite: e2e
test-plan-artifact-id: ${{ inputs.test-plan-artifact-id }}
- name: Build e2e matrix
- uses: actions/github-script@v9
id: matrix
env:
- chunks: ${{ inputs.chunks }}
+ JAVA_VERSION: 25
+ DEFAULT_RUNNER: ${{ vars.DEFAULT_RUNNER_KEY }}
+ E2E_MAX_JOBS: ${{ inputs.chunks }}
E2E_SPEC_PATHS_FILE: ${{ steps.selection.outputs.paths-file }}
- with:
- script: | # js
- const { buildMatrix } = require('${{ github.workspace }}/.github/scripts/build-e2e-matrix.js');
- const { readSpecPaths } = require('${{ github.workspace }}/e2e/runner/read-spec-paths.js');
- const specFiles = readSpecPaths(process.env.E2E_SPEC_PATHS_FILE);
- const inputChunks = Number(process.env.chunks);
- const java = 25;
- const defaultRunner = "${{ vars.DEFAULT_RUNNER_KEY }}";
- const options = {
- java,
- defaultRunner,
- };
- const { config, regularChunks } = buildMatrix(options, specFiles, inputChunks);
-
- core.setOutput("regularChunks", JSON.stringify(regularChunks));
- core.setOutput("config", JSON.stringify({ include: config }));
- core.setOutput("hasTests", config.length > 0);
-
- console.log({ regularChunks });
- console.log("matrix config", config);
+ E2E_TARGET_RUNNER_MINUTES: ${{ vars.E2E_TARGET_RUNNER_MINUTES || 8 }}
+ run: bun .github/scripts/build-e2e-matrix.ts
diff --git a/e2e/runner/read-spec-paths.js b/e2e/runner/read-spec-paths.js
index d355c743181..80e06e03e0b 100644
--- a/e2e/runner/read-spec-paths.js
+++ b/e2e/runner/read-spec-paths.js
@@ -1,5 +1,9 @@
const { readFileSync } = require("node:fs");
+/**
+ * @param {string | undefined} pathsFile
+ * @returns {string[] | null} null when no paths file is given
+ */
function readSpecPaths(pathsFile) {
if (!pathsFile) {
return null;