Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

turn cross version tests back on (#82773)

Chris Vandoodewaardff22433a51b5
diff --git a/.github/scripts/build-cross-version-matrix.js b/.github/scripts/build-cross-version-matrix.js
index ec3d59dc26d..2fe23f95813 100644
--- a/.github/scripts/build-cross-version-matrix.js
+++ b/.github/scripts/build-cross-version-matrix.js
@@ -4,15 +4,20 @@
/**
* Generate a CI matrix for cross-version migration testing (EE only).
*
- * Creates pairs of [source, target] for testing HEAD against the last N
- * minor versions in both upgrade and downgrade directions.
+ * Creates pairs of [source, target] for testing HEAD against every in-support
+ * major version, in both upgrade and downgrade directions.
*
- * @param {number} currentVersion - The current major version number (e.g., 59)
- * @param {number} count - Number of previous minor versions to test against (default: 3)
+ * The caller supplies the supported majors — in CI that comes from
+ * `getSupportedMajors()`, which reads `major_version_support` from
+ * version-info.json and keeps the lines whose `eol` is today or later. That is
+ * the same source auto-backport and schedule-minor use, so a major drops out of
+ * this matrix on the day it stops being a backport target.
+ *
+ * @param {number[]} supportedMajors - In-support major version numbers (e.g. [59, 58, 56])
* @returns {{ config: Array<{ source: string, target: string }> }}
*
* @example
- * buildMatrix(59)
+ * buildCrossVersionMatrix([59, 58, 56])
* // Returns:
* // {
* // config: [
@@ -20,31 +25,40 @@
* // { source: "v1.59.x", target: "HEAD" },
* // { source: "HEAD", target: "v1.58.x" },
* // { source: "v1.58.x", target: "HEAD" },
- * // { source: "HEAD", target: "v1.57.x" },
- * // { source: "v1.57.x", target: "HEAD" },
+ * // { source: "HEAD", target: "v1.56.x" },
+ * // { source: "v1.56.x", target: "HEAD" },
* // ]
* // }
*/
-function buildCrossVersionMatrix(currentVersion, count = 3) {
- if (!Number.isInteger(currentVersion) || currentVersion < 1) {
+function buildCrossVersionMatrix(supportedMajors) {
+ if (!Array.isArray(supportedMajors) || supportedMajors.length === 0) {
throw new Error(
- `Invalid currentVersion: ${currentVersion}. Must be a positive integer.`,
+ `Invalid supportedMajors: ${JSON.stringify(supportedMajors)}. Must be a non-empty array of positive integers.`,
);
}
- if (!Number.isInteger(count) || count < 1) {
- throw new Error(`Invalid count: ${count}. Must be a positive integer.`);
+ // findIndex rather than find, so an `undefined` entry is still reported.
+ const invalidIndex = supportedMajors.findIndex(
+ (majorVersion) => !Number.isInteger(majorVersion) || majorVersion < 1,
+ );
+
+ if (invalidIndex !== -1) {
+ throw new Error(
+ `Invalid major version: ${supportedMajors[invalidIndex]}. Must be a positive integer.`,
+ );
}
- const config = Array.from({ length: count }, (_, i) => currentVersion - i)
- .filter((majorVersion) => majorVersion >= 1)
- .flatMap((majorVersion) => {
- const version = `v1.${majorVersion}.x`;
- return [
- { source: "HEAD", target: version },
- { source: version, target: "HEAD" },
- ];
- });
+ // Newest first, so the most recently released major is tested first and a
+ // duplicated line in version-info.json can't double the matrix.
+ const majors = [...new Set(supportedMajors)].sort((a, b) => b - a);
+
+ const config = majors.flatMap((majorVersion) => {
+ const version = `v1.${majorVersion}.x`;
+ return [
+ { source: "HEAD", target: version },
+ { source: version, target: "HEAD" },
+ ];
+ });
return { config };
}
diff --git a/.github/scripts/build-cross-version-matrix.unit.spec.js b/.github/scripts/build-cross-version-matrix.unit.spec.js
index 2a7ed8d1df3..43f4b021aba 100644
--- a/.github/scripts/build-cross-version-matrix.unit.spec.js
+++ b/.github/scripts/build-cross-version-matrix.unit.spec.js
@@ -1,9 +1,9 @@
const { buildCrossVersionMatrix } = require("./build-cross-version-matrix");
describe("buildCrossVersionMatrix", () => {
- describe("default behavior", () => {
- it("should generate matrix with 3 versions by default", () => {
- const result = buildCrossVersionMatrix(59);
+ describe("supported versions", () => {
+ it("should generate a pair of entries per supported major", () => {
+ const result = buildCrossVersionMatrix([59, 58, 57]);
expect(result).toEqual({
config: [
{ source: "HEAD", target: "v1.59.x" },
@@ -16,15 +16,32 @@ describe("buildCrossVersionMatrix", () => {
});
});
+ it("should follow the supported list rather than a fixed count", () => {
+ expect(buildCrossVersionMatrix([59]).config).toHaveLength(2);
+ expect(buildCrossVersionMatrix([61, 60, 59, 58, 57]).config).toHaveLength(
+ 10,
+ );
+ });
+
+ it("should skip majors that have dropped out of support", () => {
+ // v1.57.x is out of support, so nothing migrates to or from it.
+ const result = buildCrossVersionMatrix([59, 58, 56]);
+ const versions = result.config
+ .filter((e) => e.source === "HEAD")
+ .map((e) => e.target);
+
+ expect(versions).toEqual(["v1.59.x", "v1.58.x", "v1.56.x"]);
+ expect(JSON.stringify(result)).not.toContain("v1.57.x");
+ });
+
it("should use EE prefix (v1)", () => {
- const result = buildCrossVersionMatrix(50);
+ const result = buildCrossVersionMatrix([50]);
expect(result.config[0].target).toBe("v1.50.x");
expect(result.config[1].source).toBe("v1.50.x");
});
it("should create pairs in both directions (upgrade and downgrade)", () => {
- const result = buildCrossVersionMatrix(59);
- const entries = result.config;
+ const entries = buildCrossVersionMatrix([59, 58, 57]).config;
// Check HEAD -> version (upgrade test)
expect(entries.filter((e) => e.source === "HEAD")).toHaveLength(3);
@@ -33,72 +50,37 @@ describe("buildCrossVersionMatrix", () => {
});
});
- describe("custom count", () => {
- it("should generate matrix with custom count", () => {
- const result = buildCrossVersionMatrix(59, 5);
- expect(result.config).toHaveLength(10); // 5 versions × 2 directions
- expect(result.config).toEqual([
- { source: "HEAD", target: "v1.59.x" },
- { source: "v1.59.x", target: "HEAD" },
- { source: "HEAD", target: "v1.58.x" },
- { source: "v1.58.x", target: "HEAD" },
- { source: "HEAD", target: "v1.57.x" },
- { source: "v1.57.x", target: "HEAD" },
- { source: "HEAD", target: "v1.56.x" },
- { source: "v1.56.x", target: "HEAD" },
- { source: "HEAD", target: "v1.55.x" },
- { source: "v1.55.x", target: "HEAD" },
- ]);
- });
-
- it("should generate matrix with count=1", () => {
- const result = buildCrossVersionMatrix(59, 1);
- expect(result.config).toHaveLength(2);
- expect(result.config).toEqual([
- { source: "HEAD", target: "v1.59.x" },
- { source: "v1.59.x", target: "HEAD" },
- ]);
- });
-
- it("should generate matrix with large count", () => {
- const result = buildCrossVersionMatrix(59, 10);
- expect(result.config).toHaveLength(20);
-
- // Check first and last entries
- expect(result.config[0]).toEqual({
- source: "HEAD",
- target: "v1.59.x",
- });
- expect(result.config[18]).toEqual({
- source: "HEAD",
- target: "v1.50.x",
- });
- expect(result.config[19]).toEqual({
- source: "v1.50.x",
- target: "HEAD",
- });
+ describe("normalization", () => {
+ it("should sort majors newest first regardless of input order", () => {
+ const result = buildCrossVersionMatrix([57, 59, 58]);
+ const versions = result.config
+ .filter((e) => e.source === "HEAD")
+ .map((e) => e.target);
+
+ expect(versions).toEqual(["v1.59.x", "v1.58.x", "v1.57.x"]);
});
- });
- describe("edge cases", () => {
- it("should handle version 1", () => {
- const result = buildCrossVersionMatrix(1);
+ it("should not double up on a duplicated major", () => {
+ // version-info.json's `major_version_support` is append-only, so a line
+ // can show up more than once.
+ const result = buildCrossVersionMatrix([59, 58, 59]);
+ expect(result.config).toHaveLength(4);
expect(result).toEqual({
config: [
- { source: "HEAD", target: "v1.1.x" },
- { source: "v1.1.x", target: "HEAD" },
+ { source: "HEAD", target: "v1.59.x" },
+ { source: "v1.59.x", target: "HEAD" },
+ { source: "HEAD", target: "v1.58.x" },
+ { source: "v1.58.x", target: "HEAD" },
],
});
});
+ });
- it("should stop when version would go below 1", () => {
- const result = buildCrossVersionMatrix(2, 5);
- // Can only generate v1.2.x and v1.1.x (2 versions, not 5)
- expect(result.config).toHaveLength(4);
+ describe("edge cases", () => {
+ it("should handle version 1", () => {
+ const result = buildCrossVersionMatrix([1]);
expect(result).toEqual({
config: [
- { source: "HEAD", target: "v1.2.x" },
- { source: "v1.2.x", target: "HEAD" },
{ source: "HEAD", target: "v1.1.x" },
{ source: "v1.1.x", target: "HEAD" },
],
@@ -106,7 +88,7 @@ describe("buildCrossVersionMatrix", () => {
});
it("should handle high version numbers", () => {
- const result = buildCrossVersionMatrix(100, 2);
+ const result = buildCrossVersionMatrix([100, 99]);
expect(result).toEqual({
config: [
{ source: "HEAD", target: "v1.100.x" },
@@ -119,64 +101,73 @@ describe("buildCrossVersionMatrix", () => {
});
describe("error handling", () => {
- it("should throw for undefined currentVersion", () => {
+ it("should throw for undefined supportedMajors", () => {
expect(() => buildCrossVersionMatrix()).toThrow(
- "Invalid currentVersion: undefined. Must be a positive integer.",
+ "Invalid supportedMajors: undefined. Must be a non-empty array of positive integers.",
+ );
+ });
+
+ it("should throw for an empty list", () => {
+ // getSupportedMajors throws on its own when nothing is in support, so an
+ // empty matrix here means something upstream went wrong — fail loudly
+ // instead of silently testing no versions at all.
+ expect(() => buildCrossVersionMatrix([])).toThrow(
+ "Invalid supportedMajors: []. Must be a non-empty array of positive integers.",
);
});
- it("should throw for non-integer currentVersion", () => {
- expect(() => buildCrossVersionMatrix(59.5)).toThrow(
- "Invalid currentVersion: 59.5. Must be a positive integer.",
+ it("should throw for a non-array", () => {
+ expect(() => buildCrossVersionMatrix(59)).toThrow(
+ "Invalid supportedMajors: 59. Must be a non-empty array of positive integers.",
);
});
- it("should throw for zero currentVersion", () => {
- expect(() => buildCrossVersionMatrix(0)).toThrow(
- "Invalid currentVersion: 0. Must be a positive integer.",
+ it("should throw for a non-integer major", () => {
+ expect(() => buildCrossVersionMatrix([59, 58.5])).toThrow(
+ "Invalid major version: 58.5. Must be a positive integer.",
);
});
- it("should throw for negative currentVersion", () => {
- expect(() => buildCrossVersionMatrix(-1)).toThrow(
- "Invalid currentVersion: -1. Must be a positive integer.",
+ it("should throw for a zero major", () => {
+ expect(() => buildCrossVersionMatrix([59, 0])).toThrow(
+ "Invalid major version: 0. Must be a positive integer.",
);
});
- it("should throw for NaN currentVersion", () => {
- expect(() => buildCrossVersionMatrix(NaN)).toThrow(
- "Invalid currentVersion: NaN. Must be a positive integer.",
+ it("should throw for a negative major", () => {
+ expect(() => buildCrossVersionMatrix([-1])).toThrow(
+ "Invalid major version: -1. Must be a positive integer.",
);
});
- it("should throw for non-integer count", () => {
- expect(() => buildCrossVersionMatrix(59, 2.5)).toThrow(
- "Invalid count: 2.5. Must be a positive integer.",
+ it("should throw for a NaN major", () => {
+ expect(() => buildCrossVersionMatrix([NaN])).toThrow(
+ "Invalid major version: NaN. Must be a positive integer.",
);
});
- it("should throw for zero count", () => {
- expect(() => buildCrossVersionMatrix(59, 0)).toThrow(
- "Invalid count: 0. Must be a positive integer.",
+ it("should throw for an undefined major", () => {
+ expect(() => buildCrossVersionMatrix([59, undefined])).toThrow(
+ "Invalid major version: undefined. Must be a positive integer.",
);
});
- it("should throw for negative count", () => {
- expect(() => buildCrossVersionMatrix(59, -1)).toThrow(
- "Invalid count: -1. Must be a positive integer.",
+ it("should throw for a stringified major", () => {
+ expect(() => buildCrossVersionMatrix(["59"])).toThrow(
+ "Invalid major version: 59. Must be a positive integer.",
);
});
});
describe("matrix structure for GitHub Actions", () => {
it("should return object with config array", () => {
- const result = buildCrossVersionMatrix(59);
+ const result = buildCrossVersionMatrix([59, 58, 57]);
expect(result).toHaveProperty("config");
expect(Array.isArray(result.config)).toBe(true);
});
it("should produce valid JSON for GitHub Actions", () => {
- const result = buildCrossVersionMatrix(59);
+ const result = buildCrossVersionMatrix([59, 58, 57]);
const json = JSON.stringify(result);
expect(() => JSON.parse(json)).not.toThrow();
@@ -189,7 +180,7 @@ describe("buildCrossVersionMatrix", () => {
});
it("should have consistent entry structure", () => {
- const result = buildCrossVersionMatrix(59, 5);
+ const result = buildCrossVersionMatrix([59, 58, 57, 56, 55]);
result.config.forEach((entry) => {
expect(entry).toHaveProperty("source");
@@ -200,26 +191,9 @@ describe("buildCrossVersionMatrix", () => {
expect(entry.source === "HEAD" || entry.target === "HEAD").toBeTruthy();
});
});
- });
-
- describe("version ordering", () => {
- it("should generate versions in descending order", () => {
- const result = buildCrossVersionMatrix(59, 5);
- const versions = result.config
- .filter((e) => e.source === "HEAD")
- .map((e) => e.target);
-
- expect(versions).toEqual([
- "v1.59.x",
- "v1.58.x",
- "v1.57.x",
- "v1.56.x",
- "v1.55.x",
- ]);
- });
it("should interleave upgrade and downgrade tests", () => {
- const result = buildCrossVersionMatrix(59, 2);
+ const result = buildCrossVersionMatrix([59, 58]);
expect(result.config[0]).toEqual({
source: "HEAD",
diff --git a/.github/workflows/containerize-jar.yml b/.github/workflows/containerize-jar.yml
index 449f2b402aa..99d4e960d16 100644
--- a/.github/workflows/containerize-jar.yml
+++ b/.github/workflows/containerize-jar.yml
@@ -29,6 +29,14 @@ on:
description: Platforms the image is built for.
type: string
default: linux/amd64,linux/arm64
+ outputs:
+ digest-ee:
+ description: Image digest, useful for pulling the built image from Docker Hub. Only set if this is the 'ee' edition.
+ value: ${{ jobs.containerize.outputs.digest-ee }}
+ digest-oss:
+ description: Image digest, useful for pulling the built image from Docker Hub. Only set if this is the 'oss' edition.
+ value: ${{ jobs.containerize.outputs.digest-oss }}
+
workflow_dispatch:
inputs:
edition:
@@ -70,6 +78,9 @@ jobs:
timeout-minutes: 15
outputs:
full_image: ${{ steps.image.outputs.full_image }}
+ digest: ${{ steps.build.outputs.digest }}
+ digest-ee: ${{ inputs.edition == 'ee' && steps.build.outputs.digest || '' }}
+ digest-oss: ${{ inputs.edition == 'oss' && steps.build.outputs.digest || '' }}
env:
AWS_REGION: us-east-1
steps:
@@ -130,6 +141,7 @@ jobs:
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
- name: Build and Push Images
+ id: build
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
with:
context: bin/docker/.
diff --git a/.github/workflows/cross-version.yml b/.github/workflows/cross-version.yml
index 493190efd40..1a9a804915d 100644
--- a/.github/workflows/cross-version.yml
+++ b/.github/workflows/cross-version.yml
@@ -1,6 +1,14 @@
name: Cross-Version Migration Tests
on:
+ workflow_call:
+ inputs:
+ digest:
+ type: string
+ required: false
+ default: ""
+ description: "HEAD image digest. Ensures we test against the image built for the current commit."
+
workflow_dispatch:
inputs:
source_version:
@@ -12,49 +20,55 @@ on:
required: true
type: string
- # NOTE: this workflow is temporarily disabled until the cross version tests are fixed.
- # pull_request:
- # paths:
- # - "cross-version/**"
- # - ".github/workflows/cross-version.yml"
- # - ".github/scripts/build-cross-version-matrix.js"
-
- # schedule:
- # # Run nightly at 3 AM UTC
- # - cron: "0 3 * * *"
-
jobs:
- # Generate matrix for scheduled/PR runs
- generate-matrix:
- if: github.event_name != 'workflow_dispatch' && github.repository == 'metabase/metabase'
+ generate-test-matrix:
+ if: inputs.source_version == '' && github.repository == 'metabase/metabase'
runs-on: ${{ vars.SLIM_RUNNER_KEY }}
timeout-minutes: 5
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
+ - name: Require HEAD digest
+ if: inputs.digest == ''
+ run: |
+ echo "::error::inputs.digest is empty and required to test the current commit's Docker image"
+ exit 1
+
- uses: actions/checkout@v7
with:
- sparse-checkout: .github
+ sparse-checkout: |
+ release
+ .github
+
+ - name: Prepare release scripts
+ uses: ./.github/actions/prepare-release-scripts
- name: Generate version matrix
id: set-matrix
uses: actions/github-script@v9
+ env:
+ AWS_S3_STATIC_BUCKET: ${{ vars.AWS_S3_STATIC_BUCKET }}
+ AWS_REGION: ${{ vars.AWS_REGION }}
with:
- script: |
+ script: | # js
+ const { getSupportedMajors } = require('${{ github.workspace }}/release/dist/index.cjs');
const { buildCrossVersionMatrix } = require('${{ github.workspace }}/.github/scripts/build-cross-version-matrix');
- const { config } = buildCrossVersionMatrix(${{ vars.CURRENT_VERSION }});
+
+ // run against all supported versions
+ const supportedMajors = await getSupportedMajors();
+ console.log("supported majors:", supportedMajors);
+
+ const { config } = buildCrossVersionMatrix(supportedMajors);
const matrix = JSON.stringify({ include: config });
core.setOutput("matrix", matrix);
console.log("matrix:", matrix);
- # Matrix-based tests for scheduled/PR runs
- test-matrix:
- if: github.event_name != 'workflow_dispatch' && github.repository == 'metabase/metabase'
- needs: generate-matrix
+ test:
+ needs: generate-test-matrix
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
timeout-minutes: 30
strategy:
- matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
+ matrix: ${{ fromJson(needs.generate-test-matrix.outputs.matrix) }}
fail-fast: false
steps:
@@ -69,25 +83,34 @@ jobs:
- name: Prepare Cypress environment
uses: ./.github/actions/prepare-cypress
+ - name: Download pre-built CLJS
+ id: prebuilt-cljs
+ uses: actions/download-artifact@v8
+ continue-on-error: true
+ with:
+ name: e2e-cljs-${{ github.event.pull_request.head.sha || github.sha }}
+ path: target/cljs_dev
+
+ - name: Prepare back-end environment
+ if: steps.prebuilt-cljs.outcome != 'success'
+ uses: ./.github/actions/prepare-backend
+
+ - name: Compile CLJS
+ if: steps.prebuilt-cljs.outcome != 'success'
+ run: bun run build-pure:cljs
+
- name: Run cross-version test
env:
CURRENT_VERSION: ${{ vars.CURRENT_VERSION }}
- RESULT_FILE: ${{ runner.temp }}/cross-version-result-${{ matrix.source }}-${{ matrix.target }}.json
+ HEAD_DIGEST: ${{ inputs.digest }}
+ CYPRESS_KEEP_VIDEOS: "true"
run: |
cd cross-version
./test.sh --source "${{ matrix.source }}" --target "${{ matrix.target }}"
- - name: Upload failure result
+ - name: Upload Cypress Artifacts
uses: actions/upload-artifact@v7
- if: failure()
- with:
- name: failure-result-${{ matrix.source }}-${{ matrix.target }}
- path: ${{ runner.temp }}/cross-version-result-${{ matrix.source }}-${{ matrix.target }}.json
- if-no-files-found: ignore
-
- - name: Upload Cypress Artifacts upon failure
- uses: actions/upload-artifact@v7
- if: failure()
+ if: always()
with:
name: cypress-recording-${{ matrix.source }}-${{ matrix.target }}
path: |
@@ -105,91 +128,9 @@ jobs:
echo "::endgroup::"
done
- # Notify Slack on nightly failures
- notify-on-failure:
- if: >-
- always()
- && github.event_name == 'schedule'
- && github.repository == 'metabase/metabase'
- && needs.test-matrix.result == 'failure'
- needs: [test-matrix]
- runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
- timeout-minutes: 5
- steps:
- - uses: actions/checkout@v7
- with:
- sparse-checkout: |
- release
- .github
-
- - name: Prepare release scripts
- uses: ./.github/actions/prepare-release-scripts
-
- - name: Download failure results
- uses: actions/download-artifact@v8
- continue-on-error: true
- with:
- pattern: failure-result-*
- path: failure-results
- merge-multiple: true
-
- - name: Notify Slack
- continue-on-error: true
- uses: actions/github-script@v9
- env:
- SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
- with:
- script: |
- const fs = require('fs');
- const path = require('path');
- const { buildFailureResults, sendCrossVersionSlackNotification } = require('${{ github.workspace }}/release/dist/index.cjs');
-
- // Get all failed jobs from the GitHub API
- const { owner, repo } = context.repo;
- const jobInfo = await github.rest.actions.listJobsForWorkflowRun({
- owner,
- repo,
- run_id: context.runId,
- per_page: 100,
- });
-
- const failedJobs = jobInfo.data.jobs
- .filter(job => job.status === 'completed' && job.conclusion === 'failure' && job.name.startsWith('test-matrix'))
- .map(job => ({ name: job.name, url: job.html_url }));
-
- if (failedJobs.length === 0) {
- console.log('No failed test-matrix jobs found');
- return;
- }
-
- // Load artifact data for enrichment (phase)
- const resultsDir = 'failure-results';
- const artifactData = new Map();
- if (fs.existsSync(resultsDir)) {
- for (const file of fs.readdirSync(resultsDir)) {
- if (!file.endsWith('.json')) continue;
- try {
- const result = JSON.parse(fs.readFileSync(path.join(resultsDir, file), 'utf8'));
- artifactData.set(`${result.source}-${result.target}`, result);
- } catch (e) {
- console.log(`Skipping malformed result: ${file}`);
- }
- }
- }
-
- const failures = buildFailureResults(failedJobs, artifactData);
- const runUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${context.runId}`;
- const slackBotToken = process.env.SLACK_BOT_TOKEN;
- if (!slackBotToken) {
- console.log('SLACK_BOT_TOKEN is not set; skipping Slack notification.');
- return;
- }
- await sendCrossVersionSlackNotification(failures, runUrl, slackBotToken);
- console.log('Slack notification sent');
-
# Manual test for workflow_dispatch
test-manual:
- if: github.event_name == 'workflow_dispatch'
+ if: inputs.source_version != ''
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
timeout-minutes: 30
@@ -205,16 +146,33 @@ jobs:
- name: Prepare Cypress environment
uses: ./.github/actions/prepare-cypress
+ - name: Download pre-built CLJS
+ id: prebuilt-cljs
+ uses: actions/download-artifact@v8
+ continue-on-error: true
+ with:
+ name: e2e-cljs-${{ github.event.pull_request.head.sha || github.sha }}
+ path: target/cljs_dev
+
+ - name: Prepare back-end environment
+ if: steps.prebuilt-cljs.outcome != 'success'
+ uses: ./.github/actions/prepare-backend
+
+ - name: Compile CLJS
+ if: steps.prebuilt-cljs.outcome != 'success'
+ run: bun run build-pure:cljs
+
- name: Run cross-version test
env:
CURRENT_VERSION: ${{ vars.CURRENT_VERSION }}
+ CYPRESS_KEEP_VIDEOS: "true"
run: |
cd cross-version
./test.sh --source "${{ inputs.source_version }}" --target "${{ inputs.target_version }}"
- - name: Upload Cypress Artifacts upon failure
+ - name: Upload Cypress Artifacts
uses: actions/upload-artifact@v7
- if: failure()
+ if: always()
with:
name: cypress-recording-${{ inputs.source_version }}-${{ inputs.target_version }}
path: |
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index a4e01aa8615..03feec6a760 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -74,6 +74,15 @@ jobs:
edition: ${{ matrix.edition }}
release: false
+ # Should only run when we push an image to metabase/metabase-head
+ cross-version-tests:
+ name: Cross-version migration tests
+ needs: [containerize]
+ uses: ./.github/workflows/cross-version.yml
+ secrets: inherit
+ with:
+ digest: ${{ needs.containerize.outputs.digest-ee }}
+
bundle-size:
needs: uberjar
uses: ./.github/workflows/bundle-size.yml
diff --git a/cross-version/test.sh b/cross-version/test.sh
index 9a9e47e1a5f..7c01a691d90 100755
--- a/cross-version/test.sh
+++ b/cross-version/test.sh
@@ -19,7 +19,6 @@ SOURCE_VERSION=""
TARGET_VERSION=""
METABASE_PORT="${METABASE_PORT:-3000}"
HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-120}"
-RESULT_FILE="${RESULT_FILE:-/tmp/cross-version-result.json}"
LOG_DIR="${LOG_DIR:-$SCRIPT_DIR/logs}"
# Colors for output
@@ -32,17 +31,6 @@ log() { echo -e "${GREEN}[cross-version]${NC} $*"; }
warn() { echo -e "${YELLOW}[cross-version]${NC} $*"; }
error() { echo -e "${RED}[cross-version]${NC} $*" >&2; }
-# Write a structured result file on failure so CI can categorize the failure
-write_failure() {
- local phase="$1" # "migration" or "e2e"
- jq -n \
- --arg phase "$phase" \
- --arg source "$SOURCE_VERSION" \
- --arg target "$TARGET_VERSION" \
- '{phase:$phase, source:$source, target:$target}' > "$RESULT_FILE"
- log "Wrote failure result to $RESULT_FILE"
-}
-
usage() {
cat <<EOF
Usage: $0 --source VERSION --target VERSION
@@ -404,7 +392,6 @@ main() {
if ! wait_for_health "$HEALTH_TIMEOUT"; then
error "❌ SOURCE version ($SOURCE_VERSION) failed health check"
docker compose logs metabase
- write_failure "migration"
exit 1
fi
@@ -413,7 +400,6 @@ main() {
log ""
log "Step 2: Running e2e tests (@source)..."
if ! run_e2e source "$SOURCE_VERSION"; then
- write_failure "e2e"
exit 1
fi
@@ -430,7 +416,6 @@ main() {
if ! wait_for_health "$HEALTH_TIMEOUT"; then
error "❌ TARGET version ($TARGET_VERSION) failed health check after upgrade"
docker compose logs metabase
- write_failure "migration"
exit 1
fi
log "✅ UPGRADE successful - TARGET version ($TARGET_VERSION) is healthy"
@@ -438,7 +423,6 @@ main() {
log ""
log "Step 5: Running e2e tests (@target)..."
if ! run_e2e target "$TARGET_VERSION"; then
- write_failure "e2e"
exit 1
fi
@@ -447,7 +431,6 @@ main() {
if ! check_downgrade_refused; then
error "❌ TARGET version ($TARGET_VERSION) did not properly detect downgrade"
docker compose logs metabase
- write_failure "migration"
exit 1
fi
@@ -458,7 +441,6 @@ main() {
log "Step 5: Rolling back database ($SOURCE_VERSION → $TARGET_VERSION)..."
if ! cascading_migrate_down "$SOURCE_VERSION" "$TARGET_VERSION"; then
error "❌ migrate down failed"
- write_failure "migration"
exit 1
fi
@@ -470,7 +452,6 @@ main() {
if ! wait_for_health "$HEALTH_TIMEOUT"; then
error "❌ TARGET version ($TARGET_VERSION) failed health check after migrate down"
docker compose logs metabase
- write_failure "migration"
exit 1
fi
log "✅ DOWNGRADE successful - TARGET version ($TARGET_VERSION) is healthy after migrate down"
@@ -478,7 +459,6 @@ main() {
log ""
log "Step 7: Running e2e tests (@target)..."
if ! run_e2e target "$TARGET_VERSION"; then
- write_failure "e2e"
exit 1
fi
fi
diff --git a/e2e/cross-version/latest/03-nested-model.cy.spec.ts b/e2e/cross-version/latest/03-nested-model.cy.spec.ts
index c991621fd14..5973dd61090 100644
--- a/e2e/cross-version/latest/03-nested-model.cy.spec.ts
+++ b/e2e/cross-version/latest/03-nested-model.cy.spec.ts
@@ -65,10 +65,7 @@ describe("Cross-version questions - nested model", () => {
X.visitRootCollectionAndWait();
- H.getPinnedSection()
- .findByRole("link")
- .should("contain", Q3_NAME)
- .click();
+ cy.findAllByRole("link").filter(`:contains(${Q3_NAME})`).first().click();
cy.findByTestId("question-row-count").should(
"have.text",
diff --git a/e2e/cross-version/latest/04-sql.cy.spec.ts b/e2e/cross-version/latest/04-sql.cy.spec.ts
index a5def1685cc..b5419656006 100644
--- a/e2e/cross-version/latest/04-sql.cy.spec.ts
+++ b/e2e/cross-version/latest/04-sql.cy.spec.ts
@@ -88,6 +88,15 @@ describe("Cross-version questions - sql", () => {
);
it("verify: sql question can be filtered", { tags: ["@target"] }, () => {
+ // As a result of PR #77133, cards created in >=v63 fail to load on older
+ // versions. We explicitly don't guarantee that things created in a new version of
+ // Metabase will work when rolled back to an older version so skipping this test for
+ // versions below v63 aligns with our support commitments.
+ cy.request("/api/session/properties").then(({ body }) => {
+ const major = Number(body.version.tag.match(/^v[01]\.(\d+)\./)?.[1]);
+ cy.skipOn(major < 63);
+ });
+
cy.signIn("admin", { skipCache: true });
X.visitRootCollectionAndWait();
diff --git a/e2e/cross-version/latest/helpers.ts b/e2e/cross-version/latest/helpers.ts
index 44889f5e46e..fb22ea0b12d 100644
--- a/e2e/cross-version/latest/helpers.ts
+++ b/e2e/cross-version/latest/helpers.ts
@@ -8,6 +8,18 @@ export function saveQuestion(name: string) {
cy.findByTestId("qb-header").button("Save").click();
cy.findByTestId("save-question-modal").within(() => {
cy.findByLabelText("Name").clear().type(name);
+ cy.findByLabelText(/Where do you want to save this/).click();
+ });
+ H.entityPickerModal().within(() => {
+ H.entityPickerModalLevel(0).findByText("Our analytics").click();
+ cy.findByTestId("entity-picker-select-button").click();
+ });
+ H.entityPickerModal().should("not.exist");
+ cy.findByTestId("save-question-modal").within(() => {
+ cy.findByLabelText(/Where do you want to save this/).should(
+ "contain.text",
+ "Our analytics",
+ );
cy.button("Save").click();
cy.wait("@saveQuestion");
});
diff --git a/e2e/support/config.js b/e2e/support/config.js
index ba2c83ca9ab..9751ed3560b 100644
--- a/e2e/support/config.js
+++ b/e2e/support/config.js
@@ -367,7 +367,12 @@ const defaultConfig = {
// https://docs.cypress.io/guides/guides/screenshots-and-videos#Delete-videos-for-specs-without-failing-or-retried-tests
if (results && results.video) {
// Do we have test failures?
- if (results && results.video && results.stats.failures === 0) {
+ if (
+ results &&
+ results.video &&
+ results.stats.failures === 0 &&
+ process.env.CYPRESS_KEEP_VIDEOS !== "true"
+ ) {
// delete the video if the spec passed
fs.unlinkSync(results.video);
}
diff --git a/release/src/cross-version-helpers.ts b/release/src/cross-version-helpers.ts
index 4e52850caf7..328c6a5764f 100644
--- a/release/src/cross-version-helpers.ts
+++ b/release/src/cross-version-helpers.ts
@@ -37,8 +37,28 @@ export const getHeadMajorVersion = (currentVersion: string): string => {
return String(Number(currentVersion) + 1);
};
-/** Docker image for HEAD builds. */
-export const HEAD_DOCKER_IMAGE = "metabase/metabase-enterprise-head:latest";
+const HEAD_DOCKER_REPO = "metabase/metabase-enterprise-head";
+const PR_DOCKER_REPO = "metabase/metabase-dev";
+
+/**
+ * Docker image for HEAD builds.
+ *
+ * When HEAD_DIGEST is set (the digest of the image built for the current
+ * commit), pin to that digest so we test exactly that image. PR images are
+ * pushed to metabase-dev, everything else to metabase-enterprise-head.
+ * Otherwise fall back to the :latest tag.
+ */
+export const getHeadDockerImage = (): string => {
+ const digest = process.env.HEAD_DIGEST?.trim();
+ if (digest) {
+ const repo =
+ process.env.GITHUB_EVENT_NAME === "pull_request"
+ ? PR_DOCKER_REPO
+ : HEAD_DOCKER_REPO;
+ return `${repo}@${digest.startsWith("sha256:") ? digest : `sha256:${digest}`}`;
+ }
+ return `${HEAD_DOCKER_REPO}:latest`;
+};
// ============================================================================
// Pure helpers for rolling .x tags
@@ -138,11 +158,11 @@ export const compareVersions = (
/**
* Get the Docker image for a version.
- * Supports HEAD and rolling .x tags.
+ * Supports HEAD (along with digest) and rolling .x tags.
*/
export const getDockerImage = (version: string): string => {
if (isHead(version)) {
- return HEAD_DOCKER_IMAGE;
+ return getHeadDockerImage();
}
if (!isRollingTag(version) && !isValidVersionString(version)) {
diff --git a/release/src/cross-version-helpers.unit.spec.ts b/release/src/cross-version-helpers.unit.spec.ts
index 4dc76dd0006..e3dac83df9a 100644
--- a/release/src/cross-version-helpers.unit.spec.ts
+++ b/release/src/cross-version-helpers.unit.spec.ts
@@ -4,7 +4,7 @@ import {
getHeadMajorVersion,
getMajorVersion,
getRollingTagMajorVersion,
- HEAD_DOCKER_IMAGE,
+ getHeadDockerImage,
isHead,
isRollingTag,
isRollingTagEnterprise,
@@ -81,9 +81,67 @@ describe("cross-version-helpers", () => {
});
});
- describe("HEAD_DOCKER_IMAGE", () => {
- it("is the enterprise head image", () => {
- expect(HEAD_DOCKER_IMAGE).toBe(
+ describe("getHeadDockerImage", () => {
+ const originalDigest = process.env.HEAD_DIGEST;
+ const originalEventName = process.env.GITHUB_EVENT_NAME;
+
+ beforeEach(() => {
+ delete process.env.GITHUB_EVENT_NAME;
+ });
+
+ afterEach(() => {
+ if (originalDigest === undefined) {
+ delete process.env.HEAD_DIGEST;
+ } else {
+ process.env.HEAD_DIGEST = originalDigest;
+ }
+ if (originalEventName === undefined) {
+ delete process.env.GITHUB_EVENT_NAME;
+ } else {
+ process.env.GITHUB_EVENT_NAME = originalEventName;
+ }
+ });
+
+ it("uses the latest tag when HEAD_DIGEST is not set", () => {
+ delete process.env.HEAD_DIGEST;
+ expect(getHeadDockerImage()).toBe(
+ "metabase/metabase-enterprise-head:latest",
+ );
+ });
+
+ it("uses the latest tag when HEAD_DIGEST is empty", () => {
+ process.env.HEAD_DIGEST = " ";
+ expect(getHeadDockerImage()).toBe(
+ "metabase/metabase-enterprise-head:latest",
+ );
+ });
+
+ it("pins to the digest when HEAD_DIGEST is set", () => {
+ process.env.HEAD_DIGEST = "sha256:abc123";
+ expect(getHeadDockerImage()).toBe(
+ "metabase/metabase-enterprise-head@sha256:abc123",
+ );
+ });
+
+ it("adds the sha256 prefix when the digest is bare", () => {
+ process.env.HEAD_DIGEST = "abc123";
+ expect(getHeadDockerImage()).toBe(
+ "metabase/metabase-enterprise-head@sha256:abc123",
+ );
+ });
+
+ it("uses metabase-dev when running from a pull request", () => {
+ process.env.GITHUB_EVENT_NAME = "pull_request";
+ process.env.HEAD_DIGEST = "sha256:abc123";
+ expect(getHeadDockerImage()).toBe(
+ "metabase/metabase-dev@sha256:abc123",
+ );
+ });
+
+ it("uses the enterprise head latest tag on a pull request without a digest", () => {
+ process.env.GITHUB_EVENT_NAME = "pull_request";
+ delete process.env.HEAD_DIGEST;
+ expect(getHeadDockerImage()).toBe(
"metabase/metabase-enterprise-head:latest",
);
});
@@ -209,7 +267,7 @@ describe("cross-version-helpers", () => {
});
it("returns enterprise HEAD image for HEAD", () => {
- expect(getDockerImage("HEAD")).toBe(HEAD_DOCKER_IMAGE);
+ expect(getDockerImage("HEAD")).toBe(getHeadDockerImage());
});
});
});
diff --git a/release/src/cross-version-slack.ts b/release/src/cross-version-slack.ts
deleted file mode 100644
index 1c2d2eebd9d..00000000000
--- a/release/src/cross-version-slack.ts
+++ /dev/null
@@ -1,127 +0,0 @@
-/**
- * Slack notification helpers for cross-version migration test failures.
- *
- * buildSlackPayload is a pure function (for testability).
- * sendCrossVersionSlackNotification handles the posting via @slack/web-api.
- */
-import { WebClient } from "@slack/web-api";
-
-export interface FailureResult {
- phase: "migration" | "e2e";
- source: string;
- target: string;
- jobUrl?: string;
-}
-
-interface SlackBlock {
- type: string;
- text?: { type: string; text: string; emoji?: boolean };
- elements?: Array<{ type: string; text: string }>;
-}
-
-export interface SlackPayload {
- blocks: SlackBlock[];
-}
-
-export interface FailedJob {
- name: string;
- url: string;
-}
-
-/**
- * Parse source and target versions from a matrix job name.
- * e.g. "test-matrix (HEAD, v1.59.x)" -> { source: "HEAD", target: "v1.59.x" }
- */
-export const parseJobName = (
- name: string,
-): { source: string; target: string } | null => {
- const match = name.match(/\(([^,]+),\s*([^)]+)\)/);
- if (!match) {
- return null;
- }
- return { source: match[1], target: match[2] };
-};
-
-/**
- * Build a list of FailureResults from failed jobs and artifact data.
- * Each failed job becomes a result. Artifact data enriches with phase.
- */
-export const buildFailureResults = (
- failedJobs: FailedJob[],
- artifactData: Map<string, FailureResult>,
-): FailureResult[] => {
- return failedJobs.map(job => {
- const parsed = parseJobName(job.name);
- if (!parsed) {
- return {
- phase: "e2e" as const,
- source: "?",
- target: "?",
- jobUrl: job.url,
- };
- }
-
- const key = `${parsed.source}-${parsed.target}`;
- const artifact = artifactData.get(key);
-
- if (artifact) {
- return { phase: artifact.phase, source: artifact.source, target: artifact.target, jobUrl: job.url };
- }
-
- return {
- phase: "e2e" as const,
- source: parsed.source,
- target: parsed.target,
- jobUrl: job.url,
- };
- });
-};
-
-const formatFailureItem = (f: FailureResult): string => {
- const versions = f.jobUrl
- ? `<${f.jobUrl}|${f.source} → ${f.target}>`
- : `${f.source} → ${f.target}`;
- const label = f.phase === "migration" ? "migration failure" : "e2e failure";
- return `• ${versions} (${label})`;
-};
-
-export const buildSlackPayload = (
- failures: FailureResult[],
- runUrl: string,
-): SlackPayload => {
- const lines = failures.map(formatFailureItem);
-
- if (lines.length === 0) {
- lines.push(
- "One or more matrix jobs failed. Check the workflow run for details.",
- );
- }
-
- return {
- blocks: [
- {
- type: "section",
- text: {
- type: "mrkdwn",
- text: `:warning: *Cross-version tests failing*\n${lines.join("\n")}\n<${runUrl}|View full run>`,
- },
- },
- ],
- };
-};
-
-export const sendCrossVersionSlackNotification = async (
- failures: FailureResult[],
- runUrl: string,
- slackBotToken: string,
- channel = "engineering-ci",
-): Promise<void> => {
- const slack = new WebClient(slackBotToken);
- const { blocks } = buildSlackPayload(failures, runUrl);
-
- await slack.chat.postMessage({
- channel,
- text: "Cross-version tests failing",
- blocks,
- });
-};
diff --git a/release/src/cross-version-slack.unit.spec.ts b/release/src/cross-version-slack.unit.spec.ts
deleted file mode 100644
index 5f2df7151a0..00000000000
--- a/release/src/cross-version-slack.unit.spec.ts
+++ /dev/null
@@ -1,115 +0,0 @@
-import {
- buildFailureResults,
- buildSlackPayload,
- parseJobName,
- type FailedJob,
- type FailureResult,
-} from "./cross-version-slack";
-
-const RUN_URL = "https://github.com/metabase/metabase/actions/runs/12345";
-
-describe("cross-version-slack", () => {
- describe("parseJobName", () => {
- it("parses source and target from matrix job name", () => {
- expect(parseJobName("test-matrix (HEAD, v1.59.x)")).toEqual({
- source: "HEAD",
- target: "v1.59.x",
- });
- });
-
- it("handles version-to-HEAD format", () => {
- expect(parseJobName("test-matrix (v1.58.x, HEAD)")).toEqual({
- source: "v1.58.x",
- target: "HEAD",
- });
- });
-
- it("returns null for unparseable names", () => {
- expect(parseJobName("generate-matrix")).toBeNull();
- });
- });
-
- describe("buildFailureResults", () => {
- it("enriches jobs with artifact data when available", () => {
- const failedJobs: FailedJob[] = [
- { name: "test-matrix (HEAD, v1.59.x)", url: "https://job/1" },
- ];
- const artifactData = new Map<string, FailureResult>([
- [
- "HEAD-v1.59.x",
- {
- phase: "migration",
- source: "HEAD",
- target: "v1.59.x",
- },
- ],
- ]);
-
- const results = buildFailureResults(failedJobs, artifactData);
-
- expect(results).toHaveLength(1);
- expect(results[0].phase).toBe("migration");
- expect(results[0].jobUrl).toBe("https://job/1");
- });
-
- it("defaults to e2e phase when no artifact exists", () => {
- const failedJobs: FailedJob[] = [
- { name: "test-matrix (v1.58.x, HEAD)", url: "https://job/2" },
- ];
-
- const results = buildFailureResults(failedJobs, new Map());
-
- expect(results).toHaveLength(1);
- expect(results[0].phase).toBe("e2e");
- expect(results[0].source).toBe("v1.58.x");
- expect(results[0].target).toBe("HEAD");
- });
- });
-
- describe("buildSlackPayload", () => {
- it("lists failures with phase labels", () => {
- const failures: FailureResult[] = [
- {
- phase: "migration",
- source: "HEAD",
- target: "v1.59.x",
- jobUrl: "https://job/1",
- },
- {
- phase: "e2e",
- source: "v1.58.x",
- target: "HEAD",
- jobUrl: "https://job/2",
- },
- ];
-
- const payload = buildSlackPayload(failures, RUN_URL);
- const text = payload.blocks[0].text?.text ?? "";
-
- expect(text).toContain("Cross-version tests failing");
- expect(text).toContain("<https://job/1|HEAD → v1.59.x> (migration failure)");
- expect(text).toContain("<https://job/2|v1.58.x → HEAD> (e2e failure)");
- expect(text).toContain("View full run");
- });
-
- it("shows generic message when no failures exist", () => {
- const payload = buildSlackPayload([], RUN_URL);
- const text = payload.blocks[0].text?.text ?? "";
-
- expect(text).toContain("One or more matrix jobs failed");
- });
-
- it("renders as a single block", () => {
- const failures: FailureResult[] = [
- {
- phase: "migration",
- source: "HEAD",
- target: "v1.59.x",
- },
- ];
-
- const payload = buildSlackPayload(failures, RUN_URL);
- expect(payload.blocks).toHaveLength(1);
- });
- });
-});
diff --git a/release/src/index.ts b/release/src/index.ts
index 645d5a1a40e..016350f2fa4 100644
--- a/release/src/index.ts
+++ b/release/src/index.ts
@@ -7,5 +7,4 @@ export * from "./release-notes";
export * from "./slack";
export * from "./version-helpers";
export * from "./version-info";
-export * from "./cross-version-slack";
export * from "./embedding-sdk-release-helpers";