Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Notify Slack on cross-version test failures (#71491)

Nemanja Glumace9bc4ac447fa
diff --git a/.github/workflows/cross-version.yml b/.github/workflows/cross-version.yml
index e056799aaa1..90558e4826e 100644
--- a/.github/workflows/cross-version.yml
+++ b/.github/workflows/cross-version.yml
@@ -70,10 +70,19 @@ jobs:
- name: Run cross-version test
env:
CURRENT_VERSION: ${{ vars.CURRENT_VERSION }}
+ RESULT_FILE: ${{ runner.temp }}/cross-version-result-${{ matrix.source }}-${{ matrix.target }}.json
run: |
cd cross-version
./test.sh --source "${{ matrix.source }}" --target "${{ matrix.target }}"
+ - name: Upload failure result
+ uses: actions/upload-artifact@v4
+ if: failure()
+ with:
+ name: failure-result-${{ matrix.source }}-${{ matrix.target }}
+ path: ${{ runner.temp }}/cross-version-result-${{ matrix.source }}-${{ matrix.target }}.json
+ if-no-files-found: ignore
+
- name: Upload Cypress Artifacts upon failure
uses: actions/upload-artifact@v4
if: failure()
@@ -91,6 +100,84 @@ jobs:
docker compose logs metabase || true
docker compose logs postgres || true
+ # Notify Slack on nightly failures
+ notify-on-failure:
+ if: always() && github.event_name == 'schedule' && needs.test-matrix.result == 'failure'
+ needs: [test-matrix]
+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+ timeout-minutes: 5
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ sparse-checkout: |
+ release
+ .github
+
+ - name: Prepare release scripts
+ uses: ./.github/actions/prepare-release-scripts
+
+ - name: Download failure results
+ uses: actions/download-artifact@v4
+ continue-on-error: true
+ with:
+ pattern: failure-result-*
+ path: failure-results
+ merge-multiple: true
+
+ - name: Notify Slack
+ continue-on-error: true
+ uses: actions/github-script@v7
+ env:
+ SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+ with:
+ script: |
+ const fs = require('fs');
+ const path = require('path');
+ const { buildFailureResults, sendCrossVersionSlackNotification } = require('${{ github.workspace }}/release/dist/index.cjs');
+
+ // Get all failed jobs from the GitHub API
+ const { owner, repo } = context.repo;
+ const jobInfo = await github.rest.actions.listJobsForWorkflowRun({
+ owner,
+ repo,
+ run_id: context.runId,
+ per_page: 100,
+ });
+
+ const failedJobs = jobInfo.data.jobs
+ .filter(job => job.status === 'completed' && job.conclusion === 'failure' && job.name.startsWith('test-matrix'))
+ .map(job => ({ name: job.name, url: job.html_url }));
+
+ if (failedJobs.length === 0) {
+ console.log('No failed test-matrix jobs found');
+ return;
+ }
+
+ // Load artifact data for enrichment (phase)
+ const resultsDir = 'failure-results';
+ const artifactData = new Map();
+ if (fs.existsSync(resultsDir)) {
+ for (const file of fs.readdirSync(resultsDir)) {
+ if (!file.endsWith('.json')) continue;
+ try {
+ const result = JSON.parse(fs.readFileSync(path.join(resultsDir, file), 'utf8'));
+ artifactData.set(`${result.source}-${result.target}`, result);
+ } catch (e) {
+ console.log(`Skipping malformed result: ${file}`);
+ }
+ }
+ }
+
+ const failures = buildFailureResults(failedJobs, artifactData);
+ const runUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${context.runId}`;
+ const slackBotToken = process.env.SLACK_BOT_TOKEN;
+ if (!slackBotToken) {
+ console.log('SLACK_BOT_TOKEN is not set; skipping Slack notification.');
+ return;
+ }
+ await sendCrossVersionSlackNotification(failures, runUrl, slackBotToken);
+ console.log('Slack notification sent');
+
# Manual test for workflow_dispatch
test-manual:
if: github.event_name == 'workflow_dispatch'
diff --git a/cross-version/test.sh b/cross-version/test.sh
index c4375d0d8d6..41f82967a52 100755
--- a/cross-version/test.sh
+++ b/cross-version/test.sh
@@ -19,6 +19,7 @@ SOURCE_VERSION=""
TARGET_VERSION=""
METABASE_PORT="${METABASE_PORT:-3000}"
HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-120}"
+RESULT_FILE="${RESULT_FILE:-/tmp/cross-version-result.json}"
# Colors for output
RED='\033[0;31m'
@@ -30,6 +31,17 @@ log() { echo -e "${GREEN}[cross-version]${NC} $*"; }
warn() { echo -e "${YELLOW}[cross-version]${NC} $*"; }
error() { echo -e "${RED}[cross-version]${NC} $*" >&2; }
+# Write a structured result file on failure so CI can categorize the failure
+write_failure() {
+ local phase="$1" # "migration" or "e2e"
+ jq -n \
+ --arg phase "$phase" \
+ --arg source "$SOURCE_VERSION" \
+ --arg target "$TARGET_VERSION" \
+ '{phase:$phase, source:$source, target:$target}' > "$RESULT_FILE"
+ log "Wrote failure result to $RESULT_FILE"
+}
+
usage() {
cat <<EOF
Usage: $0 --source VERSION --target VERSION
@@ -230,6 +242,11 @@ migrate_down_step() {
error "migrate down encountered errors (check logs above)"
return 1
fi
+
+ if echo "$output" | grep -q "not rolled back"; then
+ error "migrate down had changesets that were not rolled back (check logs above)"
+ return 1
+ fi
}
# Cascading migrate down: rolls back multiple major versions one at a time
@@ -355,6 +372,7 @@ main() {
if ! wait_for_health "$HEALTH_TIMEOUT"; then
error "❌ SOURCE version ($SOURCE_VERSION) failed health check"
docker compose logs metabase
+ write_failure "migration"
exit 1
fi
@@ -362,7 +380,10 @@ main() {
log ""
log "Step 2: Running e2e tests (@source)..."
- run_e2e source "$SOURCE_VERSION"
+ if ! run_e2e source "$SOURCE_VERSION"; then
+ write_failure "e2e"
+ exit 1
+ fi
log ""
log "Step 3: Stopping SOURCE version ($SOURCE_VERSION)..."
@@ -377,19 +398,24 @@ main() {
if ! wait_for_health "$HEALTH_TIMEOUT"; then
error "❌ TARGET version ($TARGET_VERSION) failed health check after upgrade"
docker compose logs metabase
+ write_failure "migration"
exit 1
fi
log "✅ UPGRADE successful - TARGET version ($TARGET_VERSION) is healthy"
log ""
log "Step 5: Running e2e tests (@target)..."
- run_e2e target "$TARGET_VERSION"
+ if ! run_e2e target "$TARGET_VERSION"; then
+ write_failure "e2e"
+ exit 1
+ fi
else
# Downgrade: should refuse to start, then we run migrate down
if ! check_downgrade_refused; then
error "❌ TARGET version ($TARGET_VERSION) did not properly detect downgrade"
docker compose logs metabase
+ write_failure "migration"
exit 1
fi
@@ -400,6 +426,7 @@ main() {
log "Step 5: Rolling back database ($SOURCE_VERSION → $TARGET_VERSION)..."
if ! cascading_migrate_down "$SOURCE_VERSION" "$TARGET_VERSION"; then
error "❌ migrate down failed"
+ write_failure "migration"
exit 1
fi
@@ -411,13 +438,17 @@ main() {
if ! wait_for_health "$HEALTH_TIMEOUT"; then
error "❌ TARGET version ($TARGET_VERSION) failed health check after migrate down"
docker compose logs metabase
+ write_failure "migration"
exit 1
fi
log "✅ DOWNGRADE successful - TARGET version ($TARGET_VERSION) is healthy after migrate down"
log ""
log "Step 7: Running e2e tests (@target)..."
- run_e2e target "$TARGET_VERSION"
+ if ! run_e2e target "$TARGET_VERSION"; then
+ write_failure "e2e"
+ exit 1
+ fi
fi
log ""
diff --git a/release/src/cross-version-slack.ts b/release/src/cross-version-slack.ts
new file mode 100644
index 00000000000..1c2d2eebd9d
--- /dev/null
+++ b/release/src/cross-version-slack.ts
@@ -0,0 +1,127 @@
+/**
+ * Slack notification helpers for cross-version migration test failures.
+ *
+ * buildSlackPayload is a pure function (for testability).
+ * sendCrossVersionSlackNotification handles the posting via @slack/web-api.
+ */
+import { WebClient } from "@slack/web-api";
+
+export interface FailureResult {
+ phase: "migration" | "e2e";
+ source: string;
+ target: string;
+ jobUrl?: string;
+}
+
+interface SlackBlock {
+ type: string;
+ text?: { type: string; text: string; emoji?: boolean };
+ elements?: Array<{ type: string; text: string }>;
+}
+
+export interface SlackPayload {
+ blocks: SlackBlock[];
+}
+
+export interface FailedJob {
+ name: string;
+ url: string;
+}
+
+/**
+ * Parse source and target versions from a matrix job name.
+ * e.g. "test-matrix (HEAD, v1.59.x)" -> { source: "HEAD", target: "v1.59.x" }
+ */
+export const parseJobName = (
+ name: string,
+): { source: string; target: string } | null => {
+ const match = name.match(/\(([^,]+),\s*([^)]+)\)/);
+ if (!match) {
+ return null;
+ }
+ return { source: match[1], target: match[2] };
+};
+
+/**
+ * Build a list of FailureResults from failed jobs and artifact data.
+ * Each failed job becomes a result. Artifact data enriches with phase.
+ */
+export const buildFailureResults = (
+ failedJobs: FailedJob[],
+ artifactData: Map<string, FailureResult>,
+): FailureResult[] => {
+ return failedJobs.map(job => {
+ const parsed = parseJobName(job.name);
+ if (!parsed) {
+ return {
+ phase: "e2e" as const,
+ source: "?",
+ target: "?",
+ jobUrl: job.url,
+ };
+ }
+
+ const key = `${parsed.source}-${parsed.target}`;
+ const artifact = artifactData.get(key);
+
+ if (artifact) {
+ return { phase: artifact.phase, source: artifact.source, target: artifact.target, jobUrl: job.url };
+ }
+
+ return {
+ phase: "e2e" as const,
+ source: parsed.source,
+ target: parsed.target,
+ jobUrl: job.url,
+ };
+ });
+};
+
+const formatFailureItem = (f: FailureResult): string => {
+ const versions = f.jobUrl
+ ? `<${f.jobUrl}|${f.source} → ${f.target}>`
+ : `${f.source} → ${f.target}`;
+ const label = f.phase === "migration" ? "migration failure" : "e2e failure";
+ return `• ${versions} (${label})`;
+};
+
+export const buildSlackPayload = (
+ failures: FailureResult[],
+ runUrl: string,
+): SlackPayload => {
+ const lines = failures.map(formatFailureItem);
+
+ if (lines.length === 0) {
+ lines.push(
+ "One or more matrix jobs failed. Check the workflow run for details.",
+ );
+ }
+
+ return {
+ blocks: [
+ {
+ type: "section",
+ text: {
+ type: "mrkdwn",
+ text: `:warning: *Cross-version tests failing*\n${lines.join("\n")}\n<${runUrl}|View full run>`,
+ },
+ },
+ ],
+ };
+};
+
+export const sendCrossVersionSlackNotification = async (
+ failures: FailureResult[],
+ runUrl: string,
+ slackBotToken: string,
+ channel = "engineering-ci",
+): Promise<void> => {
+ const slack = new WebClient(slackBotToken);
+ const { blocks } = buildSlackPayload(failures, runUrl);
+
+ await slack.chat.postMessage({
+ channel,
+ text: "Cross-version tests failing",
+ blocks,
+ });
+};
diff --git a/release/src/cross-version-slack.unit.spec.ts b/release/src/cross-version-slack.unit.spec.ts
new file mode 100644
index 00000000000..5f2df7151a0
--- /dev/null
+++ b/release/src/cross-version-slack.unit.spec.ts
@@ -0,0 +1,115 @@
+import {
+ buildFailureResults,
+ buildSlackPayload,
+ parseJobName,
+ type FailedJob,
+ type FailureResult,
+} from "./cross-version-slack";
+
+const RUN_URL = "https://github.com/metabase/metabase/actions/runs/12345";
+
+describe("cross-version-slack", () => {
+ describe("parseJobName", () => {
+ it("parses source and target from matrix job name", () => {
+ expect(parseJobName("test-matrix (HEAD, v1.59.x)")).toEqual({
+ source: "HEAD",
+ target: "v1.59.x",
+ });
+ });
+
+ it("handles version-to-HEAD format", () => {
+ expect(parseJobName("test-matrix (v1.58.x, HEAD)")).toEqual({
+ source: "v1.58.x",
+ target: "HEAD",
+ });
+ });
+
+ it("returns null for unparseable names", () => {
+ expect(parseJobName("generate-matrix")).toBeNull();
+ });
+ });
+
+ describe("buildFailureResults", () => {
+ it("enriches jobs with artifact data when available", () => {
+ const failedJobs: FailedJob[] = [
+ { name: "test-matrix (HEAD, v1.59.x)", url: "https://job/1" },
+ ];
+ const artifactData = new Map<string, FailureResult>([
+ [
+ "HEAD-v1.59.x",
+ {
+ phase: "migration",
+ source: "HEAD",
+ target: "v1.59.x",
+ },
+ ],
+ ]);
+
+ const results = buildFailureResults(failedJobs, artifactData);
+
+ expect(results).toHaveLength(1);
+ expect(results[0].phase).toBe("migration");
+ expect(results[0].jobUrl).toBe("https://job/1");
+ });
+
+ it("defaults to e2e phase when no artifact exists", () => {
+ const failedJobs: FailedJob[] = [
+ { name: "test-matrix (v1.58.x, HEAD)", url: "https://job/2" },
+ ];
+
+ const results = buildFailureResults(failedJobs, new Map());
+
+ expect(results).toHaveLength(1);
+ expect(results[0].phase).toBe("e2e");
+ expect(results[0].source).toBe("v1.58.x");
+ expect(results[0].target).toBe("HEAD");
+ });
+ });
+
+ describe("buildSlackPayload", () => {
+ it("lists failures with phase labels", () => {
+ const failures: FailureResult[] = [
+ {
+ phase: "migration",
+ source: "HEAD",
+ target: "v1.59.x",
+ jobUrl: "https://job/1",
+ },
+ {
+ phase: "e2e",
+ source: "v1.58.x",
+ target: "HEAD",
+ jobUrl: "https://job/2",
+ },
+ ];
+
+ const payload = buildSlackPayload(failures, RUN_URL);
+ const text = payload.blocks[0].text?.text ?? "";
+
+ expect(text).toContain("Cross-version tests failing");
+ expect(text).toContain("<https://job/1|HEAD → v1.59.x> (migration failure)");
+ expect(text).toContain("<https://job/2|v1.58.x → HEAD> (e2e failure)");
+ expect(text).toContain("View full run");
+ });
+
+ it("shows generic message when no failures exist", () => {
+ const payload = buildSlackPayload([], RUN_URL);
+ const text = payload.blocks[0].text?.text ?? "";
+
+ expect(text).toContain("One or more matrix jobs failed");
+ });
+
+ it("renders as a single block", () => {
+ const failures: FailureResult[] = [
+ {
+ phase: "migration",
+ source: "HEAD",
+ target: "v1.59.x",
+ },
+ ];
+
+ const payload = buildSlackPayload(failures, RUN_URL);
+ expect(payload.blocks).toHaveLength(1);
+ });
+ });
+});
diff --git a/release/src/index.ts b/release/src/index.ts
index 7e38d880a90..2365fa4f1d6 100644
--- a/release/src/index.ts
+++ b/release/src/index.ts
@@ -7,3 +7,4 @@ export * from "./release-status";
export * from "./slack";
export * from "./version-helpers";
export * from "./version-info";
+export * from "./cross-version-slack";