The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Notify Slack on cross-version test failures (#71491)
Nemanja Glumace9bc4ac447fa
diff --git a/.github/workflows/cross-version.yml b/.github/workflows/cross-version.ymlindex e056799aaa1..90558e4826e 100644--- a/.github/workflows/cross-version.yml+++ b/.github/workflows/cross-version.yml@@ -70,10 +70,19 @@ jobs:- name: Run cross-version testenv:CURRENT_VERSION: ${{ vars.CURRENT_VERSION }}+ RESULT_FILE: ${{ runner.temp }}/cross-version-result-${{ matrix.source }}-${{ matrix.target }}.jsonrun: |cd cross-version./test.sh --source "${{ matrix.source }}" --target "${{ matrix.target }}"+ - name: Upload failure result+ uses: actions/upload-artifact@v4+ if: failure()+ with:+ name: failure-result-${{ matrix.source }}-${{ matrix.target }}+ path: ${{ runner.temp }}/cross-version-result-${{ matrix.source }}-${{ matrix.target }}.json+ if-no-files-found: ignore+- name: Upload Cypress Artifacts upon failureuses: actions/upload-artifact@v4if: failure()@@ -91,6 +100,84 @@ jobs:docker compose logs metabase || truedocker compose logs postgres || true+ # Notify Slack on nightly failures+ notify-on-failure:+ if: always() && github.event_name == 'schedule' && needs.test-matrix.result == 'failure'+ needs: [test-matrix]+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}+ timeout-minutes: 5+ steps:+ - uses: actions/checkout@v4+ with:+ sparse-checkout: |+ release+ .github++ - name: Prepare release scripts+ uses: ./.github/actions/prepare-release-scripts++ - name: Download failure results+ uses: actions/download-artifact@v4+ continue-on-error: true+ with:+ pattern: failure-result-*+ path: failure-results+ merge-multiple: true++ - name: Notify Slack+ continue-on-error: true+ uses: actions/github-script@v7+ env:+ SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}+ with:+ script: |+ const fs = require('fs');+ const path = require('path');+ const { buildFailureResults, sendCrossVersionSlackNotification } = require('${{ github.workspace }}/release/dist/index.cjs');++ // Get all failed jobs from the GitHub API+ const { owner, repo } = context.repo;+ const jobInfo = await github.rest.actions.listJobsForWorkflowRun({+ owner,+ repo,+ run_id: context.runId,+ per_page: 100,+ });++ const failedJobs = jobInfo.data.jobs+ .filter(job => job.status === 'completed' && job.conclusion === 'failure' && job.name.startsWith('test-matrix'))+ .map(job => ({ name: job.name, url: job.html_url }));++ if (failedJobs.length === 0) {+ console.log('No failed test-matrix jobs found');+ return;+ }++ // Load artifact data for enrichment (phase)+ const resultsDir = 'failure-results';+ const artifactData = new Map();+ if (fs.existsSync(resultsDir)) {+ for (const file of fs.readdirSync(resultsDir)) {+ if (!file.endsWith('.json')) continue;+ try {+ const result = JSON.parse(fs.readFileSync(path.join(resultsDir, file), 'utf8'));+ artifactData.set(`${result.source}-${result.target}`, result);+ } catch (e) {+ console.log(`Skipping malformed result: ${file}`);+ }+ }+ }++ const failures = buildFailureResults(failedJobs, artifactData);+ const runUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${context.runId}`;+ const slackBotToken = process.env.SLACK_BOT_TOKEN;+ if (!slackBotToken) {+ console.log('SLACK_BOT_TOKEN is not set; skipping Slack notification.');+ return;+ }+ await sendCrossVersionSlackNotification(failures, runUrl, slackBotToken);+ console.log('Slack notification sent');+# Manual test for workflow_dispatchtest-manual:if: github.event_name == 'workflow_dispatch'diff --git a/cross-version/test.sh b/cross-version/test.shindex c4375d0d8d6..41f82967a52 100755--- a/cross-version/test.sh+++ b/cross-version/test.sh@@ -19,6 +19,7 @@ SOURCE_VERSION=""TARGET_VERSION=""METABASE_PORT="${METABASE_PORT:-3000}"HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-120}"+RESULT_FILE="${RESULT_FILE:-/tmp/cross-version-result.json}"# Colors for outputRED='\033[0;31m'@@ -30,6 +31,17 @@ log() { echo -e "${GREEN}[cross-version]${NC} $*"; }warn() { echo -e "${YELLOW}[cross-version]${NC} $*"; }error() { echo -e "${RED}[cross-version]${NC} $*" >&2; }+# Write a structured result file on failure so CI can categorize the failure+write_failure() {+ local phase="$1" # "migration" or "e2e"+ jq -n \+ --arg phase "$phase" \+ --arg source "$SOURCE_VERSION" \+ --arg target "$TARGET_VERSION" \+ '{phase:$phase, source:$source, target:$target}' > "$RESULT_FILE"+ log "Wrote failure result to $RESULT_FILE"+}+usage() {cat <<EOFUsage: $0 --source VERSION --target VERSION@@ -230,6 +242,11 @@ migrate_down_step() {error "migrate down encountered errors (check logs above)"return 1fi++ if echo "$output" | grep -q "not rolled back"; then+ error "migrate down had changesets that were not rolled back (check logs above)"+ return 1+ fi}# Cascading migrate down: rolls back multiple major versions one at a time@@ -355,6 +372,7 @@ main() {if ! wait_for_health "$HEALTH_TIMEOUT"; thenerror "❌ SOURCE version ($SOURCE_VERSION) failed health check"docker compose logs metabase+ write_failure "migration"exit 1fi@@ -362,7 +380,10 @@ main() {log ""log "Step 2: Running e2e tests (@source)..."- run_e2e source "$SOURCE_VERSION"+ if ! run_e2e source "$SOURCE_VERSION"; then+ write_failure "e2e"+ exit 1+ filog ""log "Step 3: Stopping SOURCE version ($SOURCE_VERSION)..."@@ -377,19 +398,24 @@ main() {if ! wait_for_health "$HEALTH_TIMEOUT"; thenerror "❌ TARGET version ($TARGET_VERSION) failed health check after upgrade"docker compose logs metabase+ write_failure "migration"exit 1filog "✅ UPGRADE successful - TARGET version ($TARGET_VERSION) is healthy"log ""log "Step 5: Running e2e tests (@target)..."- run_e2e target "$TARGET_VERSION"+ if ! run_e2e target "$TARGET_VERSION"; then+ write_failure "e2e"+ exit 1+ fielse# Downgrade: should refuse to start, then we run migrate downif ! check_downgrade_refused; thenerror "❌ TARGET version ($TARGET_VERSION) did not properly detect downgrade"docker compose logs metabase+ write_failure "migration"exit 1fi@@ -400,6 +426,7 @@ main() {log "Step 5: Rolling back database ($SOURCE_VERSION → $TARGET_VERSION)..."if ! cascading_migrate_down "$SOURCE_VERSION" "$TARGET_VERSION"; thenerror "❌ migrate down failed"+ write_failure "migration"exit 1fi@@ -411,13 +438,17 @@ main() {if ! wait_for_health "$HEALTH_TIMEOUT"; thenerror "❌ TARGET version ($TARGET_VERSION) failed health check after migrate down"docker compose logs metabase+ write_failure "migration"exit 1filog "✅ DOWNGRADE successful - TARGET version ($TARGET_VERSION) is healthy after migrate down"log ""log "Step 7: Running e2e tests (@target)..."- run_e2e target "$TARGET_VERSION"+ if ! run_e2e target "$TARGET_VERSION"; then+ write_failure "e2e"+ exit 1+ fifilog ""diff --git a/release/src/cross-version-slack.ts b/release/src/cross-version-slack.tsnew file mode 100644index 00000000000..1c2d2eebd9d--- /dev/null+++ b/release/src/cross-version-slack.ts@@ -0,0 +1,127 @@+/**+ * Slack notification helpers for cross-version migration test failures.+ *+ * buildSlackPayload is a pure function (for testability).+ * sendCrossVersionSlackNotification handles the posting via @slack/web-api.+ */+import { WebClient } from "@slack/web-api";++export interface FailureResult {+ phase: "migration" | "e2e";+ source: string;+ target: string;+ jobUrl?: string;+}++interface SlackBlock {+ type: string;+ text?: { type: string; text: string; emoji?: boolean };+ elements?: Array<{ type: string; text: string }>;+}++export interface SlackPayload {+ blocks: SlackBlock[];+}++export interface FailedJob {+ name: string;+ url: string;+}++/**+ * Parse source and target versions from a matrix job name.+ * e.g. "test-matrix (HEAD, v1.59.x)" -> { source: "HEAD", target: "v1.59.x" }+ */+export const parseJobName = (+ name: string,+): { source: string; target: string } | null => {+ const match = name.match(/\(([^,]+),\s*([^)]+)\)/);+ if (!match) {+ return null;+ }+ return { source: match[1], target: match[2] };+};++/**+ * Build a list of FailureResults from failed jobs and artifact data.+ * Each failed job becomes a result. Artifact data enriches with phase.+ */+export const buildFailureResults = (+ failedJobs: FailedJob[],+ artifactData: Map<string, FailureResult>,+): FailureResult[] => {+ return failedJobs.map(job => {+ const parsed = parseJobName(job.name);+ if (!parsed) {+ return {+ phase: "e2e" as const,+ source: "?",+ target: "?",+ jobUrl: job.url,+ };+ }++ const key = `${parsed.source}-${parsed.target}`;+ const artifact = artifactData.get(key);++ if (artifact) {+ return { phase: artifact.phase, source: artifact.source, target: artifact.target, jobUrl: job.url };+ }++ return {+ phase: "e2e" as const,+ source: parsed.source,+ target: parsed.target,+ jobUrl: job.url,+ };+ });+};++const formatFailureItem = (f: FailureResult): string => {+ const versions = f.jobUrl+ ? `<${f.jobUrl}|${f.source} → ${f.target}>`+ : `${f.source} → ${f.target}`;+ const label = f.phase === "migration" ? "migration failure" : "e2e failure";+ return `• ${versions} (${label})`;+};++export const buildSlackPayload = (+ failures: FailureResult[],+ runUrl: string,+): SlackPayload => {+ const lines = failures.map(formatFailureItem);++ if (lines.length === 0) {+ lines.push(+ "One or more matrix jobs failed. Check the workflow run for details.",+ );+ }++ return {+ blocks: [+ {+ type: "section",+ text: {+ type: "mrkdwn",+ text: `:warning: *Cross-version tests failing*\n${lines.join("\n")}\n<${runUrl}|View full run>`,+ },+ },+ ],+ };+};++export const sendCrossVersionSlackNotification = async (+ failures: FailureResult[],+ runUrl: string,+ slackBotToken: string,+ channel = "engineering-ci",+): Promise<void> => {+ const slack = new WebClient(slackBotToken);+ const { blocks } = buildSlackPayload(failures, runUrl);++ await slack.chat.postMessage({+ channel,+ text: "Cross-version tests failing",+ blocks,+ });+};diff --git a/release/src/cross-version-slack.unit.spec.ts b/release/src/cross-version-slack.unit.spec.tsnew file mode 100644index 00000000000..5f2df7151a0--- /dev/null+++ b/release/src/cross-version-slack.unit.spec.ts@@ -0,0 +1,115 @@+import {+ buildFailureResults,+ buildSlackPayload,+ parseJobName,+ type FailedJob,+ type FailureResult,+} from "./cross-version-slack";++const RUN_URL = "https://github.com/metabase/metabase/actions/runs/12345";++describe("cross-version-slack", () => {+ describe("parseJobName", () => {+ it("parses source and target from matrix job name", () => {+ expect(parseJobName("test-matrix (HEAD, v1.59.x)")).toEqual({+ source: "HEAD",+ target: "v1.59.x",+ });+ });++ it("handles version-to-HEAD format", () => {+ expect(parseJobName("test-matrix (v1.58.x, HEAD)")).toEqual({+ source: "v1.58.x",+ target: "HEAD",+ });+ });++ it("returns null for unparseable names", () => {+ expect(parseJobName("generate-matrix")).toBeNull();+ });+ });++ describe("buildFailureResults", () => {+ it("enriches jobs with artifact data when available", () => {+ const failedJobs: FailedJob[] = [+ { name: "test-matrix (HEAD, v1.59.x)", url: "https://job/1" },+ ];+ const artifactData = new Map<string, FailureResult>([+ [+ "HEAD-v1.59.x",+ {+ phase: "migration",+ source: "HEAD",+ target: "v1.59.x",+ },+ ],+ ]);++ const results = buildFailureResults(failedJobs, artifactData);++ expect(results).toHaveLength(1);+ expect(results[0].phase).toBe("migration");+ expect(results[0].jobUrl).toBe("https://job/1");+ });++ it("defaults to e2e phase when no artifact exists", () => {+ const failedJobs: FailedJob[] = [+ { name: "test-matrix (v1.58.x, HEAD)", url: "https://job/2" },+ ];++ const results = buildFailureResults(failedJobs, new Map());++ expect(results).toHaveLength(1);+ expect(results[0].phase).toBe("e2e");+ expect(results[0].source).toBe("v1.58.x");+ expect(results[0].target).toBe("HEAD");+ });+ });++ describe("buildSlackPayload", () => {+ it("lists failures with phase labels", () => {+ const failures: FailureResult[] = [+ {+ phase: "migration",+ source: "HEAD",+ target: "v1.59.x",+ jobUrl: "https://job/1",+ },+ {+ phase: "e2e",+ source: "v1.58.x",+ target: "HEAD",+ jobUrl: "https://job/2",+ },+ ];++ const payload = buildSlackPayload(failures, RUN_URL);+ const text = payload.blocks[0].text?.text ?? "";++ expect(text).toContain("Cross-version tests failing");+ expect(text).toContain("<https://job/1|HEAD → v1.59.x> (migration failure)");+ expect(text).toContain("<https://job/2|v1.58.x → HEAD> (e2e failure)");+ expect(text).toContain("View full run");+ });++ it("shows generic message when no failures exist", () => {+ const payload = buildSlackPayload([], RUN_URL);+ const text = payload.blocks[0].text?.text ?? "";++ expect(text).toContain("One or more matrix jobs failed");+ });++ it("renders as a single block", () => {+ const failures: FailureResult[] = [+ {+ phase: "migration",+ source: "HEAD",+ target: "v1.59.x",+ },+ ];++ const payload = buildSlackPayload(failures, RUN_URL);+ expect(payload.blocks).toHaveLength(1);+ });+ });+});diff --git a/release/src/index.ts b/release/src/index.tsindex 7e38d880a90..2365fa4f1d6 100644--- a/release/src/index.ts+++ b/release/src/index.ts@@ -7,3 +7,4 @@ export * from "./release-status";export * from "./slack";export * from "./version-helpers";export * from "./version-info";+export * from "./cross-version-slack";