Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Skip cross-version downgrades blocked by unreleased migrations (#84578)

Chris Vandoodewaard10cfe77dd344
diff --git a/.github/file-paths.yaml b/.github/file-paths.yaml
index 9c6998f8df8..b192c36cc49 100644
--- a/.github/file-paths.yaml
+++ b/.github/file-paths.yaml
@@ -7,7 +7,7 @@ default: &default
- "locales/**"
- "bin/**/!(*.md)"
-workflows:
+workflows: &workflows
- added|modified: ".github/workflows/*.{yml,yaml}"
shared_sources: &shared_sources
@@ -322,6 +322,7 @@ build_uberjar:
- *e2e_all
- *embedding_sdk_components
- *embedding_sdk_host_sample_apps
+ - *workflows
yaml:
- "**/*.yml"
diff --git a/.github/scripts/notify-unreleased-migrations.js b/.github/scripts/notify-unreleased-migrations.js
new file mode 100644
index 00000000000..ad7faf85b9a
--- /dev/null
+++ b/.github/scripts/notify-unreleased-migrations.js
@@ -0,0 +1,91 @@
+function findOverdueCommits({ migrations, notifyAfterHours }) {
+ const commits = new Map();
+ for (const { id, commit, age_hours } of migrations) {
+ if (age_hours < notifyAfterHours) {
+ continue;
+ }
+ if (!commits.has(commit)) {
+ commits.set(commit, { commit, age_hours, ids: [] });
+ }
+ commits.get(commit).ids.push(id);
+ }
+ return [...commits.values()];
+}
+
+function commitUrl({ owner, repo, commit }) {
+ return `https://github.com/${owner}/${repo}/commit/${commit}`;
+}
+
+function hoursUntilFailure({ commits, gracePeriodDays }) {
+ const oldest = Math.max(...commits.map(({ age_hours }) => age_hours));
+ return Math.max(0, gracePeriodDays * 24 - oldest);
+}
+
+function formatMessage({ commits, gracePeriodDays, owner, repo, runUrl }) {
+ return [
+ `:warning: *Cross-version tests (<${runUrl}|run>) skipped since unreleased migrations for existing release branches have been added to master*`,
+ ...commits.map(
+ ({ commit, age_hours, ids }) =>
+ `• <${commitUrl({ owner, repo, commit })}|${commit.slice(0, 11)}> added: ${ids.map((id) => `\`${id}\``).join(", ")} (${age_hours}h ago)`,
+ ),
+ `Please backport and release these migrations. Cross-version tests will start failing in ${hoursUntilFailure({ commits, gracePeriodDays })}h.`,
+ ].join("\n");
+}
+
+async function notifyOverdueChangesets({
+ context,
+ migrations,
+ notifyAfterHours,
+ repeatAfterHours,
+ gracePeriodDays,
+ channelName,
+ getRecentSlackMessages,
+ sendSlackMessage,
+}) {
+ const { owner, repo } = context.repo;
+ const runUrl = `https://github.com/${owner}/${repo}/actions/runs/${context.runId}`;
+ const overdue = findOverdueCommits({ migrations, notifyAfterHours });
+ if (overdue.length === 0) {
+ return;
+ }
+
+ const recentMessages = await getRecentSlackMessages({
+ channelName,
+ sinceHours: repeatAfterHours,
+ });
+ const allNotified = overdue.every(({ commit }) =>
+ recentMessages.some((text) =>
+ text.includes(commitUrl({ owner, repo, commit })),
+ ),
+ );
+
+ if (!allNotified) {
+ await sendSlackMessage({
+ channelName,
+ message: formatMessage({
+ commits: overdue,
+ gracePeriodDays,
+ owner,
+ repo,
+ runUrl,
+ }),
+ });
+ } else {
+ console.log(
+ [
+ "All commits with migrations that need to be backported/released already have a recent Slack notification:",
+ ...overdue.map(
+ ({ commit, age_hours, ids }) =>
+ ` ${commitUrl({ owner, repo, commit })} added: ${ids.join(", ")} (${age_hours}h ago)`,
+ ),
+ ].join("\n"),
+ );
+ }
+}
+
+module.exports = {
+ findOverdueCommits,
+ formatMessage,
+ hoursUntilFailure,
+ notifyOverdueChangesets,
+};
diff --git a/.github/scripts/notify-unreleased-migrations.unit.spec.js b/.github/scripts/notify-unreleased-migrations.unit.spec.js
new file mode 100644
index 00000000000..3248cb4c2ff
--- /dev/null
+++ b/.github/scripts/notify-unreleased-migrations.unit.spec.js
@@ -0,0 +1,133 @@
+const {
+ findOverdueCommits,
+ formatMessage,
+ hoursUntilFailure,
+ notifyOverdueChangesets,
+} = require("./notify-unreleased-migrations");
+
+const migrations = [
+ { id: "v63.a", commit: "aaaaaaaaaaaaaaaa", age_hours: 50 },
+ { id: "v63.b", commit: "aaaaaaaaaaaaaaaa", age_hours: 50 },
+ { id: "v63.c", commit: "bbbbbbbbbbbbbbbb", age_hours: 10 },
+ { id: "v64.a", commit: "cccccccccccccccc", age_hours: 48 },
+];
+
+const commitUrl = (commit) =>
+ `https://github.com/metabase/metabase/commit/${commit}`;
+
+describe("findOverdueCommits", () => {
+ it("groups the overdue migrations by commit", () => {
+ expect(findOverdueCommits({ migrations, notifyAfterHours: 48 })).toEqual([
+ { commit: "aaaaaaaaaaaaaaaa", age_hours: 50, ids: ["v63.a", "v63.b"] },
+ { commit: "cccccccccccccccc", age_hours: 48, ids: ["v64.a"] },
+ ]);
+ });
+});
+
+describe("hoursUntilFailure", () => {
+ it("counts from the oldest commit", () => {
+ expect(
+ hoursUntilFailure({
+ commits: [{ age_hours: 30 }, { age_hours: 50 }],
+ gracePeriodDays: 3,
+ }),
+ ).toBe(22);
+ });
+
+ it("is not negative", () => {
+ expect(
+ hoursUntilFailure({ commits: [{ age_hours: 80 }], gracePeriodDays: 3 }),
+ ).toBe(0);
+ });
+});
+
+describe("formatMessage", () => {
+ it("lists each commit with a link, its migrations and its age", () => {
+ expect(
+ formatMessage({
+ commits: [
+ {
+ commit: "aaaaaaaaaaaaaaaa",
+ age_hours: 50,
+ ids: ["v63.a", "v63.b"],
+ },
+ ],
+ gracePeriodDays: 3,
+ owner: "metabase",
+ repo: "metabase",
+ runUrl: "https://run",
+ }),
+ ).toBe(
+ [
+ ":warning: *Cross-version tests (<https://run|run>) skipped since unreleased migrations for existing release branches have been added to master*",
+ `• <${commitUrl("aaaaaaaaaaaaaaaa")}|aaaaaaaaaaa> added: \`v63.a\`, \`v63.b\` (50h ago)`,
+ "Please backport and release these migrations. Cross-version tests will start failing in 22h.",
+ ].join("\n"),
+ );
+ });
+});
+
+describe("notifyOverdueChangesets", () => {
+ const context = { repo: { owner: "metabase", repo: "metabase" }, runId: 1 };
+ const notify = ({ recentMessages, sendSlackMessage }) => {
+ const getRecentSlackMessages = jest.fn(async () => recentMessages);
+ return notifyOverdueChangesets({
+ context,
+ migrations,
+ notifyAfterHours: 48,
+ repeatAfterHours: 24,
+ gracePeriodDays: 3,
+ channelName: "release",
+ getRecentSlackMessages,
+ sendSlackMessage,
+ }).then(() => getRecentSlackMessages);
+ };
+
+ it("sends one message with all overdue commits when a commit has no message yet", async () => {
+ const sendSlackMessage = jest.fn(async () => ({}));
+ const getRecentSlackMessages = await notify({
+ recentMessages: [
+ `• <${commitUrl("aaaaaaaaaaaaaaaa")}|aaaaaaaaaaa> added`,
+ ],
+ sendSlackMessage,
+ });
+ expect(getRecentSlackMessages).toHaveBeenCalledWith({
+ channelName: "release",
+ sinceHours: 24,
+ });
+ expect(sendSlackMessage).toHaveBeenCalledTimes(1);
+ const { message } = sendSlackMessage.mock.calls[0][0];
+ expect(message).toContain("v64.a");
+ expect(message).toContain("v63.a");
+ expect(message).toContain("v63.b");
+ expect(message).not.toContain("v63.c");
+ });
+
+ it("does not send a message when all commits have one", async () => {
+ const sendSlackMessage = jest.fn();
+ await notify({
+ recentMessages: [
+ commitUrl("aaaaaaaaaaaaaaaa"),
+ commitUrl("cccccccccccccccc"),
+ ],
+ sendSlackMessage,
+ });
+ expect(sendSlackMessage).not.toHaveBeenCalled();
+ });
+
+ it("does not read Slack when no commit is overdue", async () => {
+ const sendSlackMessage = jest.fn();
+ const getRecentSlackMessages = jest.fn();
+ await notifyOverdueChangesets({
+ context,
+ migrations,
+ notifyAfterHours: 100,
+ gracePeriodDays: 3,
+ channelName: "release",
+ getRecentSlackMessages,
+ sendSlackMessage,
+ });
+ expect(getRecentSlackMessages).not.toHaveBeenCalled();
+ expect(sendSlackMessage).not.toHaveBeenCalled();
+ });
+});
diff --git a/.github/workflows/cross-version.yml b/.github/workflows/cross-version.yml
index 1a9a804915d..fa4de47eb4d 100644
--- a/.github/workflows/cross-version.yml
+++ b/.github/workflows/cross-version.yml
@@ -19,6 +19,11 @@ on:
description: "Target version (e.g., v1.58.7 or HEAD)"
required: true
type: string
+ grace_period:
+ description: "Time (in days) before a migration targeting a release version is expected to exist in that release branch."
+ required: false
+ type: string
+ default: 3
jobs:
generate-test-matrix:
@@ -27,6 +32,7 @@ jobs:
timeout-minutes: 5
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
+ oldest-target: ${{ steps.set-matrix.outputs.oldest-target }}
steps:
- name: Require HEAD digest
if: inputs.digest == ''
@@ -61,6 +67,7 @@ jobs:
const { config } = buildCrossVersionMatrix(supportedMajors);
const matrix = JSON.stringify({ include: config });
core.setOutput("matrix", matrix);
+ core.setOutput("oldest-target", `v1.${Math.min(...supportedMajors)}.x`);
console.log("matrix:", matrix);
test:
@@ -70,6 +77,11 @@ jobs:
strategy:
matrix: ${{ fromJson(needs.generate-test-matrix.outputs.matrix) }}
fail-fast: false
+ env:
+ # Number of days we don't fail a cross version test that is missing a migration that was recently merged to master
+ # but targets a different version. For example: a migration for v63 is added to master but master is on v66. Cross
+ # version tests will fail until that is backported and released on v63 but we should give people time to do that.
+ GRACE_PERIOD: 3
steps:
- uses: actions/checkout@v7
@@ -100,6 +112,7 @@ jobs:
run: bun run build-pure:cljs
- name: Run cross-version test
+ id: run-test
env:
CURRENT_VERSION: ${{ vars.CURRENT_VERSION }}
HEAD_DIGEST: ${{ inputs.digest }}
@@ -108,6 +121,36 @@ jobs:
cd cross-version
./test.sh --source "${{ matrix.source }}" --target "${{ matrix.target }}"
+ - name: Notify Slack about migrations that need a backport or a release
+ continue-on-error: true
+ if: >-
+ github.ref_name == 'master' &&
+ steps.run-test.outputs.skipped == 'true' &&
+ steps.run-test.outputs.unreleased_migrations != '' &&
+ matrix.target == needs.generate-test-matrix.outputs.oldest-target
+ uses: actions/github-script@v9
+ env:
+ UNRELEASED_MIGRATIONS: ${{ steps.run-test.outputs.unreleased_migrations }}
+ NOTIFY_AFTER_HOURS: 24
+ REPEAT_AFTER_HOURS: 24
+ SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+ SLACK_CHANNEL: 'engineering-ci'
+ with:
+ script: | # js
+ const { getRecentSlackMessages, sendSlackMessage } = require('${{ github.workspace }}/release/dist/index.cjs');
+ const { notifyOverdueChangesets } = require('${{ github.workspace }}/.github/scripts/notify-unreleased-migrations');
+
+ await notifyOverdueChangesets({
+ context,
+ migrations: JSON.parse(process.env.UNRELEASED_MIGRATIONS),
+ notifyAfterHours: Number(process.env.NOTIFY_AFTER_HOURS),
+ repeatAfterHours: Number(process.env.REPEAT_AFTER_HOURS),
+ gracePeriodDays: Number(process.env.GRACE_PERIOD),
+ channelName: process.env.SLACK_CHANNEL,
+ getRecentSlackMessages,
+ sendSlackMessage,
+ });
+
- name: Upload Cypress Artifacts
uses: actions/upload-artifact@v7
if: always()
@@ -133,6 +176,8 @@ jobs:
if: inputs.source_version != ''
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
timeout-minutes: 30
+ env:
+ GRACE_PERIOD: ${{ inputs.grace_period }}
steps:
- uses: actions/checkout@v7
diff --git a/cross-version/test.sh b/cross-version/test.sh
index 7c01a691d90..ff25de123a0 100755
--- a/cross-version/test.sh
+++ b/cross-version/test.sh
@@ -20,6 +20,7 @@ TARGET_VERSION=""
METABASE_PORT="${METABASE_PORT:-3000}"
HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-120}"
LOG_DIR="${LOG_DIR:-$SCRIPT_DIR/logs}"
+GRACE_PERIOD="${GRACE_PERIOD:-3}"
# Colors for output
RED='\033[0;31m'
@@ -251,11 +252,79 @@ migrate_down_step() {
fi
if grep -q "not rolled back" <<<"$output"; then
+ if changesets_in_grace_period "$output"; then
+ return 2
+ fi
error "migrate down had changesets that were not rolled back (check logs above)"
return 1
fi
}
+fetch_history() {
+ local git_=(git -C "$REPO_ROOT") head
+ [[ "$("${git_[@]}" rev-parse --is-shallow-repository)" == "true" ]] || return 0
+ head=$("${git_[@]}" rev-parse HEAD)
+ "${git_[@]}" fetch -q --no-tags --shallow-since="${GRACE_PERIOD} days ago" origin "$head" &&
+ "${git_[@]}" fetch -q --no-tags --deepen=1 origin "$head"
+}
+
+changesets_in_grace_period() {
+ local ids id id_line commit added now age_hours base json=""
+ ids=$(grep "not in the changelog file:" <<<"$1" | sed 's/.*not in the changelog file://' |
+ grep -oE 'v[0-9]+\.[0-9A-Za-z:._-]+' | sort -u || true)
+ if [[ -z "$ids" ]]; then
+ warn "Could not read the changeset ids that were not rolled back"
+ return 1
+ fi
+
+ if ! fetch_history; then
+ warn "Could not fetch the git history"
+ return 1
+ fi
+
+ base=$(git -C "$REPO_ROOT" rev-list -1 --first-parent --before="${GRACE_PERIOD} days ago" HEAD)
+ if [[ -z "$base" ]]; then
+ warn "Could not find the commit from ${GRACE_PERIOD} day(s) ago"
+ return 1
+ fi
+
+ now=$(date +%s)
+ for id in $ids; do
+ id_line="^[[:space:]]*id:[[:space:]]*['\"]?${id//./\\.}['\"]?[[:space:]]*$"
+ if git -C "$REPO_ROOT" grep -qE "$id_line" "$base" -- resources/migrations; then
+ warn "$id was already in ${base:0:11}, which is older than ${GRACE_PERIOD} day(s)"
+ return 1
+ fi
+ commit="" added=""
+ read -r commit added < <(git -C "$REPO_ROOT" -c log.showRoot=false log --since="${GRACE_PERIOD} days ago" \
+ -E -G "$id_line" --format='%H %ct' HEAD -- resources/migrations | tail -1) || true
+ if [[ -z "${commit:-}" ]]; then
+ warn "$id was not added in the last ${GRACE_PERIOD} day(s)"
+ return 1
+ fi
+ age_hours=$(( (now - added) / 3600 ))
+ warn "$id was added ${age_hours}h ago in ${commit:0:11}, which is in the grace period of ${GRACE_PERIOD} day(s)"
+ json+="${json:+,}{\"id\":\"$id\",\"commit\":\"$commit\",\"age_hours\":$age_hours}"
+ done
+
+ if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
+ echo "unreleased_migrations=[$json]" >> "$GITHUB_OUTPUT"
+ fi
+}
+
+skip_for_unreleased_changesets() {
+ local message="Skipped $SOURCE_VERSION → $TARGET_VERSION: master has migrations that the released images do not have yet. The test runs in full again after the release."
+ warn "$message"
+ echo "::warning title=Cross-version test skipped::$message"
+ if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
+ echo "⚠️ $message" >> "$GITHUB_STEP_SUMMARY"
+ fi
+ if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
+ echo "skipped=true" >> "$GITHUB_OUTPUT"
+ fi
+ exit 0
+}
+
# Cascading migrate down: rolls back multiple major versions one at a time
# Metabase only supports rolling back one major version per `migrate down` call,
# so we need to iterate through intermediate versions.
@@ -285,7 +354,11 @@ cascading_migrate_down() {
log "--------------------------------------------"
log "Cascade step $step/$steps: $version"
log "--------------------------------------------"
- if ! migrate_down_step "$version"; then
+ local step_rc=0
+ migrate_down_step "$version" || step_rc=$?
+ if (( step_rc == 2 )); then
+ return 2
+ elif (( step_rc != 0 )); then
error "Migrate down failed at step $step ($version)"
return 1
fi
@@ -439,7 +512,11 @@ main() {
log ""
log "Step 5: Rolling back database ($SOURCE_VERSION → $TARGET_VERSION)..."
- if ! cascading_migrate_down "$SOURCE_VERSION" "$TARGET_VERSION"; then
+ local cascade_rc=0
+ cascading_migrate_down "$SOURCE_VERSION" "$TARGET_VERSION" || cascade_rc=$?
+ if (( cascade_rc == 2 )); then
+ skip_for_unreleased_changesets
+ elif (( cascade_rc != 0 )); then
error "❌ migrate down failed"
exit 1
fi
diff --git a/release/src/slack.ts b/release/src/slack.ts
index c3fb5b9804a..fcce489efc4 100644
--- a/release/src/slack.ts
+++ b/release/src/slack.ts
@@ -254,6 +254,33 @@ export async function findSlackMessage({
};
}
+export async function getRecentSlackMessages({
+ channelName,
+ sinceHours,
+ limit = 100,
+}: {
+ channelName: string;
+ sinceHours?: number;
+ limit?: number;
+}): Promise<string[]> {
+ const channelId = await getSlackChannelId(channelName);
+ if (!channelId) {
+ throw new Error(`Could not find channel ${channelName}`);
+ }
+
+ const oldest =
+ sinceHours === undefined
+ ? undefined
+ : String(Math.floor(Date.now() / 1000) - sinceHours * 3600);
+ const response = await slack.conversations.history({
+ channel: channelId,
+ oldest,
+ limit,
+ });
+
+ return response.messages?.map((message) => message.text ?? "") ?? [];
+}
+
function getExistingSlackMessage(version: string, channelName: string) {
return findSlackMessage({ channelName, text: getReleaseTitle(version) });
}