Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add eval-run triggers (#77960)

lbrdnk0a5aa515e6cc
diff --git a/.github/workflows/ai-benchmark.yml b/.github/workflows/ai-benchmark.yml
index 5ccda87d754..66d2fe1f900 100644
--- a/.github/workflows/ai-benchmark.yml
+++ b/.github/workflows/ai-benchmark.yml
@@ -1,21 +1,172 @@
name: AI Benchmark
-
-# Runs the ai-benchmark E2E test suite against a Metabase instance.
-#
-# NOTE: This is a stub. The real workflow — which builds the benchmark Docker
-# image, boots Metabase + Postgres with a preloaded data dump, runs the
-# benchmark suite, and uploads results as artifacts — is being developed on a
-# feature branch. This stub exists on master so that the "Run workflow" button
-# is available in the GitHub Actions UI, which lets us dispatch the real
-# workflow against feature branches during development.
+run-name: "dispatch evals · ${{ github.event_name == 'pull_request' && format('smoke PR#{0}', github.event.pull_request.number) || (inputs.pr != '' && format('PR#{0}', inputs.pr) || (inputs.ref != '' && inputs.ref || 'master')) }}"
on:
+ pull_request:
+ types: [labeled]
workflow_dispatch:
+ inputs:
+ pr:
+ description: "Metabase PR number (sets ref to the PR head)"
+ required: false
+ default: ""
+ ref:
+ description: "Metabase ref"
+ required: false
+ default: ""
+ scope:
+ description: "Scope"
+ required: true
+ type: choice
+ options: [smoke, full, "all (EXPENSIVE)"]
+ default: smoke
+ sample:
+ description: "Sample (0 = full suite)"
+ required: false
+ default: "5"
+ evals_branch:
+ description: "Evals branch"
+ default: main
+
+concurrency:
+ group: evals-dispatch-${{ (github.event.label.name == 'evals/smoke' && github.event.pull_request.number) || github.run_id }}
+ cancel-in-progress: true
jobs:
- placeholder:
- name: Placeholder
- runs-on: ubuntu-latest
+ unsupported:
+ if: >-
+ github.event_name == 'pull_request'
+ && github.event.label.name == 'evals/smoke'
+ && github.event.pull_request.head.repo.full_name != github.repository
+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+ timeout-minutes: 5
+ steps:
+ - run: |
+ echo "::error::evals/smoke only runs on same-repo PRs; this PR's head is on a fork"
+ exit 1
+
+ announce:
+ if: >-
+ github.event_name == 'pull_request'
+ && github.event.label.name == 'evals/smoke'
+ && github.event.pull_request.head.repo.full_name == github.repository
+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+ timeout-minutes: 5
+ outputs:
+ comment_id: ${{ steps.post.outputs.comment_id }}
+ steps:
+ - name: Comment (building)
+ id: post
+ uses: actions/github-script@v9
+ with:
+ github-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ script: |
+ const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+ const body = `🤖 **Evals smoke** — ⏳ building the image… ([logs](${runUrl}))`;
+ const { owner, repo } = context.repo;
+ const issue_number = context.payload.pull_request.number;
+ // Drop earlier verdicts first: they describe shas that are no longer head, and a stale
+ // green one reads as this PR passing. The prefix survives eval-run's rewrites.
+ const existing = await github.paginate(github.rest.issues.listComments, { owner, repo, issue_number });
+ for (const c of existing.filter(c => c.body.startsWith('🤖 **Evals'))) {
+ await github.rest.issues.deleteComment({ owner, repo, comment_id: c.id }).catch(() => {});
+ }
+ const { data } = await github.rest.issues.createComment({ owner, repo, issue_number, body });
+ core.setOutput('comment_id', String(data.id));
+
+ uberjar:
+ needs: [announce]
+ if: >-
+ github.event_name == 'pull_request'
+ && github.event.label.name == 'evals/smoke'
+ && github.event.pull_request.head.repo.full_name == github.repository
+ uses: ./.github/workflows/uberjar.yml
+ secrets: inherit
+
+ smoke:
+ needs: [announce, uberjar]
+ if: >-
+ github.event_name == 'pull_request'
+ && github.event.label.name == 'evals/smoke'
+ && github.event.pull_request.head.repo.full_name == github.repository
+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+ timeout-minutes: 5
+ env:
+ COMMENT_ID: ${{ needs.announce.outputs.comment_id }}
+ steps:
+ - name: Dispatch smoke eval-run
+ env:
+ GH_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ PR: ${{ github.event.pull_request.number }}
+ HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ EVALS_BRANCH: main
+ run: |
+ gh api repos/metabase/evals/actions/workflows/eval-run.yml/dispatches \
+ -f ref="$EVALS_BRANCH" \
+ -f "inputs[pr]=$PR" \
+ -f "inputs[ref]=$HEAD_SHA" \
+ -f "inputs[scope]=smoke" \
+ -f "inputs[surfaces]=all" \
+ -f "inputs[sample]=5" \
+ -f "inputs[evals_branch]=$EVALS_BRANCH" \
+ -f "inputs[comment_id]=$COMMENT_ID"
+
+ finalize:
+ # If the build or dispatch fell over, eval-run never took over the comment — leave a
+ # failure verdict instead of a comment stuck at "building the image…".
+ needs: [announce, uberjar, smoke]
+ if: always() && needs.announce.result == 'success' && needs.smoke.result != 'success'
+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+ timeout-minutes: 5
+ env:
+ COMMENT_ID: ${{ needs.announce.outputs.comment_id }}
+ SMOKE_RESULT: ${{ needs.smoke.result }}
+ UBERJAR_RESULT: ${{ needs.uberjar.result }}
+ steps:
+ - name: Comment (build/dispatch failed)
+ uses: actions/github-script@v9
+ with:
+ github-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ script: |
+ const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+ // A concurrency-cancelled retrigger lands here too; don't stamp it as an infra failure.
+ const cancelled = [process.env.SMOKE_RESULT, process.env.UBERJAR_RESULT].includes('cancelled');
+ const body = cancelled
+ ? `🤖 **Evals smoke** — ⚪ cancelled before evals started ([logs](${runUrl}))`
+ : `🤖 **Evals smoke** — ❌ build or dispatch failed before evals started ([logs](${runUrl}))`;
+ // The replacement run deletes this comment on its way in, so a concurrency cancel can
+ // land here with nothing to patch. Losing the stamp is fine; failing the job isn't.
+ await github.rest.issues.updateComment({
+ owner: context.repo.owner, repo: context.repo.repo,
+ comment_id: Number(process.env.COMMENT_ID), body })
+ .catch(e => core.warning(`could not update comment (deleted?): ${e.message}`));
+ // eval-run only consumes the label once it has taken the run over, so anything failing
+ // before the dispatch leaves it applied — and `labeled` only fires on a transition, which
+ // makes re-applying it to retry a no-op. Hand it back.
+ await github.rest.issues.removeLabel({
+ owner: context.repo.owner, repo: context.repo.repo,
+ issue_number: context.payload.pull_request.number,
+ name: 'evals/smoke',
+ }).catch(() => {});
+
+ dispatch:
+ if: github.event_name == 'workflow_dispatch'
+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+ timeout-minutes: 5
steps:
- - name: Stub
- run: echo "ai-benchmark workflow stub — real implementation pending"
+ - name: Dispatch eval-run
+ env:
+ GH_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+ EVALS_BRANCH: ${{ github.event.inputs.evals_branch }}
+ PR: ${{ inputs.pr }}
+ REF: ${{ inputs.ref }}
+ SCOPE: ${{ inputs.scope }}
+ SAMPLE: ${{ inputs.sample }}
+ run: |
+ gh api repos/metabase/evals/actions/workflows/eval-run.yml/dispatches \
+ -f ref="$EVALS_BRANCH" \
+ -f "inputs[pr]=$PR" \
+ -f "inputs[ref]=$REF" \
+ -f "inputs[scope]=$SCOPE" \
+ -f "inputs[sample]=${SAMPLE:-5}" \
+ -f "inputs[evals_branch]=$EVALS_BRANCH"