Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Bgrabow/profile tests workflow (#80258)

Ben Grabow06f7a3ca8bca
diff --git a/.github/actions/collect-profile/action.yml b/.github/actions/collect-profile/action.yml
new file mode 100644
index 00000000000..2bb7b0ae917
--- /dev/null
+++ b/.github/actions/collect-profile/action.yml
@@ -0,0 +1,53 @@
+name: Collect profile
+description: >-
+ Attach a flamegraph and per-test wall times to a job whose tests ran under clj-async-profiler.
+ Does nothing when the run was not profiled.
+
+inputs:
+ profiled:
+ required: true
+ description: >-
+ "true" when the test step ran under the profiler. Passed in rather than read from the
+ environment so the decision stays recorded by the step that made it.
+ artifact-name:
+ required: true
+ description: Name for the uploaded artifact. Must be unique across the jobs of one run.
+
+runs:
+ using: "composite"
+ steps:
+ # Unlike a JUnit failure report, these run on a green build too — a suite that passes slowly is
+ # exactly what profiling is for.
+ - name: Summarize test wall times
+ if: ${{ inputs.profiled == 'true' }}
+ continue-on-error: true
+ shell: bash
+ run: |
+ {
+ echo "### Slowest test namespaces (seconds)"
+ echo '```'
+ grep -h '<testsuite' target/junit/*.xml 2>/dev/null \
+ | sed -E 's|.*<testsuite name="([^"]*)" time="([^"]*)".*|\2 \1|' \
+ | sort -rn | head -25
+ echo '```'
+ echo "### Slowest tests (seconds)"
+ echo '```'
+ grep -h '<testcase' target/junit/*.xml 2>/dev/null \
+ | sed -E 's|.*classname="([^"]*)" name="([^"]*)" time="([^"]*)".*|\3 \1/\2|' \
+ | sort -rn | head -25
+ echo '```'
+ } >> "$GITHUB_STEP_SUMMARY"
+
+ # Only the rendered flamegraph is kept: the raw collapsed stacks it was built from run to
+ # hundreds of MB on a long suite, and are reproducible locally from the same test selection.
+ - name: Upload profile
+ if: ${{ inputs.profiled == 'true' }}
+ continue-on-error: true
+ uses: actions/upload-artifact@v7
+ with:
+ name: ${{ inputs.artifact-name }}
+ path: |
+ target/clj-async-profiler/results/*flamegraph.html
+ target/junit/*.xml
+ retention-days: 14
+ if-no-files-found: warn
diff --git a/.github/actions/test-driver/action.yml b/.github/actions/test-driver/action.yml
index a34f1feb08d..9cf72695e9f 100644
--- a/.github/actions/test-driver/action.yml
+++ b/.github/actions/test-driver/action.yml
@@ -101,7 +101,21 @@ runs:
# vector reaches clojure -X as a single argument.
run: |
BASE="clojure -X:dev:ci:${{ inputs.edition }}:${{ inputs.edition }}-dev:drivers:drivers-dev:test"
- if [ "${{ inputs.granular-rerun }}" = "true" ] && [ "${{ github.run_attempt }}" != "1" ] && [ -s target/failed-tests ]; then
+ # `:profile-tests` overrides `:test`'s exec-fn, so it has to come last. See dev.profile-tests.
+ # The `profiled` output is what the collection steps below key off, so that the decision is
+ # recorded by the step that acts on it rather than re-derived from the environment.
+ if [ "$PROFILE_TESTS" = "true" ]; then
+ echo "Profiling this run; a flamegraph will be attached to the job."
+ echo "profiled=true" >> "$GITHUB_OUTPUT"
+ BASE="$BASE:profile-tests"
+ fi
+ # An explicit selection wins over the rerun narrowing below: it was asked for by hand, on this
+ # attempt. Names may be separated by commas or spaces; hawk wants them as one vector argument.
+ if [ -n "$ONLY_TESTS" ]; then
+ ONLY="[$(echo "$ONLY_TESTS" | tr ',' ' ')]"
+ echo "Running only: $ONLY"
+ $BASE :only "$ONLY"
+ elif [ "${{ inputs.granular-rerun }}" = "true" ] && [ "${{ github.run_attempt }}" != "1" ] && [ -s target/failed-tests ]; then
echo "Granular rerun of previously-failed tests: $(cat target/failed-tests)"
$BASE :only "$(cat target/failed-tests)"
else
@@ -113,6 +127,13 @@ runs:
# ultimately fails.
continue-on-error: true
+ - name: Collect profile
+ if: ${{ !cancelled() }}
+ uses: ./.github/actions/collect-profile
+ with:
+ profiled: ${{ steps.run-test.outputs.profiled }}
+ artifact-name: ${{ format('profile-{0}{1}', inputs.junit-name, inputs.artifact-suffix && format('-{0}', inputs.artifact-suffix) || '') }}
+
- name: Report failed tests to ci-conductor
if: ${{ !cancelled() && ( steps.run-test.outcome == 'failure' || github.run_attempt != '1' ) }}
# This step should NEVER fail the job!
diff --git a/.github/workflows/app-db.yml b/.github/workflows/app-db.yml
index d7a58b1a15c..62279a1e79d 100644
--- a/.github/workflows/app-db.yml
+++ b/.github/workflows/app-db.yml
@@ -11,6 +11,30 @@ on:
type: boolean
default: false
+ # Run one app-DB job on demand, against the ref you dispatch on, optionally under a profiler.
+ # Everything else about the job -- services, matrix legs, test args, quarantine gating -- is
+ # whatever the normal CI run uses, because it is the same job.
+ workflow_dispatch:
+ inputs:
+ app-db:
+ description: 'Which app-DB job to run. Only this one runs; the others are skipped.'
+ type: choice
+ required: true
+ options:
+ - mariadb
+ - mysql
+ - postgres
+ tests:
+ description: 'Optional: run only these tests, comma- or space-separated, e.g. metabase.db.custom-migrations-test/some-test. Bare namespaces work too. Blank runs the job''s usual set.'
+ type: string
+ required: false
+ default: ''
+ profile:
+ description: 'Profile the test run and attach a flamegraph to the job. Adds sampling overhead.'
+ type: boolean
+ required: false
+ default: false
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}-app-db
cancel-in-progress: true
@@ -21,6 +45,14 @@ concurrency:
env:
CI_CONDUCTOR_BASE_URL: ${{ secrets.CI_CONDUCTOR_BASE_URL }}
CI_CONDUCTOR_WEBHOOK_SECRET: ${{ secrets.CI_CONDUCTOR_WEBHOOK_SECRET }}
+ # Both read by the test-driver action, which is what actually runs the tests. Set here rather than
+ # passed as inputs to each job so that turning either on is one line, not one line per job.
+ # Empty (and so off) on workflow_call, where neither input exists.
+ #
+ # ONLY_TESTS replaces the job's own test selection, so every matrix leg of the chosen job runs the
+ # whole list -- each app DB has three.
+ PROFILE_TESTS: ${{ inputs.profile }}
+ ONLY_TESTS: ${{ inputs.tests }}
jobs:
app-db-matrix:
@@ -103,7 +135,7 @@ jobs:
be-tests-mariadb:
needs: app-db-matrix
- if: ${{ !inputs.skip }}
+ if: ${{ !inputs.skip && (inputs.app-db == '' || inputs.app-db == 'mariadb') }}
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
timeout-minutes: 40
strategy:
@@ -189,7 +221,7 @@ jobs:
be-tests-mysql:
needs: app-db-matrix
- if: ${{ !inputs.skip }}
+ if: ${{ !inputs.skip && (inputs.app-db == '' || inputs.app-db == 'mysql') }}
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
timeout-minutes: 40
strategy:
@@ -283,7 +315,7 @@ jobs:
be-tests-postgres:
needs: app-db-matrix
- if: ${{ !inputs.skip }}
+ if: ${{ !inputs.skip && (inputs.app-db == '' || inputs.app-db == 'postgres') }}
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
timeout-minutes: 40
strategy:
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index 7cfb922c9bf..90a67dae270 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -11,6 +11,22 @@ on:
type: boolean
default: false
+ # Run the backend test suite on demand, against the ref you dispatch on, optionally under a
+ # profiler. Everything else about the jobs -- matrix legs, tag filters, quarantine gating -- is
+ # whatever the normal CI run uses, because they are the same jobs.
+ workflow_dispatch:
+ inputs:
+ tests:
+ description: 'Optional: run only these tests, comma- or space-separated, e.g. metabase.query-processor-test/some-test. Bare namespaces work too. Blank runs each leg''s usual set.'
+ type: string
+ required: false
+ default: ''
+ profile:
+ description: 'Profile the test run and attach a flamegraph to the job. Adds sampling overhead.'
+ type: boolean
+ required: false
+ default: false
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}-backend
cancel-in-progress: true
@@ -21,6 +37,13 @@ concurrency:
env:
CI_CONDUCTOR_BASE_URL: ${{ secrets.CI_CONDUCTOR_BASE_URL }}
CI_CONDUCTOR_WEBHOOK_SECRET: ${{ secrets.CI_CONDUCTOR_WEBHOOK_SECRET }}
+ # Both read by the be-tests run step below. Set here rather than passed per job so that turning
+ # either on is one line. Empty (and so off) on workflow_call, where neither input exists.
+ #
+ # ONLY_TESTS replaces the leg's own test selection, so every matrix leg runs the whole list: the
+ # partition args no longer apply, and legs that hold none of the named tests run nothing.
+ PROFILE_TESTS: ${{ inputs.profile }}
+ ONLY_TESTS: ${{ inputs.tests }}
jobs:
be-linter-clj-kondo:
@@ -222,7 +245,21 @@ jobs:
# whole `[ns/var ...]` vector reaches clojure -X as a single argument.
run: |
BASE="clojure -X:dev:ci:test:${{ matrix.job.edition }}:${{ matrix.job.edition }}-dev"
- if [ "${{ github.run_attempt }}" != "1" ] && [ -s target/failed-tests ]; then
+ # `:profile-tests` overrides `:test`'s exec-fn, so it has to come last. See dev.profile-tests.
+ # The `profiled` output is what the collection steps below key off, so that the decision is
+ # recorded by the step that acts on it rather than re-derived from the environment.
+ if [ "$PROFILE_TESTS" = "true" ]; then
+ echo "Profiling this run; a flamegraph will be attached to the job."
+ echo "profiled=true" >> "$GITHUB_OUTPUT"
+ BASE="$BASE:profile-tests"
+ fi
+ # An explicit selection wins over the rerun narrowing below: it was asked for by hand, on this
+ # attempt. Names may be separated by commas or spaces; hawk wants them as one vector argument.
+ if [ -n "$ONLY_TESTS" ]; then
+ ONLY="[$(echo "$ONLY_TESTS" | tr ',' ' ')]"
+ echo "Running only: $ONLY"
+ $BASE :only "$ONLY"
+ elif [ "${{ github.run_attempt }}" != "1" ] && [ -s target/failed-tests ]; then
echo "Granular rerun of previously-failed tests: $(cat target/failed-tests)"
$BASE :only "$(cat target/failed-tests)"
else
@@ -231,6 +268,13 @@ jobs:
# Keep this step continue-on-error because the quarantine logic controls whether it ultimately fails.
continue-on-error: true
+ - name: Collect profile
+ if: ${{ !cancelled() }}
+ uses: ./.github/actions/collect-profile
+ with:
+ profiled: ${{ steps.run-java-tests.outputs.profiled }}
+ artifact-name: profile-be-tests-java-${{ matrix.java-version }}-${{ matrix.job.test-group-name }}
+
- name: Report failed tests to ci-conductor
if: ${{ !cancelled() && ( steps.run-java-tests.outcome == 'failure' || github.run_attempt != '1' ) }}
# This step should NEVER fail the job!
diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.yml
index 570033c33c0..bfab97b5c0c 100644
--- a/.github/workflows/drivers.yml
+++ b/.github/workflows/drivers.yml
@@ -10,6 +10,45 @@ on:
description: Run the full suite regardless of the diff (master / release branches, ci:run-all label).
type: boolean
default: false
+ # Run one driver job on demand, against the ref you dispatch on, optionally under a profiler.
+ # Everything else about the job -- credentials, services, test args, quarantine gating -- is
+ # whatever the normal CI run uses, because it is the same job.
+ workflow_dispatch:
+ inputs:
+ driver:
+ description: 'Which driver job to run. Only this one runs; the usual decision rules are bypassed.'
+ type: choice
+ required: true
+ options:
+ - athena
+ - bigquery
+ - clickhouse
+ - databricks
+ - druid-jdbc
+ - h2
+ - mongo
+ - mongo-sharded-cluster
+ - mongo-ssl
+ - mysql-mariadb
+ - oracle
+ - postgres
+ - presto-jdbc
+ - redshift
+ - snowflake
+ - sparksql
+ - sqlite
+ - sqlserver
+ - vertica
+ tests:
+ description: 'Optional: run only these tests, comma- or space-separated, e.g. metabase.driver.bigquery-cloud-sdk-test/sync-test. Bare namespaces work too. Blank runs the job''s usual set.'
+ type: string
+ required: false
+ default: ''
+ profile:
+ description: 'Profile the test run and attach a flamegraph to the job. Adds sampling overhead.'
+ type: boolean
+ required: false
+ default: false
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}-drivers
@@ -21,6 +60,14 @@ concurrency:
env:
CI_CONDUCTOR_BASE_URL: ${{ secrets.CI_CONDUCTOR_BASE_URL }}
CI_CONDUCTOR_WEBHOOK_SECRET: ${{ secrets.CI_CONDUCTOR_WEBHOOK_SECRET }}
+ # Both read by the test-driver action, which is what actually runs the tests. Set here rather than
+ # passed as inputs to each job so that turning either on is one line, not one line per driver.
+ # Empty (and so off) on workflow_call, where neither input exists.
+ #
+ # ONLY_TESTS replaces the job's own test selection, so every matrix leg of the chosen driver runs
+ # the whole list -- BigQuery, for instance, has three legs.
+ PROFILE_TESTS: ${{ inputs.profile }}
+ ONLY_TESTS: ${{ inputs.tests }}
jobs:
# This job determines which driver tests should run based on PR context.
@@ -69,6 +116,7 @@ jobs:
--force-run="${FORCE_RUN}"
--pr-labels="${{ join(github.event.pull_request.labels.*.name, ',') }}"
--skip="${{ inputs.skip }}"
+ --only-driver="${ONLY_DRIVER}"
)
echo "Running:"
echo "./bin/mage -driver-decisions ${MAGE_ARGS[*]}"
@@ -83,6 +131,7 @@ jobs:
env:
BASE_REF: ${{ github.event.pull_request.base.ref || github.ref_name }}
FORCE_RUN: ${{ inputs.force-run }}
+ ONLY_DRIVER: ${{ inputs.driver }}
shell: bash
be-tests-athena-ee:
diff --git a/.github/workflows/semantic-search.yml b/.github/workflows/semantic-search.yml
index 6473d3bd8c6..af270d45314 100644
--- a/.github/workflows/semantic-search.yml
+++ b/.github/workflows/semantic-search.yml
@@ -7,6 +7,32 @@ on:
type: boolean
default: false
+ # Run one semantic-search job on demand, against the ref you dispatch on, optionally under a
+ # profiler. Everything else about the job -- services, matrix legs, test args, quarantine gating --
+ # is whatever the normal CI run uses, because it is the same job.
+ workflow_dispatch:
+ inputs:
+ suite:
+ description: 'Which semantic-search job to run. Only this one runs; the others are skipped.'
+ type: choice
+ required: true
+ options:
+ - h2
+ - mariadb
+ - mysql
+ - postgres
+ - appdb-mode
+ tests:
+ description: 'Optional: run only these tests, comma- or space-separated, e.g. metabase-enterprise.semantic-search.index-test/some-test. Bare namespaces work too. Blank runs the job''s usual set.'
+ type: string
+ required: false
+ default: ''
+ profile:
+ description: 'Profile the test run and attach a flamegraph to the job. Adds sampling overhead.'
+ type: boolean
+ required: false
+ default: false
+
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}-semantic-search
cancel-in-progress: true
@@ -17,10 +43,18 @@ concurrency:
env:
CI_CONDUCTOR_BASE_URL: ${{ secrets.CI_CONDUCTOR_BASE_URL }}
CI_CONDUCTOR_WEBHOOK_SECRET: ${{ secrets.CI_CONDUCTOR_WEBHOOK_SECRET }}
+ # Both read by the test-driver action, which is what actually runs the tests. Set here rather than
+ # passed as inputs to each job so that turning either on is one line, not one line per job.
+ # Empty (and so off) on workflow_call, where neither input exists.
+ #
+ # ONLY_TESTS replaces the job's own test selection, so every matrix leg of the chosen job runs the
+ # whole list.
+ PROFILE_TESTS: ${{ inputs.profile }}
+ ONLY_TESTS: ${{ inputs.tests }}
jobs:
semantic-search-tests-h2:
- if: ${{ !inputs.skip }}
+ if: ${{ !inputs.skip && (inputs.suite == '' || inputs.suite == 'h2') }}
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
timeout-minutes: 40
strategy:
@@ -65,7 +99,7 @@ jobs:
quarantine-engine: ${{ vars.QUARANTINE_ENGINE }}
semantic-search-tests-mariadb:
- if: ${{ !inputs.skip }}
+ if: ${{ !inputs.skip && (inputs.suite == '' || inputs.suite == 'mariadb') }}
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
timeout-minutes: 40
strategy:
@@ -131,7 +165,7 @@ jobs:
quarantine-engine: ${{ vars.QUARANTINE_ENGINE }}
semantic-search-tests-mysql:
- if: ${{ !inputs.skip }}
+ if: ${{ !inputs.skip && (inputs.suite == '' || inputs.suite == 'mysql') }}
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
timeout-minutes: 40
strategy:
@@ -204,7 +238,7 @@ jobs:
quarantine-engine: ${{ vars.QUARANTINE_ENGINE }}
semantic-search-tests-postgres:
- if: ${{ !inputs.skip }}
+ if: ${{ !inputs.skip && (inputs.suite == '' || inputs.suite == 'postgres') }}
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
timeout-minutes: 40
strategy:
@@ -272,7 +306,7 @@ jobs:
# MB_PGVECTOR_DB_URL is deliberately unset. The dedicated-harness tests gate themselves off; the
# appdb-pgvector-mode round trip (and the URL-independent unit tests) run for real.
semantic-search-tests-appdb-mode:
- if: ${{ !inputs.skip }}
+ if: ${{ !inputs.skip && (inputs.suite == '' || inputs.suite == 'appdb-mode') }}
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
timeout-minutes: 40
strategy:
diff --git a/bb.edn b/bb.edn
index f600c02fb82..e3b474cec47 100644
--- a/bb.edn
+++ b/bb.edn
@@ -396,6 +396,7 @@
["./bin/mage -driver-decisions --pr-labels=ci:run-all-cloud-drivers" "Run all cloud drivers via label"]
["./bin/mage -driver-decisions --pr-labels=ci:run-mysql-mariadb" "Force-run a specific driver via label"]
["./bin/mage -driver-decisions --pr-labels=ci:run-all-drivers" "Force-run all drivers via label"]
+ ["./bin/mage -driver-decisions --only-driver=bigquery" "Run only the BigQuery job, whatever the other rules say"]
["./bin/mage -driver-decisions --github-output-only" "Output only GITHUB_OUTPUT format (for CI)"]]
:requires [[clojure.string :as str]
[table.core :as table]
@@ -407,6 +408,7 @@
"# How we decide to run a driver's tests. For each driver, we check these conditions in order (first match wins):\n"
(with-out-str
(table/table [["Priority" "Condition" "Result" "Reason"]
+ ["0" "--only-driver=DRIVER was passed" "RUN/SKIP" "only that one driver runs"]
["1" "Global force-run" "RUN" "master/release branch or ci:run-all label (skip list ignored)"]
["2" "Global skip (no backend changes)" "SKIP" "workflow skip (no backend changes)"]
["3" "Driver is :h2 or :postgres" "RUN" "H2/Postgres always run"]
@@ -445,6 +447,7 @@
[nil "--force-run FORCE_RUN" "Run every driver regardless of the diff"]
[nil "--pr-labels PR_LABELS" "Comma delimited labels on the pr"]
[nil "--skip SKIP"]
+ [nil "--only-driver DRIVER" "Run only this driver's job, ignoring every other rule"]
[nil "--github-output-only" "Only output GITHUB_OUTPUT format (no human-readable output)"]]
:task (task! (mage.modules/-main parsed))}
diff --git a/deps.edn b/deps.edn
index 88cb6f32127..6d445a1ec8d 100644
--- a/deps.edn
+++ b/deps.edn
@@ -900,6 +900,21 @@
:jvm-opts ["-XX:+CITime" ; print time spent in JIT compiler
"-Xlog:gc"]}
+ ;; Run tests under clj-async-profiler; writes a flamegraph to target/clj-async-profiler/results.
+ ;; Must come after `:test`, whose `:exec-fn` it overrides:
+ ;; clojure -X:dev:ci:ee:ee-dev:drivers:drivers-dev:test:profile-tests :only '[metabase.some-test]'
+ :profile-tests
+ {;; `dev.profile-tests` lives in dev/src and clj-async-profiler is otherwise a `:dev`-only dep;
+ ;; declared here too so the alias stands on its own instead of failing at exec time when `:dev`
+ ;; happens to be absent. Keep the version in step with the one in `:dev`.
+ :extra-paths ["dev/src"]
+ :extra-deps {com.clojure-goes-fast/clj-async-profiler {:mvn/version "1.6.2"}}
+ :exec-fn dev.profile-tests/run
+ :jvm-opts ["-Dclj-async-profiler.output-dir=target"
+ ;; without these, inlined frames are attributed to the wrong method
+ "-XX:+UnlockDiagnosticVMOptions"
+ "-XX:+DebugNonSafepoints"]}
+
;; get the H2 shell with clojure -X:dev:h2
:h2
{:extra-paths ["dev/src"]
diff --git a/dev/src/dev/profile_tests.clj b/dev/src/dev/profile_tests.clj
new file mode 100644
index 00000000000..3c53f9d3495
--- /dev/null
+++ b/dev/src/dev/profile_tests.clj
@@ -0,0 +1,36 @@
+(ns dev.profile-tests
+ "Run tests under clj-async-profiler and write a flamegraph next to the JUnit output.
+
+ clojure -X:dev:ci:ee:ee-dev:drivers:drivers-dev:test:profile-tests :only '[metabase.driver.bigquery-cloud-sdk-test]'
+
+ `:profile-tests` must come after `:test` so its `:exec-fn` wins."
+ (:require
+ [clj-async-profiler.core :as prof]
+ [metabase.test-runner :as test-runner]))
+
+(defn run
+ "`clojure -X` entrypoint. Takes the usual test-runner options plus:
+
+ `:profile-event` - async-profiler event (default `:wall`). `:wall` samples blocked threads too, which is
+ what you want for tests dominated by network round-trips; `:cpu` would miss them
+ entirely. `:cpu` additionally needs `kernel.perf_event_paranoid <= 1`.
+ `:profile-interval` - sampling interval in nanoseconds (default 20ms). async-profiler's own 1ms default
+ is far too fine for a suite that runs for tens of minutes: raw stacks accumulate
+ at roughly 7 MB per minute of profiling at 10ms, and every halving of the
+ interval doubles that.
+
+ Returns nothing; exits the JVM with the test result."
+ [{:keys [profile-event profile-interval]
+ :or {profile-event :wall, profile-interval 20000000}
+ :as options}]
+ (prof/start {:event profile-event, :interval profile-interval, :threads true})
+ (let [summary (try
+ ;; `:mode :repl` so hawk returns the summary instead of calling `System/exit` — an exit here
+ ;; would skip flamegraph generation. It still writes JUnit XML and prints the pretty report.
+ (test-runner/find-and-run-tests-repl
+ (-> options
+ (dissoc :profile-event :profile-interval)
+ (assoc :mode :repl)))
+ (finally
+ (println "Wrote profile to" (str (prof/stop {:title (pr-str (:only options))})))))]
+ (System/exit (if (pos? (+ (:error summary 0) (:fail summary 0))) 1 0))))
diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.clj
index 690a44805ab..9519bd2afcb 100644
--- a/mage/src/mage/modules.clj
+++ b/mage/src/mage/modules.clj
@@ -388,6 +388,19 @@
[s]
(= (str/lower-case (str s)) "true"))
+(defn- parse-only-driver
+ "Parse the `--only-driver` CLI arg into a driver keyword, or nil when unset.
+
+ Throws on an unknown driver: a typo would otherwise read as `nil` and quietly run the normal decisions
+ instead of the one job that was asked for."
+ [s]
+ (when-not (str/blank? s)
+ (let [driver (keyword (str/trim s))]
+ (when-not (contains? (set all-drivers) driver)
+ (throw (ex-info (str "Unknown driver: " (str/trim s))
+ {:driver driver, :known-drivers (mapv name all-drivers)})))
+ driver)))
+
(defn- parse-labels
"Parse comma-separated labels string into a set of label strings."
[labels-str]
@@ -414,11 +427,21 @@
- deps.edn is changed (triggers all drivers)
- Clojure modules that the 'driver' module depends on are changed"
[driver
- {:keys [force-run pr-labels skip particular-driver-changed?]}
+ {:keys [force-run pr-labels skip particular-driver-changed? only-driver]}
driver-deps-affected?
skipped-drivers
updated]
(cond
+ ;; Priority 0: a request for one named driver job (workflow_dispatch on drivers.yml). Runs exactly
+ ;; that driver and nothing else -- not H2/Postgres, and not the skip list, since asking for a job by
+ ;; name is a stronger signal than any rule below.
+ only-driver
+ (if (= driver only-driver)
+ {:should-run true
+ :reason (str "requested via --only-driver=" (name only-driver))}
+ {:should-run false
+ :reason (str "--only-driver=" (name only-driver) " requested instead")})
+
;; Priority 1: Global force-run. Every driver runs; the remote skip list is not consulted at
;; all, so a stray entry there can't silently disable a driver's tests.
force-run
@@ -499,18 +522,21 @@
--git-ref=master \\
--force-run=false \\
--pr-labels=ci:run-all-cloud-drivers,other-label \\
- --skip=false"
+ --skip=false \\
+ --only-driver=bigquery"
[{:keys [options] :as _parsed}]
(let [github-output-only? (some? (:github-output-only options))
git-ref (get options :git-ref "master")
force-run (parse-bool (:force-run options))
+ only-driver (parse-only-driver (:only-driver options))
;; Detect file changes for ALL drivers via git diff
particular-driver-changed? (drivers-with-file-changes git-ref)
ctx {:git-ref git-ref
:force-run force-run
:pr-labels (parse-labels (:pr-labels options))
:skip (parse-bool (:skip options))
- :particular-driver-changed? particular-driver-changed?}
+ :particular-driver-changed? particular-driver-changed?
+ :only-driver only-driver}
;; force-run decides every driver on its own, so the remote skip list is neither fetched
;; nor honored there.
skipped (if force-run #{} (skip-drivers))
diff --git a/mage/test/mage/modules_test.clj b/mage/test/mage/modules_test.clj
index 338231745c2..c1dc50d5c06 100644
--- a/mage/test/mage/modules_test.clj
+++ b/mage/test/mage/modules_test.clj
@@ -17,6 +17,47 @@
:particular-driver-changed? #{}}
opts))
+;;; =============================================================================
+;;; Priority 0: --only-driver (workflow_dispatch asking for one job by name)
+;;; =============================================================================
+
+(deftest only-driver-runs-just-that-driver
+ (testing "--only-driver runs the named driver and skips every other one"
+ (doseq [driver [:h2 :postgres :mysql-mariadb :bigquery]]
+ (let [result (mage.modules/driver-decision driver
+ (make-ctx {:only-driver :bigquery})
+ false ; driver-deps-affected?
+ #{} ; skipped
+ #{})] ; updated
+ (is (= (= :bigquery driver) (:should-run result))
+ (str driver " should run only when it is the requested driver"))))))
+
+(deftest only-driver-beats-every-other-rule
+ (testing "the requested driver runs even when the CI config skips it and the workflow says skip"
+ (let [result (mage.modules/driver-decision :snowflake
+ (make-ctx {:only-driver :snowflake, :skip true})
+ false
+ #{:snowflake} ; skipped by CI config
+ #{})]
+ (is (true? (:should-run result)))
+ (is (= "requested via --only-driver=snowflake" (:reason result)))))
+ (testing "and H2/Postgres lose their always-run privilege, so the run is one job wide"
+ (let [result (mage.modules/driver-decision :h2
+ (make-ctx {:only-driver :snowflake})
+ false
+ #{}
+ #{})]
+ (is (false? (:should-run result)))
+ (is (= "--only-driver=snowflake requested instead" (:reason result))))))
+
+(deftest unknown-only-driver-is-rejected
+ (testing "a typo throws instead of silently falling back to the normal decisions"
+ (is (thrown-with-msg? Exception #"Unknown driver: bigquerry"
+ (#'mage.modules/parse-only-driver "bigquerry"))))
+ (testing "blank means no request"
+ (doseq [blank [nil "" " "]]
+ (is (nil? (#'mage.modules/parse-only-driver blank))))))
+
;;; =============================================================================
;;; Priority 5: Driver's own files changed (beats the CI config skip list)
;;; =============================================================================