Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add ci:run-<driver> and ci:run-all-drivers labels to mage driver decisions (#70894)

bryan99548507aa64
diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.yml
index 12c422738a4..cd81d7d35b7 100644
--- a/.github/workflows/drivers.yml
+++ b/.github/workflows/drivers.yml
@@ -82,6 +82,9 @@ jobs:
echo ""
# First call: show human-readable output for debugging
./bin/mage -driver-decisions "${MAGE_ARGS[@]}"
+ echo ""
+ echo "Want to force-run a specific driver? Add a ci:run-<driver> or ci:run-all-drivers label to your PR."
+ echo "Run './bin/mage -driver-decisions -h' locally for the full list of drivers and labels."
# Second call: output only key=value lines for GITHUB_OUTPUT
./bin/mage -driver-decisions "${MAGE_ARGS[@]}" --github-output-only >> $GITHUB_OUTPUT
shell: bash
diff --git a/bb.edn b/bb.edn
index e09ff7857f7..0fd98f205a7 100644
--- a/bb.edn
+++ b/bb.edn
@@ -329,6 +329,8 @@
{:doc "Determine which driver tests should run based on PR context. Outputs decisions for all drivers."
:examples [["./bin/mage -driver-decisions --git-ref=master --is-master-or-release=false --pr-labels= --skip=false" "Determine driver decisions for a PR"]
["./bin/mage -driver-decisions --pr-labels=ci:all-cloud-drivers" "Run all cloud drivers via label"]
+ ["./bin/mage -driver-decisions --pr-labels=ci:run-mysql-mariadb" "Force-run a specific driver via label"]
+ ["./bin/mage -driver-decisions --pr-labels=ci:run-all-drivers" "Force-run all drivers via label"]
["./bin/mage -driver-decisions --github-output-only" "Output only GITHUB_OUTPUT format (for CI)"]]
:requires [[clojure.string :as str]
[table.core :as table]
@@ -342,22 +344,32 @@
(table/table [["Priority" "Condition" "Result" "Reason"]
["1" "Global skip (no backend changes)" "SKIP" "workflow skip (no backend changes)"]
["2" "Driver is :h2 or :postgres" "RUN" "H2/Postgres always run"]
- ["3" "Driver is quarantined" "SKIP" "driver is quarantined"]
- ["3(a)" "...and has break-quarantine-X label" "RUN" "anti-quarantine label present"]
- ["4" "On master or release branch" "RUN" "master/release branch"]
- ["5" "Cloud driver + ci:all-cloud-drivers" "RUN" "ci:all-cloud-drivers label"]
- ["6"
+ ["3" "ci:run-all-drivers or ci:run-DRIVER" "RUN" "ci:run-all-drivers or ci:run-DRIVER label"]
+ ["4" "Driver is quarantined" "SKIP" "driver is quarantined"]
+ ["4(a)" "...and has break-quarantine-X label" "RUN" "anti-quarantine label present"]
+ ["5" "On master or release branch" "RUN" "master/release branch"]
+ ["6" "Cloud driver + ci:all-cloud-drivers" "RUN" "ci:all-cloud-drivers label"]
+ ["7"
"Cloud driver + its files changed"
"RUN"
(str "any of " (pr-str driver-triggers) " affected by cloud-driver module changes")]
- ["7" "Cloud driver + query-processor updated" "RUN" "query-processor module updated"]
- ["8" "Cloud driver + driver deps affected" "RUN" "driver module affected by shared code changes"]
- ["9" "Cloud driver, no relevant changes" "SKIP" "no relevant changes for cloud driver"]
- ["10"
+ ["8" "Cloud driver + query-processor updated" "RUN" "query-processor module updated"]
+ ["9" "Cloud driver + driver deps affected" "RUN" "driver module affected by shared code changes"]
+ ["10" "Cloud driver, no relevant changes" "SKIP" "no relevant changes for cloud driver"]
+ ["11"
(str (pr-str driver-triggers) " module deps affected")
"RUN"
(str "any of " (pr-str driver-triggers) " affected by module changes")]
- ["11" "Self-hosted driver, not affected" "SKIP" "driver module not affected"]])))))
+ ["12" "Self-hosted driver, not affected" "SKIP" "driver module not affected"]]))
+ "\n\n## Labels of Interest\n\n"
+ "These PR labels influence driver test decisions:\n\n"
+ " ci:run-all-drivers Force-run ALL drivers (overrides quarantine)\n"
+ " ci:run-DRIVER Force-run a specific driver, e.g. ci:run-mysql-mariadb (overrides quarantine)\n"
+ " ci:all-cloud-drivers Run all cloud drivers (athena, bigquery, databricks, redshift, snowflake)\n"
+ " break-quarantine-DRIVER Un-quarantine a specific driver, e.g. break-quarantine-mysql\n"
+ "\n## All Drivers\n\n"
+ (str/join ", " (map name (sort mage.modules/all-drivers)))
+ "\n")))
:options [[nil "--git-ref REF" "The git ref to compute changed modules against"]
[nil "--is-master-or-release MASTER_OR_RELEASE"]
[nil "--pr-labels PR_LABELS" "Comma delimited labels on the pr"]
diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.clj
index e16e9848a84..4cb6435a99b 100644
--- a/mage/src/mage/modules.clj
+++ b/mage/src/mage/modules.clj
@@ -327,6 +327,9 @@
(defn break-quarantine-label [driver]
(str "break-quarantine-" (name driver)))
+(defn run-driver-label [driver]
+ (str "ci:run-" (name driver)))
+
(defn- driver-decision
"Determine if a driver should run and why.
@@ -356,7 +359,15 @@
{:should-run true
:reason "H2/Postgres always run"}
- ;; Priority 3: Quarantined drivers (respected even on master/release)
+ ;; Priority 3: ci:run-all-drivers or ci:run-<driver> label
+ (or (contains? pr-labels "ci:run-all-drivers")
+ (contains? pr-labels (run-driver-label driver)))
+ {:should-run true
+ :reason (if (contains? pr-labels "ci:run-all-drivers")
+ "ci:run-all-drivers label"
+ (str (run-driver-label driver) " label"))}
+
+ ;; Priority 4: Quarantined drivers (respected even on master/release)
(contains? quarantined-drivers driver)
(do
(when verbose?
@@ -367,46 +378,46 @@
{:should-run false
:reason "driver is quarantined"}))
- ;; Priority 4: Master/release branch - all (non-quarantined) drivers run
+ ;; Priority 5: Master/release branch - all (non-quarantined) drivers run
is-master-or-release
{:should-run true
:reason "master/release branch"}
- ;; Priority 5: Cloud driver + ci:all-cloud-drivers label
+ ;; Priority 6: Cloud driver + ci:all-cloud-drivers label
(and (contains? cloud-drivers driver)
(contains? pr-labels "ci:all-cloud-drivers"))
{:should-run true
:reason "ci:all-cloud-drivers label"}
- ;; Priority 6: Cloud driver + its files changed
+ ;; Priority 7: Cloud driver + its files changed
(and (contains? cloud-drivers driver)
(contains? particular-driver-changed? driver))
{:should-run true
:reason (str "driver files changed (modules/drivers/" (name driver) "/**)")}
- ;; Priority 7: Cloud driver + module triggering cloud dbs updated → run it
+ ;; Priority 8: Cloud driver + module triggering cloud dbs updated → run it
(and (contains? cloud-drivers driver)
(seq (set/intersection updated modules-triggering-cloud-drivers)))
{:should-run true
:reason "Module updated which explicitly triggers cloud drivers"}
- ;; Priority 8: Cloud driver + driver deps affected (e.g., deps.edn changed)
+ ;; Priority 9: Cloud driver + driver deps affected (e.g., deps.edn changed)
(and (contains? cloud-drivers driver)
driver-deps-affected?)
{:should-run true
:reason "driver module affected by shared code changes"}
- ;; Priority 9: Cloud driver, no relevant changes → skip
+ ;; Priority 10: Cloud driver, no relevant changes → skip
(contains? cloud-drivers driver)
{:should-run false
:reason "no relevant changes for cloud driver"}
- ;; Priority 10: Driver deps affected by shared code changes
+ ;; Priority 11: Driver deps affected by shared code changes
driver-deps-affected?
{:should-run true
:reason "driver module affected by shared code changes"}
- ;; Priority 11: Self-hosted driver, not affected
+ ;; Priority 12: Self-hosted driver, not affected
:else
{:should-run false
:reason "driver module not affected"}))
diff --git a/mage/test/mage/modules_test.clj b/mage/test/mage/modules_test.clj
index 1ec008ab5da..4010940cec3 100644
--- a/mage/test/mage/modules_test.clj
+++ b/mage/test/mage/modules_test.clj
@@ -19,7 +19,7 @@
opts))
;;; =============================================================================
-;;; Priority 3: Quarantine (respected even on master/release)
+;;; Priority 4: Quarantine (respected even on master/release)
;;; =============================================================================
(deftest quarantined-driver-skips
@@ -125,12 +125,67 @@
(is (= "workflow skip (no backend changes)" (:reason result)))))))
;;; =============================================================================
-;;; Priority 4: Master/release branch
+;;; Priority 3: ci:run-all-drivers / ci:run-<driver> labels
+;;; =============================================================================
+
+(deftest ci-run-all-drivers-forces-run
+ (testing "ci:run-all-drivers forces any driver to run"
+ (doseq [driver [:mysql :mongo :athena :bigquery :snowflake]]
+ (let [result (mage.modules/driver-decision driver
+ (make-ctx {:pr-labels #{"ci:run-all-drivers"}})
+ false ; not affected
+ #{} ; quarantined
+ #{})] ; updated
+ (is (true? (:should-run result))
+ (str driver " should run with ci:run-all-drivers"))
+ (is (= "ci:run-all-drivers label" (:reason result)))))))
+
+(deftest ci-run-specific-driver-forces-run
+ (testing "ci:run-<driver> forces that specific driver to run"
+ (let [result (mage.modules/driver-decision :mysql
+ (make-ctx {:pr-labels #{"ci:run-mysql"}})
+ false
+ #{} ; quarantined
+ #{})] ; updated
+ (is (true? (:should-run result)))
+ (is (= "ci:run-mysql label" (:reason result))))))
+
+(deftest ci-run-specific-driver-does-not-force-other-drivers
+ (testing "ci:run-<driver> for a different driver does NOT force the current driver"
+ (let [result (mage.modules/driver-decision :mongo
+ (make-ctx {:pr-labels #{"ci:run-mysql"}})
+ false
+ #{} ; quarantined
+ #{})] ; updated
+ (is (false? (:should-run result))))))
+
+(deftest ci-run-all-drivers-overrides-quarantine
+ (testing "ci:run-all-drivers overrides quarantine"
+ (let [result (mage.modules/driver-decision :mysql
+ (make-ctx {:pr-labels #{"ci:run-all-drivers"}})
+ false
+ #{:mysql} ; quarantined
+ #{})] ; updated
+ (is (true? (:should-run result)))
+ (is (= "ci:run-all-drivers label" (:reason result))))))
+
+(deftest ci-run-specific-driver-overrides-quarantine
+ (testing "ci:run-<driver> overrides quarantine"
+ (let [result (mage.modules/driver-decision :mysql
+ (make-ctx {:pr-labels #{"ci:run-mysql"}})
+ false
+ #{:mysql} ; quarantined
+ #{})] ; updated
+ (is (true? (:should-run result)))
+ (is (= "ci:run-mysql label" (:reason result))))))
+
+;;; =============================================================================
+;;; Priority 5: Master/release branch
;;; =============================================================================
(deftest master-branch-runs-all-drivers
(testing "All drivers run on master/release branch"
- ;; H2/Postgres hit priority 2 first, others hit priority 4
+ ;; H2/Postgres hit priority 2 first, others hit priority 5
(doseq [driver [:mysql :mongo :athena :bigquery :snowflake]]
(let [result (mage.modules/driver-decision driver
(make-ctx {:is-master-or-release true})
@@ -142,12 +197,12 @@
(is (= "master/release branch" (:reason result)))))))
;;; =============================================================================
-;;; Priority 10: Driver deps affected (self-hosted only)
+;;; Priority 11: Driver deps affected (self-hosted only)
;;; =============================================================================
(deftest driver-deps-affected-runs-self-hosted-drivers
(testing "Self-hosted drivers run when driver module is affected"
- ;; H2/Postgres hit priority 2 first, others hit priority 10
+ ;; H2/Postgres hit priority 2 first, others hit priority 11
(doseq [driver [:mysql :mongo :oracle :sqlserver]]
(let [result (mage.modules/driver-decision driver
(make-ctx {})
@@ -159,7 +214,7 @@
(is (= "driver module affected by shared code changes" (:reason result)))))))
;;; =============================================================================
-;;; Priority 5-9: Cloud driver special rules
+;;; Priority 6-10: Cloud driver special rules
;;; =============================================================================
(deftest cloud-driver-with-label-runs
@@ -224,7 +279,7 @@
(is (= "no relevant changes for cloud driver" (:reason result)))))))
;;; =============================================================================
-;;; Priority 11: Self-hosted drivers
+;;; Priority 12: Self-hosted drivers
;;; =============================================================================
(deftest self-hosted-driver-not-affected-skips