The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add ci:run-<driver> and ci:run-all-drivers labels to mage driver decisions (#70894)
bryan99548507aa64
diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.ymlindex 12c422738a4..cd81d7d35b7 100644--- a/.github/workflows/drivers.yml+++ b/.github/workflows/drivers.yml@@ -82,6 +82,9 @@ jobs:echo ""# First call: show human-readable output for debugging./bin/mage -driver-decisions "${MAGE_ARGS[@]}"+ echo ""+ echo "Want to force-run a specific driver? Add a ci:run-<driver> or ci:run-all-drivers label to your PR."+ echo "Run './bin/mage -driver-decisions -h' locally for the full list of drivers and labels."# Second call: output only key=value lines for GITHUB_OUTPUT./bin/mage -driver-decisions "${MAGE_ARGS[@]}" --github-output-only >> $GITHUB_OUTPUTshell: bashdiff --git a/bb.edn b/bb.ednindex e09ff7857f7..0fd98f205a7 100644--- a/bb.edn+++ b/bb.edn@@ -329,6 +329,8 @@{:doc "Determine which driver tests should run based on PR context. Outputs decisions for all drivers.":examples [["./bin/mage -driver-decisions --git-ref=master --is-master-or-release=false --pr-labels= --skip=false" "Determine driver decisions for a PR"]["./bin/mage -driver-decisions --pr-labels=ci:all-cloud-drivers" "Run all cloud drivers via label"]+ ["./bin/mage -driver-decisions --pr-labels=ci:run-mysql-mariadb" "Force-run a specific driver via label"]+ ["./bin/mage -driver-decisions --pr-labels=ci:run-all-drivers" "Force-run all drivers via label"]["./bin/mage -driver-decisions --github-output-only" "Output only GITHUB_OUTPUT format (for CI)"]]:requires [[clojure.string :as str][table.core :as table]@@ -342,22 +344,32 @@(table/table [["Priority" "Condition" "Result" "Reason"]["1" "Global skip (no backend changes)" "SKIP" "workflow skip (no backend changes)"]["2" "Driver is :h2 or :postgres" "RUN" "H2/Postgres always run"]- ["3" "Driver is quarantined" "SKIP" "driver is quarantined"]- ["3(a)" "...and has break-quarantine-X label" "RUN" "anti-quarantine label present"]- ["4" "On master or release branch" "RUN" "master/release branch"]- ["5" "Cloud driver + ci:all-cloud-drivers" "RUN" "ci:all-cloud-drivers label"]- ["6"+ ["3" "ci:run-all-drivers or ci:run-DRIVER" "RUN" "ci:run-all-drivers or ci:run-DRIVER label"]+ ["4" "Driver is quarantined" "SKIP" "driver is quarantined"]+ ["4(a)" "...and has break-quarantine-X label" "RUN" "anti-quarantine label present"]+ ["5" "On master or release branch" "RUN" "master/release branch"]+ ["6" "Cloud driver + ci:all-cloud-drivers" "RUN" "ci:all-cloud-drivers label"]+ ["7""Cloud driver + its files changed""RUN"(str "any of " (pr-str driver-triggers) " affected by cloud-driver module changes")]- ["7" "Cloud driver + query-processor updated" "RUN" "query-processor module updated"]- ["8" "Cloud driver + driver deps affected" "RUN" "driver module affected by shared code changes"]- ["9" "Cloud driver, no relevant changes" "SKIP" "no relevant changes for cloud driver"]- ["10"+ ["8" "Cloud driver + query-processor updated" "RUN" "query-processor module updated"]+ ["9" "Cloud driver + driver deps affected" "RUN" "driver module affected by shared code changes"]+ ["10" "Cloud driver, no relevant changes" "SKIP" "no relevant changes for cloud driver"]+ ["11"(str (pr-str driver-triggers) " module deps affected")"RUN"(str "any of " (pr-str driver-triggers) " affected by module changes")]- ["11" "Self-hosted driver, not affected" "SKIP" "driver module not affected"]])))))+ ["12" "Self-hosted driver, not affected" "SKIP" "driver module not affected"]]))+ "\n\n## Labels of Interest\n\n"+ "These PR labels influence driver test decisions:\n\n"+ " ci:run-all-drivers Force-run ALL drivers (overrides quarantine)\n"+ " ci:run-DRIVER Force-run a specific driver, e.g. ci:run-mysql-mariadb (overrides quarantine)\n"+ " ci:all-cloud-drivers Run all cloud drivers (athena, bigquery, databricks, redshift, snowflake)\n"+ " break-quarantine-DRIVER Un-quarantine a specific driver, e.g. break-quarantine-mysql\n"+ "\n## All Drivers\n\n"+ (str/join ", " (map name (sort mage.modules/all-drivers)))+ "\n"))):options [[nil "--git-ref REF" "The git ref to compute changed modules against"][nil "--is-master-or-release MASTER_OR_RELEASE"][nil "--pr-labels PR_LABELS" "Comma delimited labels on the pr"]diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.cljindex e16e9848a84..4cb6435a99b 100644--- a/mage/src/mage/modules.clj+++ b/mage/src/mage/modules.clj@@ -327,6 +327,9 @@(defn break-quarantine-label [driver](str "break-quarantine-" (name driver)))+(defn run-driver-label [driver]+ (str "ci:run-" (name driver)))+(defn- driver-decision"Determine if a driver should run and why.@@ -356,7 +359,15 @@{:should-run true:reason "H2/Postgres always run"}- ;; Priority 3: Quarantined drivers (respected even on master/release)+ ;; Priority 3: ci:run-all-drivers or ci:run-<driver> label+ (or (contains? pr-labels "ci:run-all-drivers")+ (contains? pr-labels (run-driver-label driver)))+ {:should-run true+ :reason (if (contains? pr-labels "ci:run-all-drivers")+ "ci:run-all-drivers label"+ (str (run-driver-label driver) " label"))}++ ;; Priority 4: Quarantined drivers (respected even on master/release)(contains? quarantined-drivers driver)(do(when verbose?@@ -367,46 +378,46 @@{:should-run false:reason "driver is quarantined"}))- ;; Priority 4: Master/release branch - all (non-quarantined) drivers run+ ;; Priority 5: Master/release branch - all (non-quarantined) drivers runis-master-or-release{:should-run true:reason "master/release branch"}- ;; Priority 5: Cloud driver + ci:all-cloud-drivers label+ ;; Priority 6: Cloud driver + ci:all-cloud-drivers label(and (contains? cloud-drivers driver)(contains? pr-labels "ci:all-cloud-drivers")){:should-run true:reason "ci:all-cloud-drivers label"}- ;; Priority 6: Cloud driver + its files changed+ ;; Priority 7: Cloud driver + its files changed(and (contains? cloud-drivers driver)(contains? particular-driver-changed? driver)){:should-run true:reason (str "driver files changed (modules/drivers/" (name driver) "/**)")}- ;; Priority 7: Cloud driver + module triggering cloud dbs updated → run it+ ;; Priority 8: Cloud driver + module triggering cloud dbs updated → run it(and (contains? cloud-drivers driver)(seq (set/intersection updated modules-triggering-cloud-drivers))){:should-run true:reason "Module updated which explicitly triggers cloud drivers"}- ;; Priority 8: Cloud driver + driver deps affected (e.g., deps.edn changed)+ ;; Priority 9: Cloud driver + driver deps affected (e.g., deps.edn changed)(and (contains? cloud-drivers driver)driver-deps-affected?){:should-run true:reason "driver module affected by shared code changes"}- ;; Priority 9: Cloud driver, no relevant changes → skip+ ;; Priority 10: Cloud driver, no relevant changes → skip(contains? cloud-drivers driver){:should-run false:reason "no relevant changes for cloud driver"}- ;; Priority 10: Driver deps affected by shared code changes+ ;; Priority 11: Driver deps affected by shared code changesdriver-deps-affected?{:should-run true:reason "driver module affected by shared code changes"}- ;; Priority 11: Self-hosted driver, not affected+ ;; Priority 12: Self-hosted driver, not affected:else{:should-run false:reason "driver module not affected"}))diff --git a/mage/test/mage/modules_test.clj b/mage/test/mage/modules_test.cljindex 1ec008ab5da..4010940cec3 100644--- a/mage/test/mage/modules_test.clj+++ b/mage/test/mage/modules_test.clj@@ -19,7 +19,7 @@opts));;; =============================================================================-;;; Priority 3: Quarantine (respected even on master/release)+;;; Priority 4: Quarantine (respected even on master/release);;; =============================================================================(deftest quarantined-driver-skips@@ -125,12 +125,67 @@(is (= "workflow skip (no backend changes)" (:reason result)))))));;; =============================================================================-;;; Priority 4: Master/release branch+;;; Priority 3: ci:run-all-drivers / ci:run-<driver> labels+;;; =============================================================================++(deftest ci-run-all-drivers-forces-run+ (testing "ci:run-all-drivers forces any driver to run"+ (doseq [driver [:mysql :mongo :athena :bigquery :snowflake]]+ (let [result (mage.modules/driver-decision driver+ (make-ctx {:pr-labels #{"ci:run-all-drivers"}})+ false ; not affected+ #{} ; quarantined+ #{})] ; updated+ (is (true? (:should-run result))+ (str driver " should run with ci:run-all-drivers"))+ (is (= "ci:run-all-drivers label" (:reason result)))))))++(deftest ci-run-specific-driver-forces-run+ (testing "ci:run-<driver> forces that specific driver to run"+ (let [result (mage.modules/driver-decision :mysql+ (make-ctx {:pr-labels #{"ci:run-mysql"}})+ false+ #{} ; quarantined+ #{})] ; updated+ (is (true? (:should-run result)))+ (is (= "ci:run-mysql label" (:reason result))))))++(deftest ci-run-specific-driver-does-not-force-other-drivers+ (testing "ci:run-<driver> for a different driver does NOT force the current driver"+ (let [result (mage.modules/driver-decision :mongo+ (make-ctx {:pr-labels #{"ci:run-mysql"}})+ false+ #{} ; quarantined+ #{})] ; updated+ (is (false? (:should-run result))))))++(deftest ci-run-all-drivers-overrides-quarantine+ (testing "ci:run-all-drivers overrides quarantine"+ (let [result (mage.modules/driver-decision :mysql+ (make-ctx {:pr-labels #{"ci:run-all-drivers"}})+ false+ #{:mysql} ; quarantined+ #{})] ; updated+ (is (true? (:should-run result)))+ (is (= "ci:run-all-drivers label" (:reason result))))))++(deftest ci-run-specific-driver-overrides-quarantine+ (testing "ci:run-<driver> overrides quarantine"+ (let [result (mage.modules/driver-decision :mysql+ (make-ctx {:pr-labels #{"ci:run-mysql"}})+ false+ #{:mysql} ; quarantined+ #{})] ; updated+ (is (true? (:should-run result)))+ (is (= "ci:run-mysql label" (:reason result))))))++;;; =============================================================================+;;; Priority 5: Master/release branch;;; =============================================================================(deftest master-branch-runs-all-drivers(testing "All drivers run on master/release branch"- ;; H2/Postgres hit priority 2 first, others hit priority 4+ ;; H2/Postgres hit priority 2 first, others hit priority 5(doseq [driver [:mysql :mongo :athena :bigquery :snowflake]](let [result (mage.modules/driver-decision driver(make-ctx {:is-master-or-release true})@@ -142,12 +197,12 @@(is (= "master/release branch" (:reason result)))))));;; =============================================================================-;;; Priority 10: Driver deps affected (self-hosted only)+;;; Priority 11: Driver deps affected (self-hosted only);;; =============================================================================(deftest driver-deps-affected-runs-self-hosted-drivers(testing "Self-hosted drivers run when driver module is affected"- ;; H2/Postgres hit priority 2 first, others hit priority 10+ ;; H2/Postgres hit priority 2 first, others hit priority 11(doseq [driver [:mysql :mongo :oracle :sqlserver]](let [result (mage.modules/driver-decision driver(make-ctx {})@@ -159,7 +214,7 @@(is (= "driver module affected by shared code changes" (:reason result)))))));;; =============================================================================-;;; Priority 5-9: Cloud driver special rules+;;; Priority 6-10: Cloud driver special rules;;; =============================================================================(deftest cloud-driver-with-label-runs@@ -224,7 +279,7 @@(is (= "no relevant changes for cloud driver" (:reason result)))))));;; =============================================================================-;;; Priority 11: Self-hosted drivers+;;; Priority 12: Self-hosted drivers;;; =============================================================================(deftest self-hosted-driver-not-affected-skips