Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix transforms module to trigger driver tests + promote pg (#68432)

bryan76fcb4d57346
diff --git a/bb.edn b/bb.edn
index e0baa9b0dad..c24929e34ec 100644
--- a/bb.edn
+++ b/bb.edn
@@ -307,32 +307,38 @@
:examples [["./bin/mage -driver-decisions --git-ref=master --is-master-or-release=false --pr-labels= --skip=false" "Determine driver decisions for a PR"]
["./bin/mage -driver-decisions --pr-labels=ci:all-cloud-drivers" "Run all cloud drivers via label"]
["./bin/mage -driver-decisions --github-output-only" "Output only GITHUB_OUTPUT format (for CI)"]]
+ :requires [[clojure.string :as str]
+ [table.core :as table]
+ [mage.modules]]
:usage-fn
- (fn [_]
- (->>
- ["# How we decide to run a driver's tests. For each driver, we check these conditions in order (first match wins):"
- " |----------|-------------------------------------|--------|-------------------------------------------|"
- " | Priority | Condition | Result | Reason |"
- " |----------|-------------------------------------|--------|-------------------------------------------|"
- " | 1 | Global skip (no backend changes) | SKIP | workflow skip (no backend changes) |"
- " | 2 | Driver is :h2 | RUN | H2 always runs |"
- " | 3 | Driver is quarantined | SKIP | driver is quarantined |"
- " | 3(a) | ...and has break-quarantine-X label | RUN | anti-quarantine label present |"
- " | 4 | On master or release branch | RUN | master/release branch |"
- " | 5 | Cloud driver + ci:all-cloud-drivers | RUN | ci:all-cloud-drivers label |"
- " | 6 | Cloud driver + its files changed | RUN | driver files changed |"
- " | 7 | Cloud driver, no relevant changes | SKIP | no relevant changes for cloud driver |"
- " | 8 | Driver deps affected by changes | RUN | driver module affected by shared code |"
- " | 9 | Self-hosted driver, not affected | SKIP | driver module not affected |"
- " |----------|-------------------------------------|--------|-------------------------------------------|"
- ""] (str/join "\n")))
+ (fn [_current-task]
+ (let [driver-triggers (vec (sort mage.modules/default-modules-which-trigger-drivers))]
+ (str
+ "# How we decide to run a driver's tests. For each driver, we check these conditions in order (first match wins):\n"
+ (with-out-str
+ (table/table [["Priority" "Condition" "Result" "Reason"]
+ ["1" "Global skip (no backend changes)" "SKIP" "workflow skip (no backend changes)"]
+ ["2" "Driver is :h2 or :postgres" "RUN" "H2/Postgres always run"]
+ ["3" "Driver is quarantined" "SKIP" "driver is quarantined"]
+ ["3(a)" "...and has break-quarantine-X label" "RUN" "anti-quarantine label present"]
+ ["4" "On master or release branch" "RUN" "master/release branch"]
+ ["5" "Cloud driver + ci:all-cloud-drivers" "RUN" "ci:all-cloud-drivers label"]
+ ["6"
+ "Cloud driver + its files changed"
+ "RUN"
+ (str "any of " (pr-str driver-triggers) " affected by cloud-driver module changes")]
+ ["7" "Cloud driver, no relevant changes" "SKIP" "no relevant changes for cloud driver"]
+ ["8"
+ (str (pr-str driver-triggers) " module deps affected")
+ "RUN"
+ (str "any of " (pr-str driver-triggers) " affected by module changes")]
+ ["9" "Self-hosted driver, not affected" "SKIP" "driver module not affected"]])))))
:options [[nil "--git-ref REF" "The git ref to compute changed modules against"]
[nil "--is-master-or-release MASTER_OR_RELEASE"]
[nil "--pr-labels PR_LABELS" "Comma delimited labels on the pr"]
[nil "--skip SKIP"]
[nil "--github-output-only" "Only output GITHUB_OUTPUT format (no human-readable output)"]]
- :requires [[clojure.string :as str]
- [mage.modules]]
+
:task (task! (mage.modules/-main parsed))}
-install-merge-drivers
diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.clj
index b98cd328736..61210032ea1 100644
--- a/mage/src/mage/modules.clj
+++ b/mage/src/mage/modules.clj
@@ -12,6 +12,10 @@
(def ^:dynamic ^:private *github-output-only?* false)
+(def default-modules-which-trigger-drivers
+ "Modules that, when affected by changes, should trigger driver tests."
+ ['driver 'enterprise/transforms])
+
;;; TODO (Cam 2025-11-07) changes to test files should only cause us to run tests for that module as well, not
;;; everything that depends on that module directly or indirectly in `src`
(defn- file->module [filename]
@@ -129,11 +133,13 @@
view-log})
(defn- affected-modules
- "Set of modules that are direct or indirect dependents of `modules`, and thus are affected by changes to them."
+ "Set of modules that are direct or indirect dependents of `modules`, and thus are affected by changes to them.
+ Includes the changed modules themselves (a module is always affected by its own changes)."
[deps modules]
- (into (sorted-set)
- (mapcat (partial indirect-dependents deps))
- modules))
+ (let [sorted-modules (into (sorted-set) modules)]
+ (into sorted-modules
+ (mapcat (partial indirect-dependents deps))
+ modules)))
(defn- unaffected-modules
"Return the set of modules that are unaffected "
@@ -201,13 +207,17 @@
filename)))
files))
-(defn- driver-deps-affected?
- "Returns true if the driver module is affected by the changed modules."
+(defn driver-deps-affected?
+ "Returns true if any of `trigger-modules` are affected by the changed modules.
+ 1-arity and 2-arity use [[default-modules-which-trigger-drivers]] for backwards compatibility."
([modules]
(driver-deps-affected? (dependencies) modules))
([deps modules]
+ (driver-deps-affected? deps modules default-modules-which-trigger-drivers))
+ ([deps modules trigger-modules]
(let [unaffected (unaffected-modules deps (remove driver-affecting-overrides modules))]
- (not (contains? unaffected 'driver)))))
+ (boolean
+ (some #(not (contains? unaffected %)) trigger-modules)))))
(defn cli-can-skip-driver-tests
"Exits with zero status code if we can skip driver tests, nonzero if we cannot.
@@ -343,10 +353,10 @@
{:should-run false
:reason "workflow skip (no backend changes)"}
- ;; Priority 2: H2 always runs when backend tests run
- (= driver :h2)
+ ;; Priority 2: H2 and Postgres always run when backend tests run
+ (#{:h2 :postgres} driver)
{:should-run true
- :reason "H2 always runs"}
+ :reason "H2/Postgres always run"}
;; Priority 3: Quarantined drivers (respected even on master/release)
(contains? quarantined-drivers driver)
diff --git a/mage/test/mage/modules_test.clj b/mage/test/mage/modules_test.clj
index f964fac7c0c..fbafd0a1e39 100644
--- a/mage/test/mage/modules_test.clj
+++ b/mage/test/mage/modules_test.clj
@@ -5,12 +5,6 @@
[clojure.test :refer [deftest is testing]]
[mage.modules]))
-;; Access private functions for testing via @#'ns/var (works in babashka)
-(defn driver-decision [& args]
- (apply @(resolve 'mage.modules/driver-decision) args))
-
-(def cloud-drivers @(resolve 'mage.modules/cloud-drivers))
-
;; Referenced by core_test.clj to ensure namespace is loaded
(def keep-me :loaded)
@@ -30,28 +24,28 @@
(deftest quarantined-driver-skips
(testing "Quarantined driver is skipped"
- (let [result (driver-decision :postgres
- (make-ctx {})
- false ; driver-module-affected?
- #{:postgres})] ; quarantined
+ (let [result (mage.modules/driver-decision :mysql
+ (make-ctx {})
+ false ; driver-module-affected?
+ #{:mysql})] ; quarantined
(is (false? (:should-run result)))
(is (= "driver is quarantined" (:reason result))))))
(deftest quarantined-driver-with-break-label-runs
(testing "Quarantined driver runs when break-quarantine label is present"
- (let [result (driver-decision :postgres
- (make-ctx {:pr-labels #{"break-quarantine-postgres"}})
- false
- #{:postgres})]
+ (let [result (mage.modules/driver-decision :mysql
+ (make-ctx {:pr-labels #{"break-quarantine-mysql"}})
+ false
+ #{:mysql})]
(is (true? (:should-run result)))
(is (re-find #"anti-quarantine label" (:reason result))))))
(deftest quarantine-respected-on-master
(testing "Quarantined driver is skipped even on master/release"
- (let [result (driver-decision :postgres
- (make-ctx {:is-master-or-release true})
- true ; driver-deps-affected?
- #{:postgres})] ; quarantined
+ (let [result (mage.modules/driver-decision :mysql
+ (make-ctx {:is-master-or-release true})
+ true ; driver-deps-affected?
+ #{:mysql})] ; quarantined
(is (false? (:should-run result)))
(is (= "driver is quarantined" (:reason result))))))
@@ -61,45 +55,49 @@
(deftest global-skip-skips-all-drivers
(testing "Global skip (no backend changes) skips all drivers"
- (doseq [driver [:postgres :mysql :mongo :athena :bigquery]]
- (let [result (driver-decision driver
- (make-ctx {:skip true})
- true ; even if affected
- #{})]
+ (doseq [driver [:h2 :postgres :mysql :mongo :athena :bigquery]]
+ (let [result (mage.modules/driver-decision driver
+ (make-ctx {:skip true})
+ true ; even if affected
+ #{})]
(is (false? (:should-run result))
(str driver " should be skipped"))
(is (= "workflow skip (no backend changes)" (:reason result)))))))
(deftest global-skip-beats-quarantine
(testing "Global skip takes priority over quarantine"
- (let [result (driver-decision :postgres
- (make-ctx {:skip true})
- false
- #{:postgres})]
+ (let [result (mage.modules/driver-decision :mysql
+ (make-ctx {:skip true})
+ false
+ #{:mysql})]
(is (false? (:should-run result)))
(is (= "workflow skip (no backend changes)" (:reason result))))))
;;; =============================================================================
-;;; Priority 2: H2 always runs
+;;; Priority 2: H2 and Postgres always run
;;; =============================================================================
-(deftest h2-always-runs
- (testing "H2 always runs when not globally skipped"
- (let [result (driver-decision :h2
- (make-ctx {:is-master-or-release false})
- false ; driver module not affected
- #{})]
- (is (true? (:should-run result)))
- (is (= "H2 always runs" (:reason result))))))
-
-(deftest h2-skipped-on-global-skip
- (testing "H2 is skipped when global skip is true"
- (let [result (driver-decision :h2
- (make-ctx {:skip true})
- false
- #{})]
- (is (false? (:should-run result)))
- (is (= "workflow skip (no backend changes)" (:reason result))))))
+(deftest h2-and-postgres-always-run
+ (testing "H2 and Postgres always run when not globally skipped"
+ (doseq [driver [:h2 :postgres]]
+ (let [result (mage.modules/driver-decision driver
+ (make-ctx {:is-master-or-release false})
+ false ; driver module not affected
+ #{})]
+ (is (true? (:should-run result))
+ (str driver " should always run"))
+ (is (= "H2/Postgres always run" (:reason result)))))))
+
+(deftest h2-and-postgres-skipped-on-global-skip
+ (testing "H2 and Postgres are skipped when global skip is true"
+ (doseq [driver [:h2 :postgres]]
+ (let [result (mage.modules/driver-decision driver
+ (make-ctx {:skip true})
+ false
+ #{})]
+ (is (false? (:should-run result))
+ (str driver " should be skipped on global skip"))
+ (is (= "workflow skip (no backend changes)" (:reason result)))))))
;;; =============================================================================
;;; Priority 4: Master/release branch
@@ -107,11 +105,12 @@
(deftest master-branch-runs-all-drivers
(testing "All drivers run on master/release branch"
- (doseq [driver [:postgres :mysql :mongo :athena :bigquery :snowflake]]
- (let [result (driver-decision driver
- (make-ctx {:is-master-or-release true})
- false ; even if not affected
- #{})]
+ ;; H2/Postgres hit priority 2 first, others hit priority 4
+ (doseq [driver [:mysql :mongo :athena :bigquery :snowflake]]
+ (let [result (mage.modules/driver-decision driver
+ (make-ctx {:is-master-or-release true})
+ false ; even if not affected
+ #{})]
(is (true? (:should-run result))
(str driver " should run on master"))
(is (= "master/release branch" (:reason result)))))))
@@ -122,11 +121,12 @@
(deftest driver-deps-affected-runs-self-hosted-drivers
(testing "Self-hosted drivers run when driver module is affected"
- (doseq [driver [:postgres :mysql :mongo :oracle :sqlserver]]
- (let [result (driver-decision driver
- (make-ctx {})
- true ; driver-deps-affected
- #{})]
+ ;; H2/Postgres hit priority 2 first, others hit priority 8
+ (doseq [driver [:mysql :mongo :oracle :sqlserver]]
+ (let [result (mage.modules/driver-decision driver
+ (make-ctx {})
+ true ; driver-deps-affected
+ #{})]
(is (true? (:should-run result))
(str driver " should run when driver module affected"))
(is (= "driver module affected by shared code changes" (:reason result)))))))
@@ -138,30 +138,30 @@
(deftest cloud-driver-with-label-runs
(testing "Cloud driver runs with ci:all-cloud-drivers label"
(doseq [driver [:athena :bigquery :databricks :redshift :snowflake]]
- (let [result (driver-decision driver
- (make-ctx {:pr-labels #{"ci:all-cloud-drivers"}})
- false ; not affected
- #{})]
+ (let [result (mage.modules/driver-decision driver
+ (make-ctx {:pr-labels #{"ci:all-cloud-drivers"}})
+ false ; not affected
+ #{})]
(is (true? (:should-run result))
(str driver " should run with label"))
(is (= "ci:all-cloud-drivers label" (:reason result)))))))
(deftest cloud-driver-with-file-changes-runs
(testing "Cloud driver runs when its files changed"
- (let [result (driver-decision :athena
- (make-ctx {:particular-driver-changed? #{:athena}})
- false
- #{})]
+ (let [result (mage.modules/driver-decision :athena
+ (make-ctx {:particular-driver-changed? #{:athena}})
+ false
+ #{})]
(is (true? (:should-run result)))
(is (re-find #"driver files changed" (:reason result))))))
(deftest cloud-driver-without-changes-skips
(testing "Cloud driver skips when no relevant changes"
(doseq [driver [:athena :bigquery :databricks :redshift :snowflake]]
- (let [result (driver-decision driver
- (make-ctx {})
- false ; not affected
- #{})]
+ (let [result (mage.modules/driver-decision driver
+ (make-ctx {})
+ false ; not affected
+ #{})]
(is (false? (:should-run result))
(str driver " should skip without changes"))
(is (= "no relevant changes for cloud driver" (:reason result)))))))
@@ -172,11 +172,12 @@
(deftest self-hosted-driver-not-affected-skips
(testing "Self-hosted driver skips when driver module not affected"
- (doseq [driver [:postgres :mysql :mongo :oracle :sqlserver]]
- (let [result (driver-decision driver
- (make-ctx {})
- false ; not affected
- #{})]
+ ;; H2/Postgres always run (priority 2), so test other self-hosted drivers
+ (doseq [driver [:mysql :mongo :oracle :sqlserver]]
+ (let [result (mage.modules/driver-decision driver
+ (make-ctx {})
+ false ; not affected
+ #{})]
(is (false? (:should-run result))
(str driver " should skip when not affected"))
(is (= "driver module not affected" (:reason result)))))))
@@ -188,4 +189,48 @@
(deftest cloud-drivers-are-correct
(testing "Cloud drivers set matches expected"
(is (= #{:athena :bigquery :databricks :redshift :snowflake}
- cloud-drivers))))
+ mage.modules/cloud-drivers))))
+
+;;; =============================================================================
+;;; Two roots trigger driver tests: driver and enterprise/transforms
+;;; =============================================================================
+
+(deftest transforms-triggers-driver-tests
+ (testing "enterprise/transforms triggers driver tests (it's a root)"
+ (is (true? (mage.modules/driver-deps-affected? ['enterprise/transforms])))))
+
+(deftest driver-triggers-driver-tests
+ (testing "driver triggers driver tests (it's a root)"
+ (is (true? (mage.modules/driver-deps-affected? ['driver])))))
+
+(deftest all-modules-triggers-themselves-test
+ (let [deps (mage.modules/dependencies)]
+ (doseq [a-module (keys (mage.modules/dependencies))]
+ (is (contains? (mage.modules/affected-modules deps [a-module]) a-module)
+ (str "The " a-module " module should trigger itself")))))
+
+;;; =============================================================================
+;;; Regression test: module graph should not become more connected
+;;; =============================================================================
+
+(defn modules-affecting-drivers []
+ (let [deps (mage.modules/dependencies)
+ all (keys deps)]
+ (filter #(mage.modules/driver-deps-affected? [%]) all)))
+
+(deftest module-graph-may-not-become-more-connected
+ (testing "The number of modules that trigger driver tests should not increase without explicit approval.
+ If this test fails, you've likely connected a module to driver that shouldn't trigger driver tests.
+ Add it to driver-affecting-overrides if it shouldn't trigger driver tests."
+ (let [modules-triggering-drivers (modules-affecting-drivers)
+ ;; This count was 33 as of 2026-01-20. Update this number ONLY if you
+ ;; intentionally want more modules to trigger driver tests.
+ max-allowed-count 33]
+ (is (<= (count modules-triggering-drivers) max-allowed-count)
+ (format "Too many modules trigger driver tests! Expected <= %d, got %d.
+ Modules triggering driver tests: %s
+ If this is intentional, update max-allowed-count.
+ Otherwise, add the new module(s) to driver-affecting-overrides."
+ max-allowed-count
+ (count modules-triggering-drivers)
+ (pr-str (sort modules-triggering-drivers)))))))