The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix transforms module to trigger driver tests + promote pg (#68432)
bryan76fcb4d57346
diff --git a/bb.edn b/bb.ednindex e0baa9b0dad..c24929e34ec 100644--- a/bb.edn+++ b/bb.edn@@ -307,32 +307,38 @@:examples [["./bin/mage -driver-decisions --git-ref=master --is-master-or-release=false --pr-labels= --skip=false" "Determine driver decisions for a PR"]["./bin/mage -driver-decisions --pr-labels=ci:all-cloud-drivers" "Run all cloud drivers via label"]["./bin/mage -driver-decisions --github-output-only" "Output only GITHUB_OUTPUT format (for CI)"]]+ :requires [[clojure.string :as str]+ [table.core :as table]+ [mage.modules]]:usage-fn- (fn [_]- (->>- ["# How we decide to run a driver's tests. For each driver, we check these conditions in order (first match wins):"- " |----------|-------------------------------------|--------|-------------------------------------------|"- " | Priority | Condition | Result | Reason |"- " |----------|-------------------------------------|--------|-------------------------------------------|"- " | 1 | Global skip (no backend changes) | SKIP | workflow skip (no backend changes) |"- " | 2 | Driver is :h2 | RUN | H2 always runs |"- " | 3 | Driver is quarantined | SKIP | driver is quarantined |"- " | 3(a) | ...and has break-quarantine-X label | RUN | anti-quarantine label present |"- " | 4 | On master or release branch | RUN | master/release branch |"- " | 5 | Cloud driver + ci:all-cloud-drivers | RUN | ci:all-cloud-drivers label |"- " | 6 | Cloud driver + its files changed | RUN | driver files changed |"- " | 7 | Cloud driver, no relevant changes | SKIP | no relevant changes for cloud driver |"- " | 8 | Driver deps affected by changes | RUN | driver module affected by shared code |"- " | 9 | Self-hosted driver, not affected | SKIP | driver module not affected |"- " |----------|-------------------------------------|--------|-------------------------------------------|"- ""] (str/join "\n")))+ (fn [_current-task]+ (let [driver-triggers (vec (sort mage.modules/default-modules-which-trigger-drivers))]+ (str+ "# How we decide to run a driver's tests. For each driver, we check these conditions in order (first match wins):\n"+ (with-out-str+ (table/table [["Priority" "Condition" "Result" "Reason"]+ ["1" "Global skip (no backend changes)" "SKIP" "workflow skip (no backend changes)"]+ ["2" "Driver is :h2 or :postgres" "RUN" "H2/Postgres always run"]+ ["3" "Driver is quarantined" "SKIP" "driver is quarantined"]+ ["3(a)" "...and has break-quarantine-X label" "RUN" "anti-quarantine label present"]+ ["4" "On master or release branch" "RUN" "master/release branch"]+ ["5" "Cloud driver + ci:all-cloud-drivers" "RUN" "ci:all-cloud-drivers label"]+ ["6"+ "Cloud driver + its files changed"+ "RUN"+ (str "any of " (pr-str driver-triggers) " affected by cloud-driver module changes")]+ ["7" "Cloud driver, no relevant changes" "SKIP" "no relevant changes for cloud driver"]+ ["8"+ (str (pr-str driver-triggers) " module deps affected")+ "RUN"+ (str "any of " (pr-str driver-triggers) " affected by module changes")]+ ["9" "Self-hosted driver, not affected" "SKIP" "driver module not affected"]]))))):options [[nil "--git-ref REF" "The git ref to compute changed modules against"][nil "--is-master-or-release MASTER_OR_RELEASE"][nil "--pr-labels PR_LABELS" "Comma delimited labels on the pr"][nil "--skip SKIP"][nil "--github-output-only" "Only output GITHUB_OUTPUT format (no human-readable output)"]]- :requires [[clojure.string :as str]- [mage.modules]]+:task (task! (mage.modules/-main parsed))}-install-merge-driversdiff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.cljindex b98cd328736..61210032ea1 100644--- a/mage/src/mage/modules.clj+++ b/mage/src/mage/modules.clj@@ -12,6 +12,10 @@(def ^:dynamic ^:private *github-output-only?* false)+(def default-modules-which-trigger-drivers+ "Modules that, when affected by changes, should trigger driver tests."+ ['driver 'enterprise/transforms])+;;; TODO (Cam 2025-11-07) changes to test files should only cause us to run tests for that module as well, not;;; everything that depends on that module directly or indirectly in `src`(defn- file->module [filename]@@ -129,11 +133,13 @@view-log})(defn- affected-modules- "Set of modules that are direct or indirect dependents of `modules`, and thus are affected by changes to them."+ "Set of modules that are direct or indirect dependents of `modules`, and thus are affected by changes to them.+ Includes the changed modules themselves (a module is always affected by its own changes)."[deps modules]- (into (sorted-set)- (mapcat (partial indirect-dependents deps))- modules))+ (let [sorted-modules (into (sorted-set) modules)]+ (into sorted-modules+ (mapcat (partial indirect-dependents deps))+ modules)))(defn- unaffected-modules"Return the set of modules that are unaffected "@@ -201,13 +207,17 @@filename)))files))-(defn- driver-deps-affected?- "Returns true if the driver module is affected by the changed modules."+(defn driver-deps-affected?+ "Returns true if any of `trigger-modules` are affected by the changed modules.+ 1-arity and 2-arity use [[default-modules-which-trigger-drivers]] for backwards compatibility."([modules](driver-deps-affected? (dependencies) modules))([deps modules]+ (driver-deps-affected? deps modules default-modules-which-trigger-drivers))+ ([deps modules trigger-modules](let [unaffected (unaffected-modules deps (remove driver-affecting-overrides modules))]- (not (contains? unaffected 'driver)))))+ (boolean+ (some #(not (contains? unaffected %)) trigger-modules)))))(defn cli-can-skip-driver-tests"Exits with zero status code if we can skip driver tests, nonzero if we cannot.@@ -343,10 +353,10 @@{:should-run false:reason "workflow skip (no backend changes)"}- ;; Priority 2: H2 always runs when backend tests run- (= driver :h2)+ ;; Priority 2: H2 and Postgres always run when backend tests run+ (#{:h2 :postgres} driver){:should-run true- :reason "H2 always runs"}+ :reason "H2/Postgres always run"};; Priority 3: Quarantined drivers (respected even on master/release)(contains? quarantined-drivers driver)diff --git a/mage/test/mage/modules_test.clj b/mage/test/mage/modules_test.cljindex f964fac7c0c..fbafd0a1e39 100644--- a/mage/test/mage/modules_test.clj+++ b/mage/test/mage/modules_test.clj@@ -5,12 +5,6 @@[clojure.test :refer [deftest is testing]][mage.modules]))-;; Access private functions for testing via @#'ns/var (works in babashka)-(defn driver-decision [& args]- (apply @(resolve 'mage.modules/driver-decision) args))--(def cloud-drivers @(resolve 'mage.modules/cloud-drivers))-;; Referenced by core_test.clj to ensure namespace is loaded(def keep-me :loaded)@@ -30,28 +24,28 @@(deftest quarantined-driver-skips(testing "Quarantined driver is skipped"- (let [result (driver-decision :postgres- (make-ctx {})- false ; driver-module-affected?- #{:postgres})] ; quarantined+ (let [result (mage.modules/driver-decision :mysql+ (make-ctx {})+ false ; driver-module-affected?+ #{:mysql})] ; quarantined(is (false? (:should-run result)))(is (= "driver is quarantined" (:reason result))))))(deftest quarantined-driver-with-break-label-runs(testing "Quarantined driver runs when break-quarantine label is present"- (let [result (driver-decision :postgres- (make-ctx {:pr-labels #{"break-quarantine-postgres"}})- false- #{:postgres})]+ (let [result (mage.modules/driver-decision :mysql+ (make-ctx {:pr-labels #{"break-quarantine-mysql"}})+ false+ #{:mysql})](is (true? (:should-run result)))(is (re-find #"anti-quarantine label" (:reason result))))))(deftest quarantine-respected-on-master(testing "Quarantined driver is skipped even on master/release"- (let [result (driver-decision :postgres- (make-ctx {:is-master-or-release true})- true ; driver-deps-affected?- #{:postgres})] ; quarantined+ (let [result (mage.modules/driver-decision :mysql+ (make-ctx {:is-master-or-release true})+ true ; driver-deps-affected?+ #{:mysql})] ; quarantined(is (false? (:should-run result)))(is (= "driver is quarantined" (:reason result))))))@@ -61,45 +55,49 @@(deftest global-skip-skips-all-drivers(testing "Global skip (no backend changes) skips all drivers"- (doseq [driver [:postgres :mysql :mongo :athena :bigquery]]- (let [result (driver-decision driver- (make-ctx {:skip true})- true ; even if affected- #{})]+ (doseq [driver [:h2 :postgres :mysql :mongo :athena :bigquery]]+ (let [result (mage.modules/driver-decision driver+ (make-ctx {:skip true})+ true ; even if affected+ #{})](is (false? (:should-run result))(str driver " should be skipped"))(is (= "workflow skip (no backend changes)" (:reason result)))))))(deftest global-skip-beats-quarantine(testing "Global skip takes priority over quarantine"- (let [result (driver-decision :postgres- (make-ctx {:skip true})- false- #{:postgres})]+ (let [result (mage.modules/driver-decision :mysql+ (make-ctx {:skip true})+ false+ #{:mysql})](is (false? (:should-run result)))(is (= "workflow skip (no backend changes)" (:reason result))))));;; =============================================================================-;;; Priority 2: H2 always runs+;;; Priority 2: H2 and Postgres always run;;; =============================================================================-(deftest h2-always-runs- (testing "H2 always runs when not globally skipped"- (let [result (driver-decision :h2- (make-ctx {:is-master-or-release false})- false ; driver module not affected- #{})]- (is (true? (:should-run result)))- (is (= "H2 always runs" (:reason result))))))--(deftest h2-skipped-on-global-skip- (testing "H2 is skipped when global skip is true"- (let [result (driver-decision :h2- (make-ctx {:skip true})- false- #{})]- (is (false? (:should-run result)))- (is (= "workflow skip (no backend changes)" (:reason result))))))+(deftest h2-and-postgres-always-run+ (testing "H2 and Postgres always run when not globally skipped"+ (doseq [driver [:h2 :postgres]]+ (let [result (mage.modules/driver-decision driver+ (make-ctx {:is-master-or-release false})+ false ; driver module not affected+ #{})]+ (is (true? (:should-run result))+ (str driver " should always run"))+ (is (= "H2/Postgres always run" (:reason result)))))))++(deftest h2-and-postgres-skipped-on-global-skip+ (testing "H2 and Postgres are skipped when global skip is true"+ (doseq [driver [:h2 :postgres]]+ (let [result (mage.modules/driver-decision driver+ (make-ctx {:skip true})+ false+ #{})]+ (is (false? (:should-run result))+ (str driver " should be skipped on global skip"))+ (is (= "workflow skip (no backend changes)" (:reason result)))))));;; =============================================================================;;; Priority 4: Master/release branch@@ -107,11 +105,12 @@(deftest master-branch-runs-all-drivers(testing "All drivers run on master/release branch"- (doseq [driver [:postgres :mysql :mongo :athena :bigquery :snowflake]]- (let [result (driver-decision driver- (make-ctx {:is-master-or-release true})- false ; even if not affected- #{})]+ ;; H2/Postgres hit priority 2 first, others hit priority 4+ (doseq [driver [:mysql :mongo :athena :bigquery :snowflake]]+ (let [result (mage.modules/driver-decision driver+ (make-ctx {:is-master-or-release true})+ false ; even if not affected+ #{})](is (true? (:should-run result))(str driver " should run on master"))(is (= "master/release branch" (:reason result)))))))@@ -122,11 +121,12 @@(deftest driver-deps-affected-runs-self-hosted-drivers(testing "Self-hosted drivers run when driver module is affected"- (doseq [driver [:postgres :mysql :mongo :oracle :sqlserver]]- (let [result (driver-decision driver- (make-ctx {})- true ; driver-deps-affected- #{})]+ ;; H2/Postgres hit priority 2 first, others hit priority 8+ (doseq [driver [:mysql :mongo :oracle :sqlserver]]+ (let [result (mage.modules/driver-decision driver+ (make-ctx {})+ true ; driver-deps-affected+ #{})](is (true? (:should-run result))(str driver " should run when driver module affected"))(is (= "driver module affected by shared code changes" (:reason result)))))))@@ -138,30 +138,30 @@(deftest cloud-driver-with-label-runs(testing "Cloud driver runs with ci:all-cloud-drivers label"(doseq [driver [:athena :bigquery :databricks :redshift :snowflake]]- (let [result (driver-decision driver- (make-ctx {:pr-labels #{"ci:all-cloud-drivers"}})- false ; not affected- #{})]+ (let [result (mage.modules/driver-decision driver+ (make-ctx {:pr-labels #{"ci:all-cloud-drivers"}})+ false ; not affected+ #{})](is (true? (:should-run result))(str driver " should run with label"))(is (= "ci:all-cloud-drivers label" (:reason result)))))))(deftest cloud-driver-with-file-changes-runs(testing "Cloud driver runs when its files changed"- (let [result (driver-decision :athena- (make-ctx {:particular-driver-changed? #{:athena}})- false- #{})]+ (let [result (mage.modules/driver-decision :athena+ (make-ctx {:particular-driver-changed? #{:athena}})+ false+ #{})](is (true? (:should-run result)))(is (re-find #"driver files changed" (:reason result))))))(deftest cloud-driver-without-changes-skips(testing "Cloud driver skips when no relevant changes"(doseq [driver [:athena :bigquery :databricks :redshift :snowflake]]- (let [result (driver-decision driver- (make-ctx {})- false ; not affected- #{})]+ (let [result (mage.modules/driver-decision driver+ (make-ctx {})+ false ; not affected+ #{})](is (false? (:should-run result))(str driver " should skip without changes"))(is (= "no relevant changes for cloud driver" (:reason result)))))))@@ -172,11 +172,12 @@(deftest self-hosted-driver-not-affected-skips(testing "Self-hosted driver skips when driver module not affected"- (doseq [driver [:postgres :mysql :mongo :oracle :sqlserver]]- (let [result (driver-decision driver- (make-ctx {})- false ; not affected- #{})]+ ;; H2/Postgres always run (priority 2), so test other self-hosted drivers+ (doseq [driver [:mysql :mongo :oracle :sqlserver]]+ (let [result (mage.modules/driver-decision driver+ (make-ctx {})+ false ; not affected+ #{})](is (false? (:should-run result))(str driver " should skip when not affected"))(is (= "driver module not affected" (:reason result)))))))@@ -188,4 +189,48 @@(deftest cloud-drivers-are-correct(testing "Cloud drivers set matches expected"(is (= #{:athena :bigquery :databricks :redshift :snowflake}- cloud-drivers))))+ mage.modules/cloud-drivers))))++;;; =============================================================================+;;; Two roots trigger driver tests: driver and enterprise/transforms+;;; =============================================================================++(deftest transforms-triggers-driver-tests+ (testing "enterprise/transforms triggers driver tests (it's a root)"+ (is (true? (mage.modules/driver-deps-affected? ['enterprise/transforms])))))++(deftest driver-triggers-driver-tests+ (testing "driver triggers driver tests (it's a root)"+ (is (true? (mage.modules/driver-deps-affected? ['driver])))))++(deftest all-modules-triggers-themselves-test+ (let [deps (mage.modules/dependencies)]+ (doseq [a-module (keys (mage.modules/dependencies))]+ (is (contains? (mage.modules/affected-modules deps [a-module]) a-module)+ (str "The " a-module " module should trigger itself")))))++;;; =============================================================================+;;; Regression test: module graph should not become more connected+;;; =============================================================================++(defn modules-affecting-drivers []+ (let [deps (mage.modules/dependencies)+ all (keys deps)]+ (filter #(mage.modules/driver-deps-affected? [%]) all)))++(deftest module-graph-may-not-become-more-connected+ (testing "The number of modules that trigger driver tests should not increase without explicit approval.+ If this test fails, you've likely connected a module to driver that shouldn't trigger driver tests.+ Add it to driver-affecting-overrides if it shouldn't trigger driver tests."+ (let [modules-triggering-drivers (modules-affecting-drivers)+ ;; This count was 33 as of 2026-01-20. Update this number ONLY if you+ ;; intentionally want more modules to trigger driver tests.+ max-allowed-count 33]+ (is (<= (count modules-triggering-drivers) max-allowed-count)+ (format "Too many modules trigger driver tests! Expected <= %d, got %d.+ Modules triggering driver tests: %s+ If this is intentional, update max-allowed-count.+ Otherwise, add the new module(s) to driver-affecting-overrides."+ max-allowed-count+ (count modules-triggering-drivers)+ (pr-str (sort modules-triggering-drivers)))))))