Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add MAGE module command; use module system for determining if we can skip driver tests (#65631)

Cam Saul52eeb00a2eff
diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.yml
index 78179cb7690..08cd86036f8 100644
--- a/.github/workflows/drivers.yml
+++ b/.github/workflows/drivers.yml
@@ -17,7 +17,7 @@ jobs:
determine-driver-skips:
runs-on: ubuntu-latest
outputs:
- skip: ${{ steps.check_team_and_branch.outputs.skip }}
+ skip: ${{ steps.check-branch-and-files-changed.outputs.skip }}
steps:
- name: Checkout repository
uses: actions/checkout@v4
@@ -25,49 +25,46 @@ jobs:
if: github.event_name == 'pull_request' && github.event.pull_request.base.ref != ''
run: |
# Fetch the base branch to ensure its history is available for diffing
- git fetch origin ${{ github.event.pull_request.base.ref }}:${{ github.event.pull_request.base.ref }}
+ git fetch --depth=1 origin ${{ github.event.pull_request.base.ref }}:${{ github.event.pull_request.base.ref }}
# Only run this if it's a pull request and the base ref exists
- - name: Check if tier3 drivers should run
- id: check_team_and_branch
+ - name: Get Date
+ id: get-date
+ shell: bash
+ run: |
+ echo "date=$(/bin/date -u "+%Y%m%d")" >> $GITHUB_OUTPUT
+ - name: Restore Babashka cache
+ uses: actions/cache@v4
+ with:
+ path: |
+ ~/.m2
+ ~/.deps.clj
+ restore-keys: |
+ mage-drivers-
+ key: >-
+ mage-drivers-${{ steps.get-date.outputs.date }}
+ - name: Check if driver tests should run
+ id: check-branch-and-files-changed
run: |
IS_MASTER_OR_RELEASE="${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-') }}"
- HAS_TEAM_LABEL="${{ contains(github.event.pull_request.labels.*.name, '.Team/Drivers') || contains(github.event.pull_request.labels.*.name, '.Team/Querying') || contains(github.event.pull_request.labels.*.name, '.Team/SemanticLayer') || contains(github.event.pull_request.labels.*.name, '.Team/DevEx') }}"
-
-
echo "Is master or release: ${IS_MASTER_OR_RELEASE}"
- echo "Has '.Team/Drivers' or '.Team/Querying' or '.Team/SemanticLayer' label: ${HAS_TEAM_LABEL}"
+
if [[ "${IS_MASTER_OR_RELEASE}" == "true" ]]; then
echo "skip=false" >> $GITHUB_OUTPUT
- echo "Merging into master or a release branch. Tier 3 driver tests will run."
- elif [[ "${HAS_TEAM_LABEL}" == "true" ]]; then
- echo "skip=false" >> $GITHUB_OUTPUT
- echo "A relevant team label is present. Tier 3 driver tests will run."
+ echo "Merging into master or a release branch. All driver tests will run."
else
- echo "Checking for changes in relevant folders..."
- CHANGED_FILES=$(git diff --name-only $(git merge-base ${{ github.event.pull_request.base.sha }} ${{ github.sha }}) ${{ github.sha }})
-
- # Initialize an empty array to store folders with changes
- CHANGED_FOLDERS=()
- for folder in "src/metabase/lib" "src/metabase/query_processor" "modules/drivers" "src/metabase/driver"; do
- if echo "$CHANGED_FILES" | grep -q "$folder"; then
- CHANGED_FOLDERS+=("$folder")
- fi
- done
-
- if [[ ${#CHANGED_FOLDERS[@]} -gt 0 ]]; then
- echo "skip=false" >> $GITHUB_OUTPUT
- echo "Changes detected in the following folders: ${CHANGED_FOLDERS[*]}."
- echo "Tier 3 driver tests will run."
+ echo "Checking for changes in relevant modules..."
+ if ./bin/mage can-skip-driver-tests "${{ github.event.pull_request.base.ref }}"; then
+ echo 'skip=true' >> $GITHUB_OUTPUT
else
- echo "skip=true" >> $GITHUB_OUTPUT
- echo "No relevant team label, not merging into master/release, and no changes in relevant folders. Tier 3 driver tests will be skipped."
+ echo 'skip=false' >> $GITHUB_OUTPUT
fi
fi
shell: bash
+
be-tests-athena-ee:
needs: determine-driver-skips
if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}
- runs-on: ubuntu-22.04
+ runs-on: ubuntu-latest
timeout-minutes: 60
env:
CI: 'true'
@@ -104,8 +101,9 @@ jobs:
trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
be-tests-bigquery-cloud-sdk-ee:
- if: ${{ !inputs.skip }}
- runs-on: ubuntu-22.04
+ needs: determine-driver-skips
+ if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}
+ runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
@@ -163,8 +161,9 @@ jobs:
logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}
be-tests-clickhouse-ee:
- if: ${{ !inputs.skip }}
- runs-on: ubuntu-22.04
+ needs: determine-driver-skips
+ if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}
+ runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
@@ -235,7 +234,7 @@ jobs:
be-tests-druid-ee:
needs: determine-driver-skips
if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}
- runs-on: ubuntu-22.04
+ runs-on: ubuntu-latest
timeout-minutes: 60
env:
CI: 'true'
@@ -272,7 +271,7 @@ jobs:
be-tests-druid-jdbc-ee:
needs: determine-driver-skips
if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}
- runs-on: ubuntu-22.04
+ runs-on: ubuntu-latest
timeout-minutes: 60
env:
CI: 'true'
@@ -309,7 +308,7 @@ jobs:
be-tests-databricks-ee:
needs: determine-driver-skips
if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}
- runs-on: ubuntu-22.04
+ runs-on: ubuntu-latest
timeout-minutes: 60
env:
CI: 'true'
@@ -346,7 +345,7 @@ jobs:
be-tests-databricks-multi-catalog-ee:
needs: determine-driver-skips
if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}
- runs-on: ubuntu-22.04
+ runs-on: ubuntu-latest
timeout-minutes: 60
env:
CI: 'true'
@@ -383,7 +382,7 @@ jobs:
be-tests-h2:
if: ${{ !inputs.skip }}
- runs-on: ubuntu-22.04
+ runs-on: ubuntu-latest
timeout-minutes: 60
env:
CI: 'true'
@@ -411,8 +410,9 @@ jobs:
trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
be-tests-mysql-mariadb:
- if: ${{ !inputs.skip }}
- runs-on: ubuntu-22.04
+ needs: determine-driver-skips
+ if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}
+ runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
@@ -547,7 +547,7 @@ jobs:
be-tests-mongo:
needs: determine-driver-skips
if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}
- runs-on: ubuntu-22.04
+ runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
@@ -616,7 +616,7 @@ jobs:
be-tests-mongo-ssl:
needs: determine-driver-skips
if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}
- runs-on: ubuntu-22.04
+ runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
@@ -681,7 +681,7 @@ jobs:
be-tests-mongo-sharded-cluster-ee:
needs: determine-driver-skips
if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}
- runs-on: ubuntu-22.04
+ runs-on: ubuntu-latest
timeout-minutes: 60
env:
CI: 'true'
@@ -715,7 +715,7 @@ jobs:
be-tests-oracle:
needs: determine-driver-skips
if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}
- runs-on: ubuntu-22.04
+ runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
@@ -786,8 +786,9 @@ jobs:
trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
be-tests-postgres:
- if: ${{ !inputs.skip }}
- runs-on: ubuntu-22.04
+ needs: determine-driver-skips
+ if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}
+ runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
@@ -896,7 +897,7 @@ jobs:
be-tests-presto-jdbc-ee:
needs: determine-driver-skips
if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}
- runs-on: ubuntu-22.04
+ runs-on: ubuntu-latest
timeout-minutes: 60
env:
CI: 'true'
@@ -964,8 +965,9 @@ jobs:
trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
be-tests-redshift-ee:
- if: ${{ !inputs.skip }}
- runs-on: ubuntu-22.04
+ needs: determine-driver-skips
+ if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}
+ runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
@@ -1033,7 +1035,7 @@ jobs:
be-tests-snowflake-ee:
if: ${{ !inputs.skip }}
- runs-on: ubuntu-22.04
+ runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
@@ -1043,6 +1045,7 @@ jobs:
test-args: >-
:only-tags [:mb/driver-tests]
:exclude-tags [:mb/transforms-python-test]
+
# DS 2025-09-26 temporarily disabled while we diagnose flakes
# - name: Transforms Python Tests
# test-args: >-
@@ -1102,7 +1105,7 @@ jobs:
be-tests-sparksql-ee:
needs: determine-driver-skips
if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}
- runs-on: ubuntu-22.04
+ runs-on: ubuntu-latest
timeout-minutes: 60
env:
CI: 'true'
@@ -1137,7 +1140,7 @@ jobs:
be-tests-sqlite-ee:
needs: determine-driver-skips
if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}
- runs-on: ubuntu-22.04
+ runs-on: ubuntu-latest
timeout-minutes: 60
env:
CI: 'true'
@@ -1167,7 +1170,7 @@ jobs:
be-tests-sqlserver:
needs: determine-driver-skips
if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}
- runs-on: ubuntu-22.04
+ runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
fail-fast: false
@@ -1239,7 +1242,7 @@ jobs:
# be-tests-starburst-ee:
# needs: determine-driver-skips
# if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}
- # runs-on: ubuntu-22.04
+ # runs-on: ubuntu-latest
# timeout-minutes: 60
# env:
# CI: 'true'
@@ -1277,7 +1280,7 @@ jobs:
login-to-ecr:
needs: determine-driver-skips
if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}
- runs-on: ubuntu-22.04
+ runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
id-token: write
@@ -1307,7 +1310,7 @@ jobs:
be-tests-vertica-ee:
needs: [determine-driver-skips, login-to-ecr]
if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}
- runs-on: ubuntu-22.04
+ runs-on: ubuntu-latest
timeout-minutes: 60
env:
CI: 'true'
diff --git a/bb.edn b/bb.edn
index 7e594e2df92..03341d2e08c 100644
--- a/bb.edn
+++ b/bb.edn
@@ -204,6 +204,20 @@
test)]
(nrepl-eval "mb.hawk.core" (str code) (:port options)))}
+ affected-modules
+ {:doc "Prints information about which backend modules have changes compared to a Git ref e.g. `master`"
+ :examples [["./bin/mage affected-modules master"]
+ ["./bin/mage affected-modules"]]
+ :requires [[mage.modules]]
+ :task (task! (mage.modules/cli-print-affected-modules (:arguments parsed)))}
+
+ can-skip-driver-tests
+ {:doc "Returns zero (success) if we can skip driver tests based on changes related to a Git ref e.g. `master`"
+ :examples [["./bin/mage can-skip-driver-tests master"]
+ ["./bin/mage can-skip-driver-tests"]]
+ :requires [[mage.modules]]
+ :task (task! (mage.modules/cli-can-skip-driver-tests (:arguments parsed)))}
+
ls {:doc "List all mage public and private tasks"
:examples [["./bin/mage ls" "List all tasks"]]
:requires [[mage.util :as u]]
diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.clj
new file mode 100644
index 00000000000..d0430c28054
--- /dev/null
+++ b/mage/src/mage/modules.clj
@@ -0,0 +1,168 @@
+(ns mage.modules
+ (:require
+ [clojure.edn :as edn]
+ [clojure.set :as set]
+ [clojure.string :as str]
+ [mage.util :as u]))
+
+(set! *warn-on-reflection* true)
+
+;;; TODO (Cam 2025-11-07) changes to test files should only cause us to run tests for that module as well, not
+;;; everything that depends on that module directly or indirectly in `src`
+(defn- file->module [filename]
+ (or
+ (when-let [[_match module] (re-matches #"^(?:(?:src)|(?:test))/metabase/([^/]+)/.*$" filename)]
+ (symbol (str/replace module #"_" "-")))
+ (when-let [[_match module] (re-matches #"^enterprise/backend/(?:(?:src)|(?:test))/metabase_enterprise/([^/]+)/.*$" filename)]
+ (symbol "enterprise" (str/replace module #"_" "-")))))
+
+(defn- updated-files->updated-modules [updated-files]
+ (into (sorted-set)
+ (keep file->module)
+ updated-files))
+
+(defn- updated-modules [git-ref]
+ (let [git-ref (or git-ref "master")
+ updated-files (u/updated-files git-ref)]
+ (updated-files->updated-modules updated-files)))
+
+(defn- module->test-directory
+ [module]
+ (case (namespace module)
+ "enterprise" (str "enterprise/backend/test/metabase_enterprise/" (str/replace (name module) #"-" "_"))
+ nil (str "test/metabase/" (str/replace (name module) #"-" "_"))))
+
+(defn- dependencies
+ "Read out the Kondo config for the modules linter; return a map of module => set of modules it directly depends on."
+ []
+ (let [config (-> (with-open [r (java.io.PushbackReader. (java.io.FileReader. ".clj-kondo/config/modules/config.edn"))]
+ (edn/read r))
+ :metabase/modules
+ ;; ignore the config for [[metabase.connection-pool]] which comes from one of our libraries.
+ (dissoc 'connection-pool))]
+ (into (sorted-map)
+ (map (fn [[k config]]
+ [k (:uses config)]))
+ config)))
+
+(defn- direct-dependents
+ "Set of modules that directly depend on `module`."
+ [deps module]
+ (into (sorted-set)
+ (keep (fn [[a-module module-deps]]
+ (when (or (= module-deps :any)
+ (contains? module-deps module))
+ a-module)))
+ deps))
+
+(comment
+ (direct-dependents (dependencies) 'driver))
+
+(defn- indirect-dependents
+ "Set of modules that either directly or indirectly depend on `module`."
+ ([deps module]
+ (indirect-dependents deps module (sorted-set)))
+ ([deps module acc]
+ (let [module-deps (direct-dependents deps module)
+ new-deps (set/difference module-deps acc)
+ acc (into acc new-deps)]
+ (reduce
+ (fn [acc new-dep]
+ (indirect-dependents deps new-dep acc))
+ acc
+ new-deps))))
+
+(defn- affected-modules
+ "Set of modules that are direct or indirect dependents of `modules`, and thus are affected by changes to them."
+ [deps modules]
+ (into (sorted-set)
+ (mapcat (partial indirect-dependents deps))
+ modules))
+
+(defn- unaffected-modules
+ "Return the set of modules that are unaffected "
+ [deps modules]
+ (set/difference
+ (into (sorted-set) (keys deps))
+ (affected-modules deps modules)))
+
+(comment
+ (unaffected-modules (dependencies) '#{enterprise/billing}))
+
+(defn- skip-driver-tests? [deps modules]
+ (let [unaffected (unaffected-modules deps modules)]
+ (contains? unaffected 'driver)))
+
+(defn- print-updated-and-unaffected-modules [deps updated]
+ (let [unaffected (unaffected-modules deps updated)]
+ (println "These modules have changed:" (pr-str updated))
+ (println)
+ (println)
+ (println "These are all the modules are unaffected by these changes:" (pr-str unaffected))
+ (println)
+ (println)
+ (println "(By unaffected, this means these modules do not have a direct or indirect dependency on the modules that have been changed.)")
+ (println)
+ (println)
+ (println "Driver tests" (if (skip-driver-tests? deps updated)
+ "CAN be skipped."
+ "MUST be run."))))
+
+(defn cli-print-affected-modules
+ [[git-ref, :as _command-line-args]]
+ (let [deps (dependencies)
+ updated (updated-modules git-ref)
+ affected (affected-modules deps updated)]
+ (print-updated-and-unaffected-modules deps updated)
+ (println)
+ (println)
+ (println "You can run tests for these modules and all downstream modules as follows:")
+ (println)
+ (println)
+ (printf "clojure -X :dev:ee:ee-dev:test :only '%s'\n" (pr-str (mapv module->test-directory affected)))
+ (flush))
+ (System/exit 0))
+
+(defn- changes-important-file-for-drivers?
+ "Whether we should always run driver tests if we have changes relative to `git-ref` to something important like
+ `deps.edn`."
+ [git-ref]
+ (some (fn [filename]
+ (when (or (str/includes? filename "deps.edn")
+ (str/includes? filename "modules/drivers/"))
+ (printf "Running driver tests because %s was changed\n" (pr-str filename))
+ (flush)
+ filename))
+ (u/updated-files (or git-ref "master"))))
+
+(defn- remove-non-driver-test-namespaces [files]
+ (into []
+ (remove (fn [filename]
+ (when (and (some #(str/includes? filename %)
+ ["test/" "enterprise/backend/test/"])
+ (not (some #(str/includes? filename %)
+ ["query_processor"
+ "driver"])))
+ (printf "Ignorning changes in test namespace %s\n" (pr-str filename))
+ (flush)
+ filename)))
+ files))
+
+(defn cli-can-skip-driver-tests
+ "Exits with zero status code if we can skip driver tests, nonzero if we cannot.
+
+ Invoke this from the CLI with
+
+ ./bin/mage can-skip-driver-tests [git-ref]"
+ [[git-ref, :as _command-line-args]]
+ (let [deps (dependencies)
+ git-ref (or git-ref "master")
+ updated-files (remove-non-driver-test-namespaces (u/updated-files git-ref))
+ updated (updated-files->updated-modules updated-files)
+ skip-tests? (skip-driver-tests? deps updated)]
+ ;; Not strictly necessary, but people looking at CI will appreciate having this extra info.
+ (print-updated-and-unaffected-modules deps updated)
+ (System/exit ^Long (cond
+ (not skip-tests?) 1
+ (changes-important-file-for-drivers? git-ref) 1
+ :else 0))))
diff --git a/src/metabase/glossary/api.clj b/src/metabase/glossary/api.clj
index fb55eff2e7c..985d4cd4337 100644
--- a/src/metabase/glossary/api.clj
+++ b/src/metabase/glossary/api.clj
@@ -1,5 +1,5 @@
(ns metabase.glossary.api
- "API endpoints for managing glossary entries."
+ "REST API endpoints for managing glossary entries."
(:require
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
diff --git a/test/metabase/lib/aggregation_test.cljc b/test/metabase/lib/aggregation_test.cljc
index ad94394b3b4..b1357d8dc6d 100644
--- a/test/metabase/lib/aggregation_test.cljc
+++ b/test/metabase/lib/aggregation_test.cljc
@@ -951,3 +951,5 @@
{:display-name "Max of Quantity: 50 bins"
:name "max"}]
(lib/aggregations-metadata query 1)))))))
+
+;; trivial change to test CI; remove this next time you see it