The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add MAGE module command; use module system for determining if we can skip driver tests (#65631)
Cam Saul52eeb00a2eff
diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.ymlindex 78179cb7690..08cd86036f8 100644--- a/.github/workflows/drivers.yml+++ b/.github/workflows/drivers.yml@@ -17,7 +17,7 @@ jobs:determine-driver-skips:runs-on: ubuntu-latestoutputs:- skip: ${{ steps.check_team_and_branch.outputs.skip }}+ skip: ${{ steps.check-branch-and-files-changed.outputs.skip }}steps:- name: Checkout repositoryuses: actions/checkout@v4@@ -25,49 +25,46 @@ jobs:if: github.event_name == 'pull_request' && github.event.pull_request.base.ref != ''run: |# Fetch the base branch to ensure its history is available for diffing- git fetch origin ${{ github.event.pull_request.base.ref }}:${{ github.event.pull_request.base.ref }}+ git fetch --depth=1 origin ${{ github.event.pull_request.base.ref }}:${{ github.event.pull_request.base.ref }}# Only run this if it's a pull request and the base ref exists- - name: Check if tier3 drivers should run- id: check_team_and_branch+ - name: Get Date+ id: get-date+ shell: bash+ run: |+ echo "date=$(/bin/date -u "+%Y%m%d")" >> $GITHUB_OUTPUT+ - name: Restore Babashka cache+ uses: actions/cache@v4+ with:+ path: |+ ~/.m2+ ~/.deps.clj+ restore-keys: |+ mage-drivers-+ key: >-+ mage-drivers-${{ steps.get-date.outputs.date }}+ - name: Check if driver tests should run+ id: check-branch-and-files-changedrun: |IS_MASTER_OR_RELEASE="${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-') }}"- HAS_TEAM_LABEL="${{ contains(github.event.pull_request.labels.*.name, '.Team/Drivers') || contains(github.event.pull_request.labels.*.name, '.Team/Querying') || contains(github.event.pull_request.labels.*.name, '.Team/SemanticLayer') || contains(github.event.pull_request.labels.*.name, '.Team/DevEx') }}"--echo "Is master or release: ${IS_MASTER_OR_RELEASE}"- echo "Has '.Team/Drivers' or '.Team/Querying' or '.Team/SemanticLayer' label: ${HAS_TEAM_LABEL}"+if [[ "${IS_MASTER_OR_RELEASE}" == "true" ]]; thenecho "skip=false" >> $GITHUB_OUTPUT- echo "Merging into master or a release branch. Tier 3 driver tests will run."- elif [[ "${HAS_TEAM_LABEL}" == "true" ]]; then- echo "skip=false" >> $GITHUB_OUTPUT- echo "A relevant team label is present. Tier 3 driver tests will run."+ echo "Merging into master or a release branch. All driver tests will run."else- echo "Checking for changes in relevant folders..."- CHANGED_FILES=$(git diff --name-only $(git merge-base ${{ github.event.pull_request.base.sha }} ${{ github.sha }}) ${{ github.sha }})-- # Initialize an empty array to store folders with changes- CHANGED_FOLDERS=()- for folder in "src/metabase/lib" "src/metabase/query_processor" "modules/drivers" "src/metabase/driver"; do- if echo "$CHANGED_FILES" | grep -q "$folder"; then- CHANGED_FOLDERS+=("$folder")- fi- done-- if [[ ${#CHANGED_FOLDERS[@]} -gt 0 ]]; then- echo "skip=false" >> $GITHUB_OUTPUT- echo "Changes detected in the following folders: ${CHANGED_FOLDERS[*]}."- echo "Tier 3 driver tests will run."+ echo "Checking for changes in relevant modules..."+ if ./bin/mage can-skip-driver-tests "${{ github.event.pull_request.base.ref }}"; then+ echo 'skip=true' >> $GITHUB_OUTPUTelse- echo "skip=true" >> $GITHUB_OUTPUT- echo "No relevant team label, not merging into master/release, and no changes in relevant folders. Tier 3 driver tests will be skipped."+ echo 'skip=false' >> $GITHUB_OUTPUTfifishell: bash+be-tests-athena-ee:needs: determine-driver-skipsif: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}- runs-on: ubuntu-22.04+ runs-on: ubuntu-latesttimeout-minutes: 60env:CI: 'true'@@ -104,8 +101,9 @@ jobs:trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}be-tests-bigquery-cloud-sdk-ee:- if: ${{ !inputs.skip }}- runs-on: ubuntu-22.04+ needs: determine-driver-skips+ if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ runs-on: ubuntu-latesttimeout-minutes: 60strategy:fail-fast: false@@ -163,8 +161,9 @@ jobs:logs-artifact-suffix: ${{ matrix.job.name }} ${{ matrix.version.name }}be-tests-clickhouse-ee:- if: ${{ !inputs.skip }}- runs-on: ubuntu-22.04+ needs: determine-driver-skips+ if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ runs-on: ubuntu-latesttimeout-minutes: 60strategy:fail-fast: false@@ -235,7 +234,7 @@ jobs:be-tests-druid-ee:needs: determine-driver-skipsif: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}- runs-on: ubuntu-22.04+ runs-on: ubuntu-latesttimeout-minutes: 60env:CI: 'true'@@ -272,7 +271,7 @@ jobs:be-tests-druid-jdbc-ee:needs: determine-driver-skipsif: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}- runs-on: ubuntu-22.04+ runs-on: ubuntu-latesttimeout-minutes: 60env:CI: 'true'@@ -309,7 +308,7 @@ jobs:be-tests-databricks-ee:needs: determine-driver-skipsif: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}- runs-on: ubuntu-22.04+ runs-on: ubuntu-latesttimeout-minutes: 60env:CI: 'true'@@ -346,7 +345,7 @@ jobs:be-tests-databricks-multi-catalog-ee:needs: determine-driver-skipsif: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}- runs-on: ubuntu-22.04+ runs-on: ubuntu-latesttimeout-minutes: 60env:CI: 'true'@@ -383,7 +382,7 @@ jobs:be-tests-h2:if: ${{ !inputs.skip }}- runs-on: ubuntu-22.04+ runs-on: ubuntu-latesttimeout-minutes: 60env:CI: 'true'@@ -411,8 +410,9 @@ jobs:trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}be-tests-mysql-mariadb:- if: ${{ !inputs.skip }}- runs-on: ubuntu-22.04+ needs: determine-driver-skips+ if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ runs-on: ubuntu-latesttimeout-minutes: 60strategy:fail-fast: false@@ -547,7 +547,7 @@ jobs:be-tests-mongo:needs: determine-driver-skipsif: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}- runs-on: ubuntu-22.04+ runs-on: ubuntu-latesttimeout-minutes: 60strategy:fail-fast: false@@ -616,7 +616,7 @@ jobs:be-tests-mongo-ssl:needs: determine-driver-skipsif: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}- runs-on: ubuntu-22.04+ runs-on: ubuntu-latesttimeout-minutes: 60strategy:fail-fast: false@@ -681,7 +681,7 @@ jobs:be-tests-mongo-sharded-cluster-ee:needs: determine-driver-skipsif: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}- runs-on: ubuntu-22.04+ runs-on: ubuntu-latesttimeout-minutes: 60env:CI: 'true'@@ -715,7 +715,7 @@ jobs:be-tests-oracle:needs: determine-driver-skipsif: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}- runs-on: ubuntu-22.04+ runs-on: ubuntu-latesttimeout-minutes: 60strategy:fail-fast: false@@ -786,8 +786,9 @@ jobs:trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}be-tests-postgres:- if: ${{ !inputs.skip }}- runs-on: ubuntu-22.04+ needs: determine-driver-skips+ if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ runs-on: ubuntu-latesttimeout-minutes: 60strategy:fail-fast: false@@ -896,7 +897,7 @@ jobs:be-tests-presto-jdbc-ee:needs: determine-driver-skipsif: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}- runs-on: ubuntu-22.04+ runs-on: ubuntu-latesttimeout-minutes: 60env:CI: 'true'@@ -964,8 +965,9 @@ jobs:trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}be-tests-redshift-ee:- if: ${{ !inputs.skip }}- runs-on: ubuntu-22.04+ needs: determine-driver-skips+ if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}+ runs-on: ubuntu-latesttimeout-minutes: 60strategy:fail-fast: false@@ -1033,7 +1035,7 @@ jobs:be-tests-snowflake-ee:if: ${{ !inputs.skip }}- runs-on: ubuntu-22.04+ runs-on: ubuntu-latesttimeout-minutes: 60strategy:fail-fast: false@@ -1043,6 +1045,7 @@ jobs:test-args: >-:only-tags [:mb/driver-tests]:exclude-tags [:mb/transforms-python-test]+# DS 2025-09-26 temporarily disabled while we diagnose flakes# - name: Transforms Python Tests# test-args: >-@@ -1102,7 +1105,7 @@ jobs:be-tests-sparksql-ee:needs: determine-driver-skipsif: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}- runs-on: ubuntu-22.04+ runs-on: ubuntu-latesttimeout-minutes: 60env:CI: 'true'@@ -1137,7 +1140,7 @@ jobs:be-tests-sqlite-ee:needs: determine-driver-skipsif: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}- runs-on: ubuntu-22.04+ runs-on: ubuntu-latesttimeout-minutes: 60env:CI: 'true'@@ -1167,7 +1170,7 @@ jobs:be-tests-sqlserver:needs: determine-driver-skipsif: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}- runs-on: ubuntu-22.04+ runs-on: ubuntu-latesttimeout-minutes: 30strategy:fail-fast: false@@ -1239,7 +1242,7 @@ jobs:# be-tests-starburst-ee:# needs: determine-driver-skips# if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}- # runs-on: ubuntu-22.04+ # runs-on: ubuntu-latest# timeout-minutes: 60# env:# CI: 'true'@@ -1277,7 +1280,7 @@ jobs:login-to-ecr:needs: determine-driver-skipsif: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}- runs-on: ubuntu-22.04+ runs-on: ubuntu-latesttimeout-minutes: 15permissions:id-token: write@@ -1307,7 +1310,7 @@ jobs:be-tests-vertica-ee:needs: [determine-driver-skips, login-to-ecr]if: ${{ !inputs.skip && !(needs.determine-driver-skips.outputs.skip == 'true') }}- runs-on: ubuntu-22.04+ runs-on: ubuntu-latesttimeout-minutes: 60env:CI: 'true'diff --git a/bb.edn b/bb.ednindex 7e594e2df92..03341d2e08c 100644--- a/bb.edn+++ b/bb.edn@@ -204,6 +204,20 @@test)](nrepl-eval "mb.hawk.core" (str code) (:port options)))}+ affected-modules+ {:doc "Prints information about which backend modules have changes compared to a Git ref e.g. `master`"+ :examples [["./bin/mage affected-modules master"]+ ["./bin/mage affected-modules"]]+ :requires [[mage.modules]]+ :task (task! (mage.modules/cli-print-affected-modules (:arguments parsed)))}++ can-skip-driver-tests+ {:doc "Returns zero (success) if we can skip driver tests based on changes related to a Git ref e.g. `master`"+ :examples [["./bin/mage can-skip-driver-tests master"]+ ["./bin/mage can-skip-driver-tests"]]+ :requires [[mage.modules]]+ :task (task! (mage.modules/cli-can-skip-driver-tests (:arguments parsed)))}+ls {:doc "List all mage public and private tasks":examples [["./bin/mage ls" "List all tasks"]]:requires [[mage.util :as u]]diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.cljnew file mode 100644index 00000000000..d0430c28054--- /dev/null+++ b/mage/src/mage/modules.clj@@ -0,0 +1,168 @@+(ns mage.modules+ (:require+ [clojure.edn :as edn]+ [clojure.set :as set]+ [clojure.string :as str]+ [mage.util :as u]))++(set! *warn-on-reflection* true)++;;; TODO (Cam 2025-11-07) changes to test files should only cause us to run tests for that module as well, not+;;; everything that depends on that module directly or indirectly in `src`+(defn- file->module [filename]+ (or+ (when-let [[_match module] (re-matches #"^(?:(?:src)|(?:test))/metabase/([^/]+)/.*$" filename)]+ (symbol (str/replace module #"_" "-")))+ (when-let [[_match module] (re-matches #"^enterprise/backend/(?:(?:src)|(?:test))/metabase_enterprise/([^/]+)/.*$" filename)]+ (symbol "enterprise" (str/replace module #"_" "-")))))++(defn- updated-files->updated-modules [updated-files]+ (into (sorted-set)+ (keep file->module)+ updated-files))++(defn- updated-modules [git-ref]+ (let [git-ref (or git-ref "master")+ updated-files (u/updated-files git-ref)]+ (updated-files->updated-modules updated-files)))++(defn- module->test-directory+ [module]+ (case (namespace module)+ "enterprise" (str "enterprise/backend/test/metabase_enterprise/" (str/replace (name module) #"-" "_"))+ nil (str "test/metabase/" (str/replace (name module) #"-" "_"))))++(defn- dependencies+ "Read out the Kondo config for the modules linter; return a map of module => set of modules it directly depends on."+ []+ (let [config (-> (with-open [r (java.io.PushbackReader. (java.io.FileReader. ".clj-kondo/config/modules/config.edn"))]+ (edn/read r))+ :metabase/modules+ ;; ignore the config for [[metabase.connection-pool]] which comes from one of our libraries.+ (dissoc 'connection-pool))]+ (into (sorted-map)+ (map (fn [[k config]]+ [k (:uses config)]))+ config)))++(defn- direct-dependents+ "Set of modules that directly depend on `module`."+ [deps module]+ (into (sorted-set)+ (keep (fn [[a-module module-deps]]+ (when (or (= module-deps :any)+ (contains? module-deps module))+ a-module)))+ deps))++(comment+ (direct-dependents (dependencies) 'driver))++(defn- indirect-dependents+ "Set of modules that either directly or indirectly depend on `module`."+ ([deps module]+ (indirect-dependents deps module (sorted-set)))+ ([deps module acc]+ (let [module-deps (direct-dependents deps module)+ new-deps (set/difference module-deps acc)+ acc (into acc new-deps)]+ (reduce+ (fn [acc new-dep]+ (indirect-dependents deps new-dep acc))+ acc+ new-deps))))++(defn- affected-modules+ "Set of modules that are direct or indirect dependents of `modules`, and thus are affected by changes to them."+ [deps modules]+ (into (sorted-set)+ (mapcat (partial indirect-dependents deps))+ modules))++(defn- unaffected-modules+ "Return the set of modules that are unaffected "+ [deps modules]+ (set/difference+ (into (sorted-set) (keys deps))+ (affected-modules deps modules)))++(comment+ (unaffected-modules (dependencies) '#{enterprise/billing}))++(defn- skip-driver-tests? [deps modules]+ (let [unaffected (unaffected-modules deps modules)]+ (contains? unaffected 'driver)))++(defn- print-updated-and-unaffected-modules [deps updated]+ (let [unaffected (unaffected-modules deps updated)]+ (println "These modules have changed:" (pr-str updated))+ (println)+ (println)+ (println "These are all the modules are unaffected by these changes:" (pr-str unaffected))+ (println)+ (println)+ (println "(By unaffected, this means these modules do not have a direct or indirect dependency on the modules that have been changed.)")+ (println)+ (println)+ (println "Driver tests" (if (skip-driver-tests? deps updated)+ "CAN be skipped."+ "MUST be run."))))++(defn cli-print-affected-modules+ [[git-ref, :as _command-line-args]]+ (let [deps (dependencies)+ updated (updated-modules git-ref)+ affected (affected-modules deps updated)]+ (print-updated-and-unaffected-modules deps updated)+ (println)+ (println)+ (println "You can run tests for these modules and all downstream modules as follows:")+ (println)+ (println)+ (printf "clojure -X :dev:ee:ee-dev:test :only '%s'\n" (pr-str (mapv module->test-directory affected)))+ (flush))+ (System/exit 0))++(defn- changes-important-file-for-drivers?+ "Whether we should always run driver tests if we have changes relative to `git-ref` to something important like+ `deps.edn`."+ [git-ref]+ (some (fn [filename]+ (when (or (str/includes? filename "deps.edn")+ (str/includes? filename "modules/drivers/"))+ (printf "Running driver tests because %s was changed\n" (pr-str filename))+ (flush)+ filename))+ (u/updated-files (or git-ref "master"))))++(defn- remove-non-driver-test-namespaces [files]+ (into []+ (remove (fn [filename]+ (when (and (some #(str/includes? filename %)+ ["test/" "enterprise/backend/test/"])+ (not (some #(str/includes? filename %)+ ["query_processor"+ "driver"])))+ (printf "Ignorning changes in test namespace %s\n" (pr-str filename))+ (flush)+ filename)))+ files))++(defn cli-can-skip-driver-tests+ "Exits with zero status code if we can skip driver tests, nonzero if we cannot.++ Invoke this from the CLI with++ ./bin/mage can-skip-driver-tests [git-ref]"+ [[git-ref, :as _command-line-args]]+ (let [deps (dependencies)+ git-ref (or git-ref "master")+ updated-files (remove-non-driver-test-namespaces (u/updated-files git-ref))+ updated (updated-files->updated-modules updated-files)+ skip-tests? (skip-driver-tests? deps updated)]+ ;; Not strictly necessary, but people looking at CI will appreciate having this extra info.+ (print-updated-and-unaffected-modules deps updated)+ (System/exit ^Long (cond+ (not skip-tests?) 1+ (changes-important-file-for-drivers? git-ref) 1+ :else 0))))diff --git a/src/metabase/glossary/api.clj b/src/metabase/glossary/api.cljindex fb55eff2e7c..985d4cd4337 100644--- a/src/metabase/glossary/api.clj+++ b/src/metabase/glossary/api.clj@@ -1,5 +1,5 @@(ns metabase.glossary.api- "API endpoints for managing glossary entries."+ "REST API endpoints for managing glossary entries."(:require[metabase.api.common :as api][metabase.api.macros :as api.macros]diff --git a/test/metabase/lib/aggregation_test.cljc b/test/metabase/lib/aggregation_test.cljcindex ad94394b3b4..b1357d8dc6d 100644--- a/test/metabase/lib/aggregation_test.cljc+++ b/test/metabase/lib/aggregation_test.cljc@@ -951,3 +951,5 @@{:display-name "Max of Quantity: 50 bins":name "max"}](lib/aggregations-metadata query 1)))))))++;; trivial change to test CI; remove this next time you see it