The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add hierarchy to the backend module system (#82301)
Chris Trutereff3209a34bb
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex f5d4d6f31c8..a4960169213 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1399,7 +1399,6 @@metabase.metabot.settingsmetabase.metabot.tools.entity-details}:friends #{agent-api- enterprise/metabotenterprise/metabot-analyticsslackbot}:uses #{activity-feed@@ -2388,7 +2387,6 @@metabase.sso.coremetabase.sso.initmetabase.sso.settings}- :friends #{enterprise/sso}:uses #{apiauth-identityconfig@@ -2597,8 +2595,7 @@tracingtransforms-baseutil}- :friends #{models- enterprise/transforms-python}+ :friends #{enterprise/transforms-python}:model-exports #{:model/Transform:model/TransformDagRun:model/TransformJob@@ -3581,9 +3578,6 @@:api #{metabase-enterprise.metabot.apimetabase-enterprise.metabot.api.routesmetabase-enterprise.metabot.init}- :friends #{slackbot- mcp- metabot}:uses #{apiliblib-bediff --git a/.clj-kondo/src/hooks/clojure/core/ns.clj b/.clj-kondo/src/hooks/clojure/core/ns.cljindex 338023737f4..8cc79c82c61 100644--- a/.clj-kondo/src/hooks/clojure/core/ns.clj+++ b/.clj-kondo/src/hooks/clojure/core/ns.clj@@ -92,7 +92,7 @@(defn- lint-modules [ns-form-node config](let [ns-symb (ns-form-node->ns-symb ns-form-node)](when-not (modules/ignored-namespace? config ns-symb)- (when-let [current-module (modules/module ns-symb)]+ (when-let [current-module (modules/module config ns-symb)](let [required-namespace-symb-nodes (-> ns-form-nodens-form-node->require-noderequire-node->namespace-symb-nodes)]diff --git a/.clj-kondo/src/hooks/clojure/core/require.clj b/.clj-kondo/src/hooks/clojure/core/require.cljindex 669091d8032..552072164dc 100644--- a/.clj-kondo/src/hooks/clojure/core/require.clj+++ b/.clj-kondo/src/hooks/clojure/core/require.clj@@ -11,10 +11,10 @@(every? simple-symbol? required-namespaces)]}(when-not (modules/ignored-namespace? config current-ns);; ignore namespaces outside of the module system- (when-let [current-module (modules/module current-ns)]+ (when-let [current-module (modules/module config current-ns)](doseq [required-namespace required-namespaces];; ignore namespaces outside of the module system.- (when (modules/module required-namespace)+ (when (modules/module config required-namespace)(when-let [error (modules/usage-error config current-module required-namespace)](hooks/reg-finding! (assoc (meta node):message error@@ -22,7 +22,7 @@(defn- lint-dynamic-require* [node current-ns config](when (and (not (modules/ignored-namespace? config current-ns))- (modules/module current-ns))+ (modules/module config current-ns))(hooks/reg-finding! (assoc (meta node):message "Module dependency cannot be statically determined from a dynamic require":type :metabase/modules))))diff --git a/.clj-kondo/src/hooks/clojure/test.clj b/.clj-kondo/src/hooks/clojure/test.cljindex 3c93457ba27..2a838825971 100644--- a/.clj-kondo/src/hooks/clojure/test.clj+++ b/.clj-kondo/src/hooks/clojure/test.clj@@ -194,7 +194,7 @@(not (contains? @warned-namespaces ns-symb)))(swap! warned-namespaces conj ns-symb)(let [known-modules (set (keys (:metabase/modules (modules/config input))))- current-module (modules/module ns-symb)]+ current-module (modules/module (modules/config input) ns-symb)](when (or (not current-module)(not (contains? known-modules current-module)))(hooks/reg-finding! (assoc (meta node)diff --git a/.clj-kondo/src/hooks/common/modules.clj b/.clj-kondo/src/hooks/common/modules.cljindex 3f6dd7af6d4..e9927904bec 100644--- a/.clj-kondo/src/hooks/common/modules.clj+++ b/.clj-kondo/src/hooks/common/modules.clj@@ -1,4 +1,8 @@(ns hooks.common.modules+ "Resolves namespaces to modules and enforces module boundaries.++ The Kondo linter, config generator, and CI tooling share this namespace so+ they apply the same rules."(:require[clojure.string :as str]))@@ -12,25 +16,130 @@(merge (get-in config [:linters :metabase/modules])(select-keys config [:metabase/modules])))-(defn module+;;;; Module tree. These functions take the `:metabase/modules` map, or the prefix map built from it.+;;;;+;;;; Dots express nesting: `lib.schema` is a child of `lib`. A declared+;;;; `enterprise/X` module is also a child of OSS module `X`. A namespace+;;;; belongs to the module with the most specific matching `:ns-prefix`.++(defn default-ns-prefix+ "Namespace prefix a module owns unless it sets `:ns-prefix`: `metabase.lib.schema` for `lib.schema`,+ `metabase-enterprise.foo` for `enterprise/foo`."+ [module]+ (str (if (= "enterprise" (namespace module)) "metabase-enterprise." "metabase.") (name module)))++(defn module-ns-prefix+ "Namespace prefix `module` owns: its `:ns-prefix`, else [[default-ns-prefix]]."+ [modules module]+ (or (get-in modules [module :ns-prefix])+ (default-ns-prefix module)))++(defn build-prefix->module+ "Map each declared namespace prefix to its module, for [[declared-module]]."+ [modules]+ (into {}+ (map (fn [module] [(module-ns-prefix modules module) module]))+ (keys modules)))++(defn declared-module+ "Resolve `ns-symb` to its declared module, or `nil`.++ Chooses the most specific dotted prefix and ignores a trailing `-test`."+ [prefix->module ns-symb]+ (loop [candidate (str/replace (str ns-symb) #"-test$" "")]+ (or (get prefix->module candidate)+ (when-let [dot (str/last-index-of candidate ".")]+ (recur (subs candidate 0 dot))))))++(defn resolve-module"E.g.- (module 'metabase.qp.middleware.wow) => 'qp- (module 'metabase-enterprise.whatever.core) => enterprise/whatever"- [ns-symb]+ (resolve-module prefix->module 'metabase.qp.middleware.wow) => 'qp+ (resolve-module prefix->module 'metabase-enterprise.whatever.core) => enterprise/whatever++ An unmatched Metabase namespace resolves to its first segment so the linter can report an undeclared module."+ [prefix->module ns-symb]{:pre [(simple-symbol? ns-symb)]};; treat something like `metabase.driver-test` (for a module that hasn't fully been updated to use `.core`;; namespaces) as being in the `driver` module(let [ns-symb (if (str/ends-with? (name ns-symb) "-test")(symbol (str/replace (name ns-symb) #"-test$" ""))ns-symb)]- (or (some->> (re-find #"^metabase-enterprise\.([^.]+)" (str ns-symb))+ (or (declared-module prefix->module ns-symb)+ (some->> (re-find #"^metabase-enterprise\.([^.]+)" (str ns-symb))second(symbol "enterprise"))(some-> (re-find #"^metabase\.([^.]+)" (str ns-symb))secondsymbol))))+(defn parent-module+ "The direct parent of `module`, or `nil` when it is top-level."+ [modules module]+ (let [module-name (name module)]+ (if-let [dot (str/last-index-of module-name ".")]+ (symbol (namespace module) (subs module-name 0 dot))+ (when (= "enterprise" (namespace module))+ (let [oss (symbol module-name)]+ (when (contains? modules oss)+ oss))))))++(defn descendant-of?+ "Whether `module` is `ancestor` or one of its descendants."+ [modules module ancestor]+ (boolean (some #{ancestor} (take-while some? (iterate #(parent-module modules %) module)))))++(defn- exports-child?+ [modules parent child]+ (or (contains? (set (get-in modules [parent :module-exports])) child)+ ;; OSS module `X` implicitly exports its `enterprise/X` companion.+ (and (not= (namespace parent) (namespace child))+ (contains? modules child))))++(defn- blocking-export+ "The first missing export from `module` toward the root.++ Returns `{:ancestor parent, :child child}`, or `nil` when every link is+ exported."+ [modules module]+ (loop [child module]+ (when-let [ancestor (parent-module modules child)]+ (if (exports-child? modules ancestor child)+ (recur ancestor)+ {:ancestor ancestor, :child child}))))++(defn namability-error+ "Explain why `caller` may not refer to `target` in `:uses`, or return `nil` if it may.++ A nested module is private to the nearest ancestor with a missing export.+ Exporting every link makes it public."+ [modules caller target]+ (when-let [{:keys [ancestor child]} (blocking-export modules target)]+ (when-not (descendant-of? modules caller ancestor)+ (format (str "Module %s is nested and not exported by its ancestors; %s may not use it. Add %s to %s's "+ ":module-exports, or move the caller into the %s subtree. [:metabase/modules %s :module-exports]")+ target caller child ancestor ancestor ancestor))))++;;;; Lint rules. These functions take the full linter config.++(defonce ^:private prefix->module-cache+ (atom nil))++(defn- prefix->module+ "[[build-prefix->module]] for `config`, cached because Kondo hands every hook call the same `:metabase/modules` map."+ [config]+ (let [modules (:metabase/modules config)+ cached @prefix->module-cache]+ (if (identical? modules (:modules cached))+ (:prefix->module cached)+ (:prefix->module (reset! prefix->module-cache {:modules modules+ :prefix->module (build-prefix->module modules)})))))++(defn module+ "Resolve `ns-symb` to its module under `config`; see [[resolve-module]]."+ [config ns-symb]+ (resolve-module (prefix->module config) ns-symb))+(defn- module-api-namespaces"Set of API namespace symbols for a given module. `:any` means you can use anything, there are no API namespaces forthis module (yet). If unspecified, the default is just the `<module>.core` namespace."@@ -44,9 +153,7 @@module-config:else- (let [ns-prefix (if (= (namespace module) "enterprise")- (str "metabase-enterprise." (name module))- (name module))]+ (let [ns-prefix (module-ns-prefix (:metabase/modules config) module)]#{(symbol (str ns-prefix ".api"))(symbol (str ns-prefix ".core"))(symbol (str ns-prefix ".init"))}))))@@ -68,15 +175,17 @@(contains? (set allowed-modules) required-module))))(defn- allowed-module-namespace? [config current-module ns-symb]- (let [module (module ns-symb)+ (let [module (module config ns-symb)module-api-namespaces (module-api-namespaces config module)module-friends (module-friends config module)](or (nil? module-api-namespaces)(contains? module-api-namespaces ns-symb)- (contains? module-friends current-module))))+ (contains? module-friends current-module)+ ;; a child may use its ancestors' internals; a parent still goes through its child's `:api`+ (descendant-of? (:metabase/modules config) current-module module))))(defn- rest-module? [module]- (str/ends-with? module "-rest"))+ (re-find #"[.-]rest$" (str module)))(defn- routes-module? [module](str/ends-with? module "-routes"))@@ -89,7 +198,7 @@error type if there is one, otherwise `nil` if there are no errors."[config current-module required-namespace];; ignore stuff not in a module i.e. non-Metabase stuff.- (when-let [required-module (module required-namespace)]+ (when-let [required-module (module config required-namespace)](when-not (= current-module required-module)(cond(not (allowed-module? config current-module required-module))@@ -113,4 +222,8 @@required-modulecurrent-modulerequired-module- (symbol (str/replace required-module #"-rest$" "")))))))+ (symbol (str/replace required-module #"[.-]rest$" "")))++ ;; Config tests check explicit `:uses`; wildcard callers are checked here.+ (= :any (allowed-modules config current-module))+ (namability-error (:metabase/modules config) current-module required-module)))))diff --git a/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj b/.clj-kondo/src/hooks/metabase/toucan/db_ns.cljindex db2296c7a69..cdfe8d20099 100644--- a/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj+++ b/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj@@ -8,10 +8,20 @@Registered as an `:analyze-call` hook on each of those functions in `.clj-kondo/config.edn`. The hookreturns its input unchanged so Kondo's normal analysis of the call (arity, var usage) still runs."(:require- [clj-kondo.hooks-api :as hooks]))+ [clj-kondo.hooks-api :as hooks]+ [hooks.common.modules :as modules]))-(defn- db-namespace? [ns-sym]- (boolean (re-matches #"^metabase(?:-enterprise)?\.(?:driver\.)?[^.]+\.db$" (name ns-sym))))+(def ^:private driver-db-namespace+ "Matches `metabase.driver.<driver>.db`; individual drivers are not modules."+ #"^metabase\.driver\.[^.]+\.db$")++(defn- db-namespace? [config ns-sym]+ (let [ns-str (name ns-sym)]+ (boolean+ ;; Resolve ownership first: a nested `.db` name alone does not make a module.+ (or (when-let [module (modules/module config ns-sym)]+ (= ns-str (str (modules/module-ns-prefix (:metabase/modules config) module) ".db")))+ (re-matches driver-db-namespace ns-str)))))(defn- test-file?"Whether `filename` is in a test source tree. Test namespaces are exempt through the `test-namespaces` group in@@ -24,7 +34,7 @@namespace."[{:keys [node ns filename] :as input}](when (and ns- (not (db-namespace? ns))+ (not (db-namespace? (modules/config input) ns))(not (test-file? filename)))(let [fn-node (first (:children node))](hooks/reg-finding!diff --git a/.clj-kondo/test/hooks/common/modules_test.clj b/.clj-kondo/test/hooks/common/modules_test.cljindex 98021fe42ef..cba7f15f10d 100644--- a/.clj-kondo/test/hooks/common/modules_test.clj+++ b/.clj-kondo/test/hooks/common/modules_test.clj@@ -5,6 +5,6 @@(deftest ^:parallel module-test(are [symb expected] (= expected- (hooks.common.modules/module symb))+ (hooks.common.modules/module {} symb))'metabase.qp.middleware.wow 'qp'metabase-enterprise.whatever.core 'enterprise/whatever))diff --git a/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.clj b/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.cljindex 2109ad98d1d..99bd57cd346 100644--- a/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.clj+++ b/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.clj@@ -5,14 +5,15 @@[clojure.test :refer :all][hooks.metabase.toucan.db-ns :as toucan.db-ns]))-(defn- lint-query-call [form ns-sym & [filename]]+(defn- lint-query-call [form ns-sym & [filename modules]](binding [clj-kondo.impl.utils/*ctx* {:config {:linters {:metabase/t2-query-namespace {:level :warning}}}:ignores (atom nil):findings (atom []):namespaces (atom {})}](let [input {:node (hooks/parse-string (pr-str form)):ns ns-sym- :filename (or filename "src/metabase/foo/bar.clj")}+ :filename (or filename "src/metabase/foo/bar.clj")+ :config {:metabase/modules modules}}output (toucan.db-ns/lint-query-call input)](is (identical? (:node input) (:node output))"the hook must return the node unchanged so Kondo's normal analysis still runs")@@ -48,3 +49,25 @@(is (=? [{:type :metabase/t2-query-namespace:message #".*`toucan2.core/insert!`.*"}](lint-query-call '(toucan2.core/insert! :model/Card {}) 'metabase.queries.models.card)))))++(deftest ^:parallel t2-query-namespace-follows-the-module-tree-test+ (let [modules '{queries {}+ metabot {}+ metabot.llm {}+ lib.be {:ns-prefix "metabase.lib-be"}+ enterprise/sandbox {}}]+ (testing "a nested module owns its own db namespace"+ (is (empty? (lint-query-call '(t2/select :model/Card) 'metabase.metabot.llm.db nil modules))))+ (testing "the parent's db namespace stays its own"+ (is (empty? (lint-query-call '(t2/select :model/Card) 'metabase.metabot.db nil modules))))+ (testing "a module with an explicit :ns-prefix is resolved through it, not its dotted name"+ (is (empty? (lint-query-call '(t2/select :model/Card) 'metabase.lib-be.db nil modules)))+ (is (=? [{:type :metabase/t2-query-namespace}]+ (lint-query-call '(t2/select :model/Card) 'metabase.lib.be.db nil modules))))+ (testing "a db namespace under a directory naming no module is still a finding"+ (is (=? [{:type :metabase/t2-query-namespace}]+ (lint-query-call '(t2/query {:select [:*]}) 'metabase.queries.models.db nil modules)))+ (is (=? [{:type :metabase/t2-query-namespace}]+ (lint-query-call '(t2/query {:select [:*]}) 'metabase.metabot.llm.models.db nil modules))))+ (testing "enterprise modules resolve through the metabase-enterprise root"+ (is (empty? (lint-query-call '(t2/select :model/Card) 'metabase-enterprise.sandbox.db nil modules))))))diff --git a/CLAUDE.md b/CLAUDE.mdindex 742c9f14a18..169b5b0daa4 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -84,6 +84,20 @@ Run all repository-level checks, or one named suite:./bin/mage project-tests <backend|migrations|modules|ratchets>```+### Nested modules++Module names form a tree: `lib.schema` is a child of `lib`. When OSS module `search` exists,+`enterprise/search` is its child. Run `./bin/mage modules-tree` to inspect the hierarchy.++- Namespace ownership uses the most specific matching prefix. Declaring `lib.schema` assigns+ `metabase.lib.schema.*` to it without moving files. Use `:ns-prefix` when namespaces do not match the+ module name.+- Every cross-module dependency still requires `:uses`.+- A child may use an ancestor's internal namespaces. Parents, siblings, and unrelated modules must use the+ target's `:api`.+- Each `:module-exports` entry widens a nested module's visibility by one ancestor. Export every link to+ make it available everywhere. OSS module `X` exports its `enterprise/X` companion automatically.+## Kondo Ignore Ratchets`.clj-kondo/ratchets.edn` records, per linter, how many inline `:clj-kondo/ignore` forms the backend sourcediff --git a/bb.edn b/bb.ednindex 5358474ba16..ac1ee105440 100644--- a/bb.edn+++ b/bb.edn@@ -2,7 +2,8 @@;; we put path as bin, and everything is in the ./mage subdirectory,;; so the namespaces are mage.cli, mage.format, etc.;; dev/src is here for tasks whose logic is shared with JVM tests (e.g. dev.kondo-ratchet)- :paths ["mage/src" "mage/test" "bin/lint-migrations-file/src" "dev/src"]+ ;; Let Mage use the same module-resolution rules as the Kondo hooks.+ :paths ["mage/src" "mage/test" "bin/lint-migrations-file/src" "dev/src" ".clj-kondo/src"];; use Git version of dep temporarily until a new release > 0.16.4 is cut:deps {dev.weavejester/cljfmt {:git/sha "075eae26e3c8dad433b88f05927bcd50e3571328":git/url "https://github.com/weavejester/cljfmt"} ;; run cljfmt from bb@@ -260,6 +261,16 @@:requires [[mage.modules]]:task (task! (mage.modules/cli-print-affected-modules (:arguments parsed)))}+ modules-tree+ {:doc "Print the module hierarchy, nesting enterprise extensions under their OSS module"+ :examples [["./bin/mage modules-tree" "Print the full tree; * marks a custom :ns-prefix"]+ ["./bin/mage modules-tree --nested-only" "Print only roots with nested modules"]+ ["./bin/mage modules-tree --prefixes" "Show custom prefixes instead of *"]]+ :options [["-n" "--nested-only" "Only print modules that have nested children"]+ ["-p" "--prefixes" "Print the :ns-prefix instead of the star"]]+ :requires [[mage.modules]]+ :task (task! (mage.modules/cli-print-module-tree parsed))}+can-skip-driver-tests{:doc "Can we skip driver tests for this branch?":examples [["./bin/mage can-skip-driver-tests master" "Returns zero (success) if we can skip driver tests based on changes related to `master`"]diff --git a/dev/src/dev/deps_graph.clj b/dev/src/dev/deps_graph.cljindex 6d92018bdc3..13804771aa9 100644--- a/dev/src/dev/deps_graph.clj+++ b/dev/src/dev/deps_graph.clj@@ -8,6 +8,7 @@[clojure.tools.namespace.find :as ns.find][clojure.tools.namespace.parse :as ns.parse][clojure.walk :as walk]+ [hooks.common.modules :as modules][lambdaisland.deep-diff2 :as ddiff][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]@@ -84,19 +85,6 @@(mapcat ns.find/find-sources-in-dir(list* (source-root) (enterprise-source-root) (plugin-source-roots))))-(mu/defn- module :- [:maybe symbol?]- "E.g.-- (module 'metabase.qp.middleware.wow) => 'qp- (module 'metabase-enterprise.whatever.core) => enterprise/whatever"- [ns-symb :- simple-symbol?]- (or (some->> (re-find #"^metabase-enterprise\.([^.]+)" (str ns-symb))- second- (symbol "enterprise"))- (some-> (re-find #"^metabase\.([^.]+)" (str ns-symb))- second- symbol)))-(def ^:private require-symbols'#{requireclojure.core/require@@ -225,7 +213,8 @@[:namespace simple-symbol?][:module symbol?][:dynamic {:optional true} :keyword]]]]]- [file :- [:or+ [prefix->module :- map?+ file :- [:orstring?[:fn {:error/message "Instance of a java.io.File"} #(instance? java.io.File %)]]](try@@ -249,10 +238,10 @@#_defenterprise-schema-deps])]{:namespace ns-symb:filename (file->path-relative-to-project-root file)- :module (module ns-symb)+ :module (modules/resolve-module prefix->module ns-symb):deps (sort-by pr-str(keep (fn [required-ns]- (when-let [module (module required-ns)]+ (when-let [module (modules/resolve-module prefix->module required-ns)](when-not (some-> ignored-dependencies ns-symb required-ns)(merge{:namespace required-ns@@ -265,18 +254,24 @@{:file file}e)))))+(declare kondo-config)+(comment- (file-dependencies "src/metabase/app_db/setup.clj")+ (file-dependencies (modules/build-prefix->module (kondo-config)) "src/metabase/app_db/setup.clj");; should ignore the entries from [[ignored-dependencies]]- (file-dependencies "src/metabase/config.clj")+ (file-dependencies (modules/build-prefix->module (kondo-config)) "src/metabase/config.clj")- (file-dependencies "src/metabase/query_processor/middleware/permissions.clj"))+ (file-dependencies (modules/build-prefix->module (kondo-config)) "src/metabase/query_processor/middleware/permissions.clj"))(defn dependencies- "Calculate information about all the modules dependencies for all *SOURCE* files in the Metabase project by parsing- the files."- []- (map file-dependencies (find-source-files)))+ "Parse source files into module dependency data.++ Namespaces resolve through `prefix->module`, which defaults to the current+ module config."+ ([]+ (dependencies (modules/build-prefix->module (kondo-config))))+ ([prefix->module]+ (map (partial file-dependencies prefix->module) (find-source-files))))(defn external-usages"All usages of a module named by `module-symb` outside that module."@@ -313,17 +308,23 @@[kondo-config module-symb](get-in kondo-config [module-symb :friends]))-(declare kondo-config)-(defn externally-used-namespaces-ignoring-friends- "All namespaces from a module that are used outside that module, excluding usages by `:friends` of the module."+ "Namespaces that external callers require from `module-symb`.++ Ignores friends and descendants, which may use internal namespaces. A+ descendant still preserves an existing API entry that it uses."([module-symb](externally-used-namespaces-ignoring-friends (dependencies) (kondo-config) module-symb))([deps kondo-config module-symb]- (let [friends (module-friends kondo-config module-symb)]+ (let [friends (module-friends kondo-config module-symb)+ api (get-in kondo-config [module-symb :api])](into (sorted-set)(comp (remove #(contains? friends (:module %)))+ ;; Descendant use does not create API, but it preserves existing entries.+ (remove (fn [{:keys [module depends-on-namespace]}]+ (and (modules/descendant-of? kondo-config module module-symb)+ (not (and (set? api) (contains? api depends-on-namespace))))))(map :depends-on-namespace))(external-usages deps module-symb)))))@@ -397,13 +398,12 @@"Like [[dependencies]] but also includes transient dependencies."[deps](let [deps-graph (module-dependencies deps)+ ;; Keep the seed set so cycles converge instead of oscillating.expand-deps (fn expand-deps [deps]- (let [deps' (into (sorted-set)- (mapcat deps-graph)- deps)]+ (let [deps' (into deps (mapcat deps-graph deps))](if (= deps deps')deps- (expand-deps deps'))))]+ (recur deps'))))](into (sorted-map)(map (fn [[k v]][k (expand-deps v)]))@@ -436,6 +436,12 @@;; ignore the config for [[metabase.connection-pool]] which comes from one of our libraries.(dissoc 'connection-pool)))+(defn module-team+ "Team owning `module`: its own `:team`, else its nearest ancestor's."+ [config module]+ (some #(get-in config [% :team])+ (take-while some? (iterate #(modules/parent-module config %) module))))+(defn- kondo-config-diff-ignore-any"Ignore entries in the config that use `:any`."[diff]@@ -455,7 +461,7 @@([deps](let [kondo-config (kondo-config)](-> (ddiff/diff- (update-vals kondo-config #(dissoc % :team :friends :model-imports :model-exports))+ (update-vals kondo-config #(dissoc % :team :friends :model-imports :model-exports :module-exports :ns-prefix))(generate-config deps kondo-config))ddiff/minimizekondo-config-diff-ignore-any@@ -577,23 +583,23 @@(defn- simulate-rename"Create a new version of `deps` as they would appear if you renamed namespace(s).- (simulate-rename (dependencies) '{metabase.users.api metabase.users-rest.api})"- ([deps old-namespace new-namespace]+ (simulate-rename deps prefix->module '{metabase.users.api metabase.users-rest.api})"+ ([deps prefix->module old-namespace new-namespace](for [dep deps](-> dep(cond-> (= (:namespace dep) old-namespace)(assoc :namespace new-namespace- :module (module new-namespace)))+ :module (modules/resolve-module prefix->module new-namespace)))(update :deps (fn [deps](for [dep deps](if (= (:namespace dep) old-namespace)- {:namespace new-namespace, :module (module new-namespace)}+ {:namespace new-namespace, :module (modules/resolve-module prefix->module new-namespace)}dep)))))))- ([deps old-namespace->new-namespace]+ ([deps prefix->module old-namespace->new-namespace](reduce(fn [deps [old-namespace new-namespace]]- (simulate-rename deps old-namespace new-namespace))+ (simulate-rename deps prefix->module old-namespace new-namespace))depsold-namespace->new-namespace)))@@ -603,9 +609,10 @@(dependencies-eliminated-by-renaming-namespaces 'users '{metabase.users.api metabase.users-rest.api})"[module old-namespace->new-namespace]- (let [deps (dependencies)+ (let [prefix->module (modules/build-prefix->module (kondo-config))+ deps (dependencies prefix->module)old-module-deps (into (sorted-set) (keys (all-module-deps-paths deps module)))- new-deps (simulate-rename deps old-namespace->new-namespace)+ new-deps (simulate-rename deps prefix->module old-namespace->new-namespace)new-module-deps (into (sorted-set) (keys (all-module-deps-paths new-deps module)))](set/difference old-module-deps new-module-deps)))@@ -641,12 +648,13 @@"Given a collection of `test-filenames`, return the set of source filenames (relative to the project root directory)that when changed should trigger these tests."([test-filenames]- (test-filenames->relevant-source-filenames (dependencies) test-filenames))- ([deps test-filenames]+ (let [prefix->module (modules/build-prefix->module (kondo-config))]+ (test-filenames->relevant-source-filenames (dependencies prefix->module) prefix->module test-filenames)))+ ([deps prefix->module test-filenames](into(sorted-set)(comp (map file->namespace)- (map module)+ (map #(modules/resolve-module prefix->module %))(distinct)(mapcat (fn [module](into #{module} (module->all-deps deps module))))@@ -700,36 +708,60 @@(comment(module->dependents (dependencies) 'settings-rest))-(defn- module->test-directory [module]- (let [parent-dir (case (namespace module)- nil "test/metabase/"- "enterprise" "enterprise/backend/test/metabase_enterprise/")- module-dir (str/replace (name module) #"-" "_")]- (str parent-dir module-dir)))+(def ^:private test-source-file-extensions+ [".clj" ".cljc" ".cljs" ".bb"])++(defn- module->test-path-prefix [modules-config module]+ (let [ns-prefix (modules/module-ns-prefix modules-config module)]+ (str (when (str/starts-with? ns-prefix "metabase-enterprise.") "enterprise/backend/")+ "test/"+ (-> ns-prefix (str/replace "." "/") (str/replace "-" "_")))))++(defn- existing-test-file-paths [path-prefix]+ (into (sorted-set)+ (keep (fn [extension]+ (let [file (io/file (str path-prefix "_test" extension))]+ (when (.isFile file)+ (file->path-relative-to-project-root file)))))+ test-source-file-extensions))(mu/defn- module->test-files :- [:set :string]"Return the set of test filenames associated with a `module`."- [module :- :symbol]- (let [test-dir (module->test-directory module)- test-filenames (ns.find/find-sources-in-dir (io/file test-dir))]- (into- (sorted-set)- (map file->path-relative-to-project-root)- test-filenames)))+ ([modules-config :- map?+ module-sym :- :symbol]+ (module->test-files modules-config (modules/build-prefix->module modules-config) module-sym))+ ([modules-config :- map?+ prefix->module :- map?+ module-sym :- :symbol]+ (let [path-prefix (module->test-path-prefix modules-config module-sym)+ test-dir (io/file path-prefix)+ nested-tests (when (.isDirectory test-dir)+ (into+ (sorted-set)+ (comp (filter #(= module-sym+ (modules/resolve-module prefix->module (file->namespace %))))+ (map file->path-relative-to-project-root))+ (ns.find/find-sources-in-dir test-dir)))]+ (into (existing-test-file-paths path-prefix)+ nested-tests))))(defn source-filenames->relevant-test-filenames"Given a collection of `source-filenames`, return the set of test filenames (relative to the project root directory)that we should re-run when any of `source-filenames` change."([source-filenames]- (source-filenames->relevant-test-filenames (dependencies) source-filenames))- ([deps source-filenames]+ (let [modules-config (kondo-config)+ prefix->module (modules/build-prefix->module modules-config)]+ (source-filenames->relevant-test-filenames+ (dependencies prefix->module) modules-config prefix->module source-filenames)))+ ([deps modules-config prefix->module source-filenames](into(sorted-set)(comp (map file->namespace)- (map module)+ (keep #(modules/resolve-module prefix->module %))(distinct)(mapcat #(module->dependents deps %))- (mapcat module->test-files))+ (distinct)+ (mapcat #(module->test-files modules-config prefix->module %)))source-filenames)))(comment@@ -782,18 +814,18 @@@models))(defn model-ownership- "Scan all source files via [[find-model-definitions]], building a map of `:model/X` => module symbol.- The module is derived from the defining namespace via [[module]]."+ "Map each defined `:model/X` to the module that owns its namespace."[]- (into (sorted-map)- (for [file (find-source-files)- :let [ns-symb (-> (ns.file/read-file-ns-decl file)- ns.parse/name-from-ns-decl)- mod (module ns-symb)- models (find-model-definitions file)]- :when mod- model models]- [model mod])))+ (let [prefix->mod (modules/build-prefix->module (kondo-config))]+ (into (sorted-map)+ (for [file (find-source-files)+ :let [ns-symb (-> (ns.file/read-file-ns-decl file)+ ns.parse/name-from-ns-decl)+ mod (modules/resolve-module prefix->mod ns-symb)+ models (find-model-definitions file)]+ :when mod+ model models]+ [model mod]))))(def ^:private model-boundary-exempt-namespaces"Namespaces that are exempt from model boundary checking. These are 'glue' namespaces that intentionally reference@@ -827,24 +859,25 @@referenced in each module's source files. Exempt namespaces (e.g. `metabase.models.resolution`) are excluded.Includes all modules (including bypass modules) — callers filter as needed."[]- (reduce- (fn [acc file]- (try- (let [ns-symb (-> (ns.file/read-file-ns-decl file)- ns.parse/name-from-ns-decl)- mod (module ns-symb)]- (if (and mod (not (contains? model-boundary-exempt-namespaces ns-symb)))- (let [models (find-model-keywords file)]- (if (seq models)- (update acc mod (fnil into (sorted-set)) models)- acc))- acc))- (catch Throwable e- (throw (ex-info (format "Error scanning model references in %s" (str file))- {:file file}- e)))))- (sorted-map)- (find-source-files)))+ (let [prefix->mod (modules/build-prefix->module (kondo-config))]+ (reduce+ (fn [acc file]+ (try+ (let [ns-symb (-> (ns.file/read-file-ns-decl file)+ ns.parse/name-from-ns-decl)+ mod (modules/resolve-module prefix->mod ns-symb)]+ (if (and mod (not (contains? model-boundary-exempt-namespaces ns-symb)))+ (let [models (find-model-keywords file)]+ (if (seq models)+ (update acc mod (fnil into (sorted-set)) models)+ acc))+ acc))+ (catch Throwable e+ (throw (ex-info (format "Error scanning model references in %s" (str file))+ {:file file}+ e)))))+ (sorted-map)+ (find-source-files))))(defn model-boundary-violations"Find all model boundary violations across the codebase.@@ -862,36 +895,37 @@([kondo-config](model-boundary-violations kondo-config (model-ownership)))([kondo-config ownership]- (into []- (comp- (mapcat- (fn [file]- (try- (let [ns-symb (-> (ns.file/read-file-ns-decl file)- ns.parse/name-from-ns-decl)- mod (module ns-symb)]- (when (and mod- (not (contains? model-boundary-exempt-namespaces ns-symb)))- (let [model-imports (get-in kondo-config [mod :model-imports] #{})- models (find-model-keywords file)- rel-path (file->path-relative-to-project-root file)]- (for [model models- :let [defining-mod (get ownership model)]- :when (not= defining-mod mod)- :let [model-exports (when defining-mod- (get-in kondo-config [defining-mod :model-exports] #{}))]- violation-type (model-reference-violations- model defining-mod model-exports model-imports)]- {:file rel-path- :module mod- :model model- :defining-module defining-mod- :violation-type violation-type}))))- (catch Throwable e- (throw (ex-info (format "Error checking model boundaries in %s" (str file))- {:file file}- e)))))))- (find-source-files))))+ (let [prefix->mod (modules/build-prefix->module kondo-config)]+ (into []+ (comp+ (mapcat+ (fn [file]+ (try+ (let [ns-symb (-> (ns.file/read-file-ns-decl file)+ ns.parse/name-from-ns-decl)+ mod (modules/resolve-module prefix->mod ns-symb)]+ (when (and mod+ (not (contains? model-boundary-exempt-namespaces ns-symb)))+ (let [model-imports (get-in kondo-config [mod :model-imports] #{})+ models (find-model-keywords file)+ rel-path (file->path-relative-to-project-root file)]+ (for [model models+ :let [defining-mod (get ownership model)]+ :when (not= defining-mod mod)+ :let [model-exports (when defining-mod+ (get-in kondo-config [defining-mod :model-exports] #{}))]+ violation-type (model-reference-violations+ model defining-mod model-exports model-imports)]+ {:file rel-path+ :module mod+ :model model+ :defining-module defining-mod+ :violation-type violation-type}))))+ (catch Throwable e+ (throw (ex-info (format "Error checking model boundaries in %s" (str file))+ {:file file}+ e)))))))+ (find-source-files)))))(comment(model-ownership)diff --git a/dev/src/dev/module_viz.clj b/dev/src/dev/module_viz.cljindex fa8ab0f60d3..0cab5b84c86 100644--- a/dev/src/dev/module_viz.clj+++ b/dev/src/dev/module_viz.clj@@ -50,6 +50,9 @@:else(str "src/" path ".clj"))))+;; TODO (Chris 2026-09-11) -- Not aware of nested modules: paths come from the module name rather than its+;; `:ns-prefix`, a parent's LOC includes its children's files, and teams ignore inheritance. Consolidate with+;; `hooks.common.modules/module-ns-prefix` and `dev.deps-graph/module-team`.(defn- module->github-path"Best-effort repo-relative source path for a module symbol."[module-symb]diff --git a/dev/test/dev/modules_config_test.clj b/dev/test/dev/modules_config_test.cljindex de7465e489a..bae00819e6c 100644--- a/dev/test/dev/modules_config_test.clj+++ b/dev/test/dev/modules_config_test.clj@@ -104,6 +104,17 @@:uses #{other}:model-imports #{:model/X}}}))))))+(deftest ^:parallel nested-module-metadata-is-preserved-test+ (let [input (str "{:metabase/modules {parent {:team \"T\" :api #{}} "+ "parent.child {:ns-prefix \"metabase.special-child\" "+ ":module-exports #{parent.grandchild} :api #{}}}}")+ output (rewrite input '{parent {:api #{}}+ parent.child {:api #{metabase.special-child.core}}})+ modules (parse-modules output)]+ (is (= "T" (get-in modules ['parent :team])))+ (is (= "metabase.special-child" (get-in modules ['parent.child :ns-prefix])))+ (is (= '#{parent.grandchild} (get-in modules ['parent.child :module-exports])))))+(deftest ^:parallel missing-key-is-appended-test(let [input "{:metabase/modules {m {:team \"T\"}}}"output (rewrite input '{m {:api #{metabase.m.core}}})]diff --git a/dev/test/metabase/core/modules_test.clj b/dev/test/metabase/core/modules_test.cljindex 1dcc1b449b9..858f7691381 100644--- a/dev/test/metabase/core/modules_test.clj+++ b/dev/test/metabase/core/modules_test.clj@@ -7,6 +7,7 @@[clojure.test :refer :all][dev.deps-graph][dev.model-boundary-config]+ [hooks.common.modules :as modules][metabase.util.json :as json][rewrite-clj.node :as n][rewrite-clj.parser :as r.parser]@@ -37,13 +38,27 @@(deftest all-modules-have-teams-test(testing "All modules should have a valid :team owner"- (let [teams (teams)]- (doseq [[module config] (modules-config)]+ (let [teams (teams)+ config (modules-config)]+ (doseq [module (keys config)+ :let [team (dev.deps-graph/module-team config module)]](testing (format "\n'%s' module" module)- (is (or (contains? teams (:team config))- (contains? teams-to-reassign (:team config)))+ (is (or (contains? teams team)+ (contains? teams-to-reassign team))"Should have a valid :team key"))))))+(deftest ^:parallel module-team-test+ (let [config '{parent {:team "Parent"}+ parent.child {}+ parent.child.leaf {:team "Leaf"}+ enterprise/parent {}+ enterprise/standalone {:team "Enterprise"}}]+ (are [module team] (= team (dev.deps-graph/module-team config module))+ 'parent.child "Parent" ; inherited from the nearest configured ancestor+ 'parent.child.leaf "Leaf" ; an explicit team overrides its ancestors+ 'enterprise/parent "Parent" ; an EE companion inherits from its OSS parent+ 'enterprise/standalone "Enterprise")))+(defn- modules-config-zipper"Return a zipper pointing to the modules config map node (the value of the `:metabase/modules` key)."[]@@ -173,8 +188,59 @@(testing (format "Remove %s from %s" (pr-str extraneous) (pr-str ks))(is (empty? extraneous)))))))+(deftest ^:parallel uses-references-must-be-namable-test+ (testing "every module named in :uses is visible to the caller"+ (let [config (dev.deps-graph/kondo-config)]+ (doseq [[caller {:keys [uses]}] config+ :when (set? uses)+ target uses+ ;; The staleness test reports undeclared targets.+ :when (contains? config target)]+ (testing (format "\n[%s :uses %s]" caller target)+ (is (nil? (modules/namability-error config caller target))))))))++(deftest ^:parallel ns-prefixes-test+ (let [config (dev.deps-graph/kondo-config)]+ (testing "explicit prefixes are strings under a Metabase namespace root"+ (doseq [[module {:keys [ns-prefix]}] config+ :when (some? ns-prefix)]+ (is (and (string? ns-prefix)+ (re-find #"^metabase(?:-enterprise)?\." ns-prefix))+ (format "Module %s has :ns-prefix %s." module (pr-str ns-prefix)))))+ (testing "effective prefixes are unique"+ (doseq [[prefix claimants] (group-by #(modules/module-ns-prefix config %) (keys config))+ :when (> (count claimants) 1)]+ (is (= 1 (count claimants))+ (format "Modules %s share :ns-prefix %s." (pr-str (sort claimants)) (pr-str prefix)))))))++(deftest ^:parallel nested-modules-have-declared-parents-test+ (testing "every nested module has a declared direct parent"+ (let [config (dev.deps-graph/kondo-config)]+ (doseq [module (keys config)+ :let [parent (modules/parent-module config module)]+ :when parent]+ (testing (format "\n%s is nested under %s" module parent)+ (is (contains? config parent)+ (format "Declare parent module %s before declaring nested module %s."+ parent+ module)))))))++(deftest ^:parallel module-exports-are-declared-direct-children-test+ (testing "every :module-exports entry names a declared direct child"+ (let [config (dev.deps-graph/kondo-config)]+ (doseq [[parent module-config] config+ child (:module-exports module-config)]+ (testing (format "\n[%s :module-exports %s]" parent child)+ (is (contains? config child)+ (format "Exported child %s is not a declared module." child))+ (is (= parent (modules/parent-module config child))+ (format "%s may only export direct children; %s has parent %s."+ parent+ child+ (modules/parent-module config child))))))))+(defn- rest-module? [module]- (str/ends-with? module "-rest"))+ (re-find #"[.-]rest$" (str module)))(deftest do-not-use-rest-modules-in-other-modules-test(doseq [[module {:keys [uses], :as _config}] (dev.deps-graph/kondo-config)@@ -186,7 +252,7 @@used-modulemoduleused-module- (symbol (str/replace used-module #"-rest$" ""))))))+ (symbol (str/replace used-module #"[.-]rest$" ""))))));;;; Model boundary testsdiff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.cljindex b7c25053d5f..15451a983be 100644--- a/mage/src/mage/modules.clj+++ b/mage/src/mage/modules.clj@@ -1,8 +1,10 @@(ns mage.modules(:require[clojure.edn :as edn]+ [clojure.java.io :as io][clojure.set :as set][clojure.string :as str]+ [hooks.common.modules :as modules][mage.be-dev :as be-dev][mage.color :as c][mage.util :as u]))@@ -22,35 +24,73 @@;;; TODO (Cam 2025-11-07) changes to test files should only cause us to run tests for that module as well, not;;; everything that depends on that module directly or indirectly in `src`-(defn- file->module [filename]+(defn- file->ns-symbol [filename]+ (when (re-find #"^(?:(?:src|test)/metabase|enterprise/backend/(?:src|test)/metabase_enterprise)/" filename)+ (-> filename+ (str/replace #"^(?:enterprise/backend/)?(?:src|test)/" "")+ (str/replace #"\.[^./]+$" "")+ (str/replace "/" ".")+ (str/replace "_" "-")+ symbol)))++(defn- file->module [prefix->module filename](or+ (some->> (file->ns-symbol filename) (modules/declared-module prefix->module))+ ;; otherwise a file inside a directory belongs to the (undeclared) module that directory names(when-let [[_match module] (re-matches #"^(?:(?:src)|(?:test))/metabase/([^/]+)/.*$" filename)](symbol (str/replace module #"_" "-")))(when-let [[_match module] (re-matches #"^enterprise/backend/(?:(?:src)|(?:test))/metabase_enterprise/([^/]+)/.*$" filename)](symbol "enterprise" (str/replace module #"_" "-")))))+(defn- read-modules-config []+ (-> (with-open [r (java.io.PushbackReader. (java.io.FileReader. ".clj-kondo/config/modules/config.edn"))]+ (edn/read r))+ :metabase/modules))+(defn- updated-files->updated-modules [updated-files]- (into (sorted-set)- (keep file->module)- updated-files))+ (let [prefix->module (modules/build-prefix->module (read-modules-config))]+ (into (sorted-set)+ (keep #(file->module prefix->module %))+ updated-files)))(defn- updated-modules [git-ref](let [git-ref (or git-ref "master")updated-files (u/updated-files git-ref)](updated-files->updated-modules updated-files)))-(defn- module->test-directory- [module]- (case (namespace module)- "enterprise" (str "enterprise/backend/test/metabase_enterprise/" (str/replace (name module) #"-" "_"))- nil (str "test/metabase/" (str/replace (name module) #"-" "_"))))+(def ^:private backend-test-source-file-extensions+ [".clj" ".cljc"])++(defn- module->test-path-prefix [modules-config module]+ (let [ns-prefix (modules/module-ns-prefix modules-config module)]+ (str (when (str/starts-with? ns-prefix "metabase-enterprise.") "enterprise/backend/")+ "test/"+ (-> ns-prefix (str/replace "." "/") (str/replace "-" "_")))))++(defn- module->test-paths [modules-config module]+ (let [prefix->module (modules/build-prefix->module modules-config)+ path-prefix (module->test-path-prefix modules-config module)+ test-dir (io/file path-prefix)+ test-files (concat+ (for [extension backend-test-source-file-extensions+ :let [file (io/file (str path-prefix "_test" extension))]+ :when (.isFile file)]+ file)+ (when (.isDirectory test-dir)+ (for [file (file-seq test-dir)+ :when (and (.isFile ^java.io.File file)+ (some #(str/ends-with? (str file) %)+ backend-test-source-file-extensions))]+ file)))]+ (into (sorted-set)+ (comp (filter #(= module (file->module prefix->module (str %))))+ (map str))+ test-files)))(defn- dependencies"Read out the Kondo config for the modules linter; return a map of module => set of modules it directly depends on."[]- (let [config (-> (with-open [r (java.io.PushbackReader. (java.io.FileReader. ".clj-kondo/config/modules/config.edn"))]- (edn/read r))- :metabase/modules+ (let [config (-> (read-modules-config);; ignore the config for [[metabase.connection-pool]] which comes from one of our libraries.(dissoc 'connection-pool))](into (sorted-map)@@ -202,7 +242,8 @@(defn cli-print-affected-modules"CLI entry point: print modules affected by changes since `git-ref`, plus driver-test guidance."[[git-ref, :as _command-line-args]]- (let [deps (dependencies)+ (let [modules-config (read-modules-config)+ deps (dependencies)updated (updated-modules git-ref)affected (affected-modules deps updated)driver-deps-affected? (not (contains? (unaffected-modules deps updated) 'driver))]@@ -212,10 +253,102 @@(println "You can run tests for these modules and all downstream modules as follows:")(println)(println)- (printf "clojure -X :dev:ee:ee-dev:test :only '%s'\n" (pr-str (mapv module->test-directory affected)))+ (printf "clojure -X :dev:ee:ee-dev:test :only '%s'\n"+ (pr-str (into [] (mapcat #(module->test-paths modules-config %)) affected)))(flush)(u/exit 0)))+;;;; =============================================================================+;;;; Module tree+;;;; =============================================================================++(defn- module->tree-path+ "The path segments used to display `module` in the tree.++ An enterprise module appears under its OSS counterpart when one exists.+ Otherwise its full `enterprise/` name appears at the root."+ [modules-config module]+ (let [segments (str/split (name module) #"\.")]+ (if (= (namespace module) "enterprise")+ (if (contains? modules-config (symbol (first segments)))+ ;; Keep an enterprise subtree together under its OSS module.+ (into [(first segments) "enterprise"] (rest segments))+ (into [(str "enterprise/" (first segments))] (rest segments)))+ segments)))++(defn- explicit-ns-prefix+ "Return a module's custom `:ns-prefix`, or `nil` when it uses the default."+ [modules-config module]+ (let [prefix (get-in modules-config [module :ns-prefix])]+ (when (and prefix (not= prefix (modules/default-ns-prefix module)))+ prefix)))++(defn- module-display-tree+ "Build the nested map consumed by [[tree-node-lines]].++ Grouping nodes that are not modules omit `:module`."+ [modules-config]+ (reduce (fn [tree module]+ (update-in tree+ (into [] (mapcat (fn [segment] [:children segment])) (module->tree-path modules-config module))+ assoc :module module))+ {}+ (keys modules-config)))++(defn- sorted-children+ "Sort child nodes alphabetically, with enterprise nodes last."+ [node]+ (sort-by (fn [[segment _]] [(if (or (= segment "enterprise")+ (str/starts-with? segment "enterprise/"))+ 1+ 0)+ segment])+ (:children node)))++(defn- tree-node-lines+ "Render a tree node and its descendants.++ Dashes show depth. Grouping nodes are dimmed. Modules with a custom prefix+ show `*`, or the prefix itself when `show-prefixes?` is true."+ [modules-config show-prefixes? path node]+ (let [depth (dec (count path))+ module (:module node)+ display (str/join "." path)+ line (str (when (pos? depth)+ (str (apply str (repeat depth "-")) " "))+ (if module display (c/dark display))+ (when-let [prefix (and module (explicit-ns-prefix modules-config module))]+ (if show-prefixes?+ (str " " (c/yellow (str "(" prefix ")")))+ (str " " (c/yellow "*")))))]+ (into [line]+ (mapcat (fn [[segment child]]+ (tree-node-lines modules-config show-prefixes? (conj path segment) child)))+ (sorted-children node))))++(defn cli-print-module-tree+ "Print the module tree, nesting enterprise extensions under their OSS module.++ Marks modules whose namespace prefix differs from their name."+ [{:keys [options] :as _parsed}]+ (let [modules-config (read-modules-config)+ tree (module-display-tree modules-config)+ roots (cond->> (sorted-children tree)+ (:nested-only options) (filter (fn [[_ node]] (seq (:children node)))))+ starred (count (keep #(explicit-ns-prefix modules-config %) (keys modules-config)))]+ (doseq [[segment node] roots+ line (tree-node-lines modules-config (:prefixes options) [segment] node)]+ (println line))+ (println)+ (println (c/dark (str (count modules-config) " modules, "+ (count (filter #(= (namespace %) "enterprise") (keys modules-config)))+ " enterprise"+ (when (pos? starred)+ (str ", " starred " custom prefixes ("+ (if (:prefixes options) "shown in parentheses" "marked with *")+ ")")))))+ (u/exit 0)))+(defn- changes-important-file-for-drivers?"Whether we should always run driver tests because `updated-files` touches something important like`deps.edn`."@@ -230,13 +363,19 @@(defn driver-deps-affected?"Returns true if any of `trigger-modules` are affected by the changed modules.- 1-arity and 2-arity use [[default-modules-which-trigger-drivers]] for backwards compatibility."+ 1-arity and 2-arity trigger on the union of [[default-modules-which-trigger-drivers]] and+ [[modules-triggering-cloud-drivers]]."([modules](driver-deps-affected? (dependencies) modules))([deps modules](driver-deps-affected? deps modules (set/union default-modules-which-trigger-driversmodules-triggering-cloud-drivers)))([deps modules trigger-modules]+ ;; Fail clearly when a renamed trigger is missing from the config.+ (when-let [missing (seq (remove #(contains? deps %) trigger-modules))]+ (throw (ex-info (str "Driver-trigger module(s) not declared in the module config: "+ (pr-str missing))+ {:missing missing})))(let [unaffected (unaffected-modules deps (remove driver-affecting-overrides modules))](boolean(some #(not (contains? unaffected %)) trigger-modules)))))diff --git a/mage/test/mage/modules_test.clj b/mage/test/mage/modules_test.cljindex 2bdff4a5e3c..ded07593e59 100644--- a/mage/test/mage/modules_test.clj+++ b/mage/test/mage/modules_test.clj@@ -3,6 +3,8 @@Run `mage -driver-decisions -h` to see the priority order."(:require[clojure.test :refer [deftest is testing]]+ [hooks.common.modules :as modules]+ [mage.color][mage.modules]));; Referenced by core_test.clj to ensure namespace is loaded@@ -325,3 +327,55 @@(is (-> [changed-file]mage.modules/updated-files->updated-modulesmage.modules/driver-deps-affected?)))))++(deftest module-tree-lines-test+ (let [config '{lib {}+ lib.be {:ns-prefix "metabase.lib-be"}+ transforms {}+ transforms.base {:ns-prefix "metabase.transforms-base"}+ transforms.base.deep {}+ transforms.python {:ns-prefix "metabase.transforms-python"}+ enterprise-tools {}+ enterprise/transforms {}+ enterprise/transforms.python {:ns-prefix "metabase-enterprise.transforms-python"}+ enterprise/billing {}}+ lines (binding [mage.color/*disable-colors* true]+ (into []+ (mapcat (fn [[segment node]]+ (#'mage.modules/tree-node-lines config false [segment] node)))+ (#'mage.modules/sorted-children (#'mage.modules/module-display-tree config))))]+ (testing "the tree sorts and marks modules with custom prefixes"+ (is (= ["enterprise-tools"+ "lib"+ "- lib.be *"+ "transforms"+ "- transforms.base *"+ "-- transforms.base.deep"+ "- transforms.python *"+ "- transforms.enterprise"+ "-- transforms.enterprise.python *"+ "enterprise/billing"]+ lines)))))++(deftest module-tree-enterprise-default-prefix-not-starred-test+ (testing "default enterprise prefixes are not marked as custom"+ (is (nil? (#'mage.modules/explicit-ns-prefix '{enterprise/billing {}} 'enterprise/billing)))+ (is (nil? (#'mage.modules/explicit-ns-prefix '{enterprise/billing {:ns-prefix "metabase-enterprise.billing"}}+ 'enterprise/billing)))+ (is (= "metabase.lib-be"+ (#'mage.modules/explicit-ns-prefix '{lib.be {:ns-prefix "metabase.lib-be"}} 'lib.be)))))++(deftest dotted-module-files-mark-correct-module-changes+ (testing "dotted module files resolve to the dotted module when its prefix exists"+ (let [prefix->module (modules/build-prefix->module {'lib.schema {}})]+ (is (= 'lib.schema+ (#'mage.modules/file->module prefix->module "test/metabase/lib/schema_test.cljc")))+ (is (= 'lib.schema+ (#'mage.modules/file->module prefix->module "src/metabase/lib/schema/config.edn"))))))++(deftest top-level-files-belong-only-to-declared-modules-test+ (testing "a file directly under metabase/ belongs to a module only when a declared prefix owns its namespace"+ (let [prefix->module (modules/build-prefix->module '{driver {}})]+ (is (= 'driver (#'mage.modules/file->module prefix->module "src/metabase/driver.clj")))+ (is (nil? (#'mage.modules/file->module prefix->module "test/metabase/test_runner.clj")))+ (is (nil? (#'mage.modules/file->module prefix->module "src/metabase/DO_NOT_ADD_NEW_FILES_HERE.txt"))))))diff --git a/src/metabase/util/log.clj b/src/metabase/util/log.cljindex a68ca6b9611..45fec4d1a1e 100644--- a/src/metabase/util/log.clj+++ b/src/metabase/util/log.clj@@ -109,23 +109,42 @@`(with-thread-context-fn ~context-map(fn [] ~@body)))-(let [config (-> (if (config/jar?)- (io/resource "metabase/config/modules.edn")- (io/file ".clj-kondo/config/modules/config.edn"))- slurp edn/read-string :metabase/modules)- first-segment (fn first-segment [ns-sym]- (-> (str/split (name ns-sym) #"\.") second))- chop (fn chop [ns-sym]- (if (str/starts-with? (name ns-sym) "metabase-enterprise")- (symbol "enterprise" (first-segment ns-sym))- (symbol (first-segment ns-sym))))]+(defn- module-for-ns+ "Copy of `hooks.common.modules/declared-module`, which the packaged application cannot load."+ [prefix->module ns-sym]+ (loop [candidate (str/replace (str ns-sym) #"-test$" "")]+ (or (get prefix->module candidate)+ (when-let [dot (str/last-index-of candidate ".")]+ (recur (subs candidate 0 dot))))))++(let [config (-> (if (config/jar?)+ (io/resource "metabase/config/modules.edn")+ (io/file ".clj-kondo/config/modules/config.edn"))+ slurp edn/read-string :metabase/modules)+ prefix->module (into {}+ (map (fn [[module {:keys [ns-prefix]}]]+ [(or ns-prefix+ (str (if (= "enterprise" (namespace module)) "metabase-enterprise." "metabase.")+ (name module)))+ module]))+ config)+ parent (fn [module]+ (let [module-name (name module)]+ (if-let [dot (str/last-index-of module-name ".")]+ (symbol (namespace module) (subs module-name 0 dot))+ (when (= "enterprise" (namespace module))+ (let [oss (symbol module-name)]+ (when (contains? config oss)+ oss))))))](defn ns->team*- "Chops the namespace symbol to look up which team owns this namespace in the module config. Assumes that the first- segment of the namespace is a module. Should be memoized for speed."+ "Return the nearest configured team for the namespace's module.++ The caller should memoize this lookup."[ns-sym](if ('#{metabase.server.middleware.log} ns-sym)::skip- (-> ns-sym chop config :team))))+ (when-let [module (module-for-ns prefix->module ns-sym)]+ (some #(get-in config [% :team]) (take-while some? (iterate parent module)))))))(let [attribution (if (config/config-bool :mb-log-team-attribution)(memoize ns->team*)diff --git a/test/metabase/core/modules_consistency_test.clj b/test/metabase/core/modules_consistency_test.cljnew file mode 100644index 00000000000..aae4459d320--- /dev/null+++ b/test/metabase/core/modules_consistency_test.clj@@ -0,0 +1,165 @@+(ns metabase.core.modules-consistency-test+ "Cross-checks module resolution across runtime boundaries.++ Mage infers a namespace from each changed path and runs under Babashka, so+ these tests invoke it through `bb`. Logging ships in the application jar and+ cannot load the shared hook, so it keeps a small resolver copy."+ (:require+ [clojure.edn :as edn]+ [clojure.java.shell :as shell]+ [clojure.string :as str]+ [clojure.test :refer :all]+ [dev.deps-graph]+ [hooks.common.modules :as modules]+ [metabase.util.log :as log]))++(set! *warn-on-reflection* true)++(def ^:private babashka+ (delay (some (fn [executable]+ (try+ (when (zero? (:exit (shell/sh executable "--version")))+ executable)+ (catch java.io.IOException _+ nil)))+ ["./bin/bb" "bb"])))++(defn- bb-eval+ "Evaluate `form` with `mage.modules` loaded in Babashka."+ [form]+ ;; Missing Babashka must fail because this is the only cross-runtime check.+ (let [bb (or @babashka+ (throw (ex-info "No working babashka at ./bin/bb or on PATH. Run any ./bin/mage task once to install it."+ {})))+ expr (pr-str `(do (require 'mage.modules) (~'prn ~form)))+ {:keys [exit out err]} (shell/sh bb "-e" expr)]+ (when-not (zero? exit)+ (throw (ex-info "babashka evaluation failed" {:exit exit, :stderr err, :expr expr})))+ (edn/read-string out)))++(defn- mage-test-paths [modules-config module]+ (bb-eval `(#'mage.modules/module->test-paths '~modules-config '~module)))++(deftest ^:parallel mage-file-resolution-test+ (testing "Mage and the linter resolve equivalent files and namespaces to the same module"+ (let [config '{actions {}+ actions.rest {:ns-prefix "metabase.actions-rest"}+ queries {}+ queries.rest {:ns-prefix "metabase.queries-rest"}+ lib {}+ lib.schema {}+ lib.be {:ns-prefix "metabase.lib-be"}+ enterprise/transforms {}+ enterprise/transforms.python {}}+ cases '[[metabase.actions-rest.api "src/metabase/actions_rest/api.clj" actions.rest]+ [metabase.queries-rest.middleware-test "test/metabase/queries_rest/middleware_test.clj" queries.rest]+ [metabase.lib.schema.util "src/metabase/lib/schema/util.clj" lib.schema]+ [metabase.lib.schema "src/metabase/lib/schema.cljc" lib.schema]+ [metabase.lib.schema-test "test/metabase/lib/schema_test.cljc" lib.schema]+ [metabase.lib.schema.config "src/metabase/lib/schema/config.edn" lib.schema]+ [metabase.lib-be.core "src/metabase/lib_be/core.clj" lib.be]+ [metabase.lib-bert.core "src/metabase/lib_bert/core.clj" lib-bert]+ [metabase.query-processor.middleware.permissions+ "src/metabase/query_processor/middleware/permissions.clj" query-processor]+ [metabase-enterprise.transforms.python.runner+ "enterprise/backend/src/metabase_enterprise/transforms/python/runner.clj"+ enterprise/transforms.python]]+ from-files (bb-eval `(mapv (partial #'mage.modules/file->module (modules/build-prefix->module '~config))+ ~(mapv second cases)))]+ (doseq [[[ns-symb file want] from-file] (map vector cases from-files)]+ (testing (str ns-symb " <-> " file)+ (is (= want+ (modules/module {:metabase/modules config} ns-symb)+ from-file)))))))++;; Use real directories but avoid fixed filenames, so ordinary test renames do+;; not break ownership checks.++(defn- all-under?+ "Whether `paths` is non-empty and every path is `dir`'s test file or lies below `dir`."+ [dir paths]+ (and (seq paths)+ (every? (fn [^String p]+ (or (re-matches (re-pattern (str "\\Q" dir "\\E_test\\.\\w+")) p)+ (.startsWith p (str dir "/"))))+ paths)))++(deftest ^:parallel dotted-module-test-paths-test+ (testing "dotted modules find tests under default and custom prefixes"+ (let [modules-config '{actions.rest {:ns-prefix "metabase.actions-rest"}+ lib.schema {}}]+ (testing "dev.deps-graph finds both forms"+ (is (all-under? "test/metabase/actions_rest"+ (#'dev.deps-graph/module->test-files modules-config 'actions.rest))+ "an explicit :ns-prefix resolves into the prefix's directory")+ (let [schema-tests (#'dev.deps-graph/module->test-files modules-config 'lib.schema)]+ (is (all-under? "test/metabase/lib/schema" schema-tests)+ "a default-derived prefix resolves into the dotted module's directory")+ ;; Check both file shapes; `all-under?` alone would accept either one.+ (is (some #(re-matches #"test/metabase/lib/schema_test\.\w+" %) schema-tests)+ "the module's own test file is discovered")+ (is (some #(.startsWith ^String % "test/metabase/lib/schema/") schema-tests)+ "and so are tests nested beneath it")))+ (testing "mage.modules emits the same directories"+ (is (all-under? "test/metabase/actions_rest" (mage-test-paths modules-config 'actions.rest)))+ (is (all-under? "test/metabase/lib/schema" (mage-test-paths modules-config 'lib.schema)))))))++(deftest ^:parallel parent-test-discovery-excludes-declared-child-tests-test+ (let [child-dir "test/metabase/query_processor/middleware/cache_backend"+ modules-config '{query-processor {}+ query-processor.cache-backend {:ns-prefix "metabase.query-processor.middleware.cache-backend"}}+ in-child? #(.startsWith ^String % (str child-dir "/"))]+ (testing "dev dependency tooling assigns a nested test only to its exact owner"+ (is (all-under? child-dir (#'dev.deps-graph/module->test-files modules-config 'query-processor.cache-backend)))+ (is (not-any? in-child? (#'dev.deps-graph/module->test-files modules-config 'query-processor))))+ (testing "Mage affected-test paths assign a nested test only to its exact owner"+ (is (all-under? child-dir (mage-test-paths modules-config 'query-processor.cache-backend)))+ (is (not-any? in-child? (mage-test-paths modules-config 'query-processor))))))++(deftest ^:parallel mage-affected-tests-are-jvm-loadable-test+ (let [paths (mage-test-paths '{lib {}} 'lib)]+ (is (seq paths) "expected at least one test path")+ (is (not-any? #(str/ends-with? ^String % ".cljs") paths)+ "mage affected-test paths feed a JVM runner, so ClojureScript-only tests must not appear")))++(deftest ^:parallel explicit-prefix-map-overloads-remain-pure-test+ (testing "path helpers use the supplied prefix map instead of live config"+ (let [deps [{:module 'lib.schema.util+ :deps []+ :filename "src/metabase/lib/schema/util/core.cljc"}]+ prefix->mod {"metabase.lib.schema.util" 'lib.schema.util}]+ (is (= #{"src/metabase/lib/schema/util/core.cljc"}+ (set (dev.deps-graph/test-filenames->relevant-source-filenames+ deps+ prefix->mod+ ["test/metabase/lib/schema/util/core_test.cljc"]))))+ (is (= #{"test/metabase/lib/schema/util_test.cljc"}+ (set (dev.deps-graph/source-filenames->relevant-test-filenames+ deps+ {'lib.schema.util {}}+ prefix->mod+ ["src/metabase/lib/schema/util/core.cljc"])))))))++(deftest ^:parallel exact-dotted-module-test-files-round-trip-test+ (testing "a dotted module's top-level test file resolves back to that module"+ (let [deps [{:module 'lib.schema+ :deps []+ :filename "src/metabase/lib/schema.cljc"}+ {:module 'lib+ :deps []+ :filename "src/metabase/lib.clj"}]]+ (is (= #{"src/metabase/lib/schema.cljc"}+ (set (dev.deps-graph/test-filenames->relevant-source-filenames+ deps+ {"metabase.lib.schema" 'lib.schema}+ ["test/metabase/lib/schema_test.cljc"])))))))++(deftest ^:parallel log-team-attribution-agrees-with-deps-graph-test+ (testing "logging and dev tooling assign the same team to every module"+ (let [config (dev.deps-graph/kondo-config)]+ (is (< 100 (count config)) "expected the full module config")+ (doseq [module (sort (keys config))+ :let [ns-symb (symbol (modules/module-ns-prefix config module))]]+ (testing (str "\n" ns-symb)+ (is (= (dev.deps-graph/module-team config module)+ (log/ns->team* ns-symb))))))))diff --git a/test/metabase/core/modules_nesting_test.clj b/test/metabase/core/modules_nesting_test.cljnew file mode 100644index 00000000000..f2037c07575--- /dev/null+++ b/test/metabase/core/modules_nesting_test.clj@@ -0,0 +1,460 @@+(ns metabase.core.modules-nesting-test+ "Tests nested-module resolution and access rules against fixture configs."+ (:require+ [clojure.test :refer :all]+ [dev.deps-graph]+ [hooks.common.modules :as modules]))++(set! *warn-on-reflection* true)++;;;; -------------------------------------------------------------------------+;;;; Namespace resolution+;;;; -------------------------------------------------------------------------++(deftest ^:parallel module-resolution-longest-prefix-test+ (let [config {:metabase/modules {'lib {}+ 'lib.schema {}}}]+ (testing "resolves to the most-specific declared ancestor"+ (is (= 'lib.schema (modules/module config 'metabase.lib.schema.foo)))+ (is (= 'lib.schema (modules/module config 'metabase.lib.schema.nested.deeper))))+ (testing "falls back to parent when deeper child isn't declared"+ (is (= 'lib (modules/module config 'metabase.lib.core))))+ (testing "an undeclared metabase namespace resolves to a module named for its first segment"+ (is (= 'query-processor (modules/module config 'metabase.query-processor.foo)))+ (is (= 'enterprise/transforms (modules/module config 'metabase-enterprise.transforms.core))))+ (testing "namespaces outside metabase resolve to nil"+ (is (nil? (modules/module config 'clojure.core)))+ (is (nil? (modules/module config 'clj-kondo.impl.config))))))++(deftest ^:parallel module-resolution-enterprise-dotted-children-test+ (let [config {:metabase/modules {'enterprise/transforms {}+ 'enterprise/transforms.python {}}}]+ (is (= 'enterprise/transforms.python+ (modules/module config 'metabase-enterprise.transforms.python.runner)))+ (is (= 'enterprise/transforms+ (modules/module config 'metabase-enterprise.transforms.core)))))++(deftest ^:parallel module-resolution-three-level-nesting-test+ (let [config {:metabase/modules {'outer {}+ 'outer.middle {}+ 'outer.middle.deepest {}}}]+ (is (= 'outer.middle.deepest (modules/module config 'metabase.outer.middle.deepest.foo)))+ (is (= 'outer.middle (modules/module config 'metabase.outer.middle.other)))+ (is (= 'outer (modules/module config 'metabase.outer.top)))))++(deftest ^:parallel module-resolution-test-namespace-suffix-stripping-test+ (testing "a trailing `-test` resolves like the namespace it tests"+ (let [config {:metabase/modules {'driver {}+ 'lib {}+ 'lib.schema {}}}]+ (is (= 'driver (modules/module config 'metabase.driver-test)))+ (is (= 'lib.schema (modules/module config 'metabase.lib.schema-test))))))++;;;; -------------------------------------------------------------------------+;;;; Custom namespace prefixes+;;;; -------------------------------------------------------------------------++(deftest ^:parallel default-ns-prefix-test+ (is (= "metabase.lib" (modules/default-ns-prefix 'lib)))+ (is (= "metabase.lib.schema" (modules/default-ns-prefix 'lib.schema)))+ (is (= "metabase.query-processor" (modules/default-ns-prefix 'query-processor)))+ (is (= "metabase-enterprise.transforms" (modules/default-ns-prefix 'enterprise/transforms)))+ (is (= "metabase-enterprise.transforms.python" (modules/default-ns-prefix 'enterprise/transforms.python))))++(deftest ^:parallel module-ns-prefix-explicit-override-test+ (let [modules '{lib {}+ lib.be {:ns-prefix "metabase.lib-be"}+ lib.schema {}}]+ (is (= "metabase.lib" (modules/module-ns-prefix modules 'lib)))+ (is (= "metabase.lib-be" (modules/module-ns-prefix modules 'lib.be)))+ (is (= "metabase.lib.schema" (modules/module-ns-prefix modules 'lib.schema)))))++(deftest ^:parallel module-resolution-explicit-ns-prefix-test+ (let [config {:metabase/modules {'lib {}+ 'lib.schema {}+ 'lib.be {:ns-prefix "metabase.lib-be"}+ 'lib.legacy-mbql {:ns-prefix "metabase.legacy-mbql"}}}]+ (testing "hyphenated source namespace resolves to the nested module"+ (is (= 'lib.be (modules/module config 'metabase.lib-be.core)))+ (is (= 'lib.be (modules/module config 'metabase.lib-be.models.query))))+ (testing "unrelated hyphenated namespace resolves to its own nested module"+ (is (= 'lib.legacy-mbql (modules/module config 'metabase.legacy-mbql.util)))+ (is (= 'lib.legacy-mbql (modules/module config 'metabase.legacy-mbql.schema.macros))))+ (testing "default-prefixed sibling still resolves correctly"+ (is (= 'lib.schema (modules/module config 'metabase.lib.schema.foo))))+ (testing "parent resolves to itself for its own namespaces"+ (is (= 'lib (modules/module config 'metabase.lib.core))))))++(deftest ^:parallel module-resolution-segment-boundary-test+ (let [config {:metabase/modules {'lib {}+ 'lib.be {:ns-prefix "metabase.lib-be"}}}]+ (testing "`metabase.lib-be.foo` matches `metabase.lib-be` at segment boundary"+ (is (= 'lib.be (modules/module config 'metabase.lib-be.foo))))+ (testing "`metabase.lib-bert.foo` does not match `metabase.lib-be` mid-segment"+ (is (= 'lib-bert (modules/module config 'metabase.lib-bert.foo))))+ (testing "exact match works"+ (is (= 'lib.be (modules/module config 'metabase.lib-be))))))++(deftest ^:parallel build-prefix->module-test+ (is (= {"metabase.lib" 'lib+ "metabase.lib-be" 'lib.be+ "metabase.query-processor" 'query-processor}+ (modules/build-prefix->module '{lib {}+ lib.be {:ns-prefix "metabase.lib-be"}+ query-processor {}}))))++(deftest ^:parallel descendant-api-generation-is-monotonic-test+ (let [deps [{:namespace 'metabase.parent.child.core+ :module 'parent.child+ :deps [{:namespace 'metabase.parent.internal+ :module 'parent}]}]+ base-config '{parent {:api #{}}+ parent.child {:uses #{parent}}}]+ (testing "a new descendant-only use does not become public API"+ (is (= #{}+ (dev.deps-graph/externally-used-namespaces-ignoring-friends+ deps base-config 'parent))))+ (testing "a pre-existing API remains stable while a descendant still uses it"+ (is (= '#{metabase.parent.internal}+ (dev.deps-graph/externally-used-namespaces-ignoring-friends+ deps (assoc-in base-config ['parent :api] '#{metabase.parent.internal}) 'parent))))))++(deftest ^:parallel simulate-rename-preserves-nested-module-ownership-test+ (let [prefix->module {"metabase.parent" 'parent+ "metabase.parent.child" 'parent.child}+ deps [{:namespace 'metabase.consumer.core+ :module 'consumer+ :deps [{:namespace 'metabase.old.core+ :module 'old}]}]+ renamed (#'dev.deps-graph/simulate-rename+ deps prefix->module {'metabase.old.core 'metabase.parent.child.core})]+ (is (= 'parent.child (:module (first (:deps (first renamed))))))))++;;;; -------------------------------------------------------------------------+;;;; Module tree+;;;; -------------------------------------------------------------------------++(deftest ^:parallel parent-module-test+ (testing "a dotted name's parent is its prefix"+ (is (= 'lib (modules/parent-module {} 'lib.schema)))+ (is (= 'lib.schema (modules/parent-module {} 'lib.schema.foo)))+ (is (= 'enterprise/transforms (modules/parent-module '{transforms {}} 'enterprise/transforms.python))))+ (testing "top-level modules have no parent"+ (is (nil? (modules/parent-module {} 'lib)))+ (is (nil? (modules/parent-module {} 'enterprise/internal-stats))))+ (testing "`enterprise/X` sits under OSS `X` only when `X` is declared"+ (is (= 'internal-stats (modules/parent-module '{internal-stats {}} 'enterprise/internal-stats)))+ (is (nil? (modules/parent-module '{other-module {}} 'enterprise/internal-stats)))))++(deftest ^:parallel descendant-of?-test+ (let [modules '{lib {}, lib.schema {}, lib.schema.foo {}, transforms {}, enterprise/transforms {}}]+ (is (modules/descendant-of? modules 'lib.schema.foo 'lib))+ (is (modules/descendant-of? modules 'lib 'lib) "a module sits in its own subtree")+ (is (not (modules/descendant-of? modules 'lib 'lib.schema)) "an ancestor is not a descendant")+ (is (modules/descendant-of? modules 'enterprise/transforms 'transforms))+ (is (not (modules/descendant-of? modules 'lib.schema 'transforms)))))++(deftest ^:parallel blocking-export-test+ (let [blocking-export #'modules/blocking-export+ leaf-config '{outer {}, outer.a {}, outer.a.leaf {}}]+ (testing "top-level modules are never blocked"+ (is (nil? (blocking-export '{lib {}} 'lib))))+ (testing "an unexported child is blocked at its parent"+ (is (= '{:ancestor lib, :child lib.schema}+ (blocking-export '{lib {}, lib.schema {}} 'lib.schema))))+ (testing "an exported child is not"+ (is (nil? (blocking-export '{lib {:module-exports #{lib.schema}}, lib.schema {}} 'lib.schema))))+ (testing "each export widens the scope by exactly one level"+ (is (= '{:ancestor outer.a, :child outer.a.leaf}+ (blocking-export leaf-config 'outer.a.leaf)))+ (is (= '{:ancestor outer, :child outer.a}+ (blocking-export (assoc-in leaf-config ['outer.a :module-exports] '#{outer.a.leaf}) 'outer.a.leaf)))+ (is (nil? (blocking-export (-> leaf-config+ (assoc-in ['outer.a :module-exports] '#{outer.a.leaf})+ (assoc-in ['outer :module-exports] '#{outer.a}))+ 'outer.a.leaf))))+ (testing "`X` exports its declared `enterprise/X` child implicitly"+ (is (nil? (blocking-export '{cache {}, enterprise/cache {}} 'enterprise/cache))))+ (testing "a dotted child of an `enterprise/` module needs an explicit export"+ (is (= '{:ancestor enterprise/transforms, :child enterprise/transforms.python}+ (blocking-export '{transforms {}, enterprise/transforms {}, enterprise/transforms.python {}}+ 'enterprise/transforms.python))))))++;;;; -------------------------------------------------------------------------+;;;; Access rules+;;;; -------------------------------------------------------------------------++(deftest ^:parallel usage-error-parent-needs-explicit-uses-and-api-test+ (testing "a parent must declare its child in :uses and use the child's :api"+ (let [config-without-uses+ {:metabase/modules {'lib {:uses #{}}+ 'lib.schema {:api #{'metabase.lib.schema.foo}+ :uses #{}}}}]+ (is (some? (modules/usage-error config-without-uses 'lib 'metabase.lib.schema.foo))+ "lib does not declare :uses #{lib.schema}"))+ (let [config-with-uses+ {:metabase/modules {'lib {:uses #{'lib.schema}}+ 'lib.schema {:api #{'metabase.lib.schema.foo}+ :uses #{}}}}]+ (testing "namespace in lib.schema's :api — allowed"+ (is (nil? (modules/usage-error config-with-uses 'lib 'metabase.lib.schema.foo))))+ (testing "namespace not in lib.schema's :api — denied even though lib is the parent"+ (is (some? (modules/usage-error config-with-uses 'lib 'metabase.lib.schema.private-ns)))))))++(deftest ^:parallel usage-error-siblings-need-explicit-uses-and-api-test+ (testing "siblings must declare :uses and use each other's :api"+ (let [config-without-uses+ {:metabase/modules {'lib {}+ 'lib.schema {:api #{'metabase.lib.schema.foo}+ :uses #{}}+ 'lib.be {:api :any+ :uses #{}}}}]+ (is (some? (modules/usage-error config-without-uses 'lib.be 'metabase.lib.schema.foo))+ "lib.be does not declare :uses #{lib.schema}"))+ (let [config-with-uses+ {:metabase/modules {'lib {}+ 'lib.schema {:api #{'metabase.lib.schema.foo}+ :uses #{}}+ 'lib.be {:api :any+ :uses #{'lib.schema}}}}]+ (is (nil? (modules/usage-error config-with-uses 'lib.be 'metabase.lib.schema.foo))+ "the namespace is in lib.schema's API")+ (is (some? (modules/usage-error config-with-uses 'lib.be 'metabase.lib.schema.private-ns))+ "the namespace is not in lib.schema's API"))))++(deftest ^:parallel usage-error-child-must-declare-uses-on-parent-test+ (testing "a child must declare its parent in :uses but may use internal namespaces"+ (let [config-without-uses+ {:metabase/modules {'lib {:api #{'metabase.lib.core}+ :uses #{}}+ 'lib.schema {:api :any+ :uses #{}}}}]+ (is (some? (modules/usage-error config-without-uses 'lib.schema 'metabase.lib.core))+ "lib.schema does not declare :uses #{lib}"))+ (let [config-with-uses+ {:metabase/modules {'lib {:api #{'metabase.lib.core}+ :uses #{}}+ 'lib.schema {:api :any+ :uses #{'lib}}}}]+ (testing "namespace in lib's :api — allowed"+ (is (nil? (modules/usage-error config-with-uses 'lib.schema 'metabase.lib.core))))+ (testing "namespace not in lib's :api — also allowed (subtree trust)"+ (is (nil? (modules/usage-error config-with-uses 'lib.schema 'metabase.lib.internal)))))))++(deftest ^:parallel usage-error-encapsulated-grandchild-denied-test+ (testing "an outside module cannot reach an unexported nested module"+ (let [config {:metabase/modules {'lib {:module-exports #{}+ :api #{'metabase.lib.core}+ :uses #{}}+ 'lib.schema {:api #{'metabase.lib.schema.public-ns}+ :uses #{}}+ 'query-processor {:uses #{'lib}+ :api :any}}}]+ (testing "access to a private descendant namespace is denied"+ (is (some? (modules/usage-error config 'query-processor 'metabase.lib.schema.private-ns))))+ (testing "access to lib's own API is allowed"+ (is (nil? (modules/usage-error config 'query-processor 'metabase.lib.core))))+ (testing "access to lib.schema's own :api is also denied — lib does not export lib.schema"+ (is (some? (modules/usage-error config 'query-processor 'metabase.lib.schema.public-ns)))))))++(deftest ^:parallel usage-error-opened-child-allowed-test+ (testing "an outside module can reach a child exported by its parent"+ (let [config {:metabase/modules {'lib {:module-exports #{'lib.schema}+ :api :any}+ 'lib.schema {:api :any+ :uses #{}}+ 'query-processor {:uses #{'lib.schema}+ :api :any}}}]+ (is (nil? (modules/usage-error config 'query-processor 'metabase.lib.schema.foo))))))++(deftest ^:parallel usage-error-subtree-trust-is-unidirectional-test+ (let [config {:metabase/modules {'outer {:uses #{'outer.middle.inner}+ :api #{'metabase.outer.public}}+ 'outer.middle {:api #{}}+ 'outer.middle.inner {:api #{'metabase.outer.middle.inner.public}+ :uses #{'outer}}}}]+ (testing "grandchild → grandparent bypasses :api"+ (is (nil? (modules/usage-error config 'outer.middle.inner 'metabase.outer.internal))))+ (testing "grandparent → grandchild must go through grandchild's :api"+ (is (some? (modules/usage-error config 'outer 'metabase.outer.middle.inner.internal))))+ (testing "grandparent → grandchild can access the grandchild's public :api"+ (is (nil? (modules/usage-error config 'outer 'metabase.outer.middle.inner.public))))))++(deftest ^:parallel usage-error-subtree-trust-does-not-extend-to-cousins-test+ (let [config {:metabase/modules {'outer {}+ 'outer.a {}+ 'outer.a.leaf {:api #{'metabase.outer.a.leaf.public}+ :uses #{}}+ 'outer.b {}+ 'outer.b.leaf {:api :any+ :uses #{'outer.a.leaf}}}}]+ (testing "namespace in cousin's :api — allowed"+ (is (nil? (modules/usage-error config 'outer.b.leaf 'metabase.outer.a.leaf.public))))+ (testing "namespace not in cousin's :api — denied"+ (is (some? (modules/usage-error config 'outer.b.leaf 'metabase.outer.a.leaf.internal))))))++(deftest ^:parallel usage-error-uses-must-name-resolved-module-exactly-test+ (testing "`:uses` must name the resolved module exactly"+ (let [config {:metabase/modules {'lib {:api :any+ :uses #{}}+ 'lib.schema {:api :any+ :uses #{}}+ 'query-processor {:uses #{'lib}+ :api :any}}}]+ (is (some? (modules/usage-error config 'query-processor 'metabase.lib.schema.foo))+ ":uses #{lib} does not cover lib.schema even though lib is its parent")+ (is (nil? (modules/usage-error config 'query-processor 'metabase.lib.core))+ "the namespace resolves directly to lib")+ (is (nil? (modules/usage-error (assoc-in config [:metabase/modules 'query-processor :uses] #{'lib 'lib.schema})+ 'query-processor+ 'metabase.lib.schema.foo))+ "query-processor declares lib.schema explicitly"))))++(deftest ^:parallel usage-error-explicit-empty-api-denies-external-access-test+ (let [config {:metabase/modules {'private-module {:api #{}}+ 'caller {:uses #{'private-module}}}}]+ (testing "an explicit empty API exports no namespaces"+ (is (some? (modules/usage-error config 'caller 'metabase.private-module.api))))+ (testing "`:api :any` remains the unrestricted API sentinel"+ (is (nil? (modules/usage-error (assoc-in config [:metabase/modules 'private-module :api] :any)+ 'caller+ 'metabase.private-module.internal))))+ (testing "friends may still bypass an explicitly empty API"+ (is (nil? (modules/usage-error (assoc-in config [:metabase/modules 'private-module :friends] #{'caller})+ 'caller+ 'metabase.private-module.internal))))))++(deftest ^:parallel usage-error-uses-any-namability-test+ (testing "`:uses :any` may name any module it could name with a set-valued `:uses`"+ (let [config {:metabase/modules {'lib {:api :any}+ 'lib.schema {:api :any}+ 'caller {:uses :any+ :api :any}}}]+ (testing "top-level modules are always visible"+ (is (nil? (modules/usage-error config 'caller 'metabase.lib.core))))+ (testing "a nested child not in its parent's :module-exports is private to its subtree"+ (is (some? (modules/usage-error config 'caller 'metabase.lib.schema.foo))))+ (testing "exporting the child makes it visible everywhere"+ (is (nil? (modules/usage-error (assoc-in config [:metabase/modules 'lib :module-exports] #{'lib.schema})+ 'caller+ 'metabase.lib.schema.foo))))+ (testing "same-subtree callers may name private children"+ (is (nil? (modules/usage-error (assoc-in config [:metabase/modules 'lib.other] {:uses :any, :api :any})+ 'lib.other+ 'metabase.lib.schema.foo)))))))++(deftest ^:parallel usage-error-privacy-scoped-to-nearest-non-opening-ancestor-test+ (testing "An unexported child is private to its nearest non-exporting ancestor's subtree, not the top-level one"+ (let [config {:metabase/modules {'outer {:module-exports #{}+ :uses :any+ :api :any}+ 'outer.a {:module-exports #{}+ :uses :any+ :api :any}+ 'outer.a.leaf {:uses :any+ :api :any}+ 'outer.a.sib {:uses :any+ :api :any}+ 'outer.b {:module-exports #{}+ :uses :any+ :api :any}+ 'outer.b.deep {:uses :any+ :api :any}}}]+ (testing "the nearest non-opening ancestor and its descendants may name it"+ (is (nil? (modules/usage-error config 'outer.a 'metabase.outer.a.leaf.foo)))+ (is (nil? (modules/usage-error config 'outer.a.sib 'metabase.outer.a.leaf.foo))))+ (testing "a cousin sharing only the top-level ancestor may not name it"+ (is (some? (modules/usage-error config 'outer.b 'metabase.outer.a.leaf.foo)))+ (is (some? (modules/usage-error config 'outer.b.deep 'metabase.outer.a.leaf.foo))))+ (testing "an ancestor above the nearest non-opening ancestor may not name it either"+ (is (some? (modules/usage-error config 'outer 'metabase.outer.a.leaf.foo))))+ (testing "exporting the leaf one level up widens the scope to the whole `outer` subtree"+ (let [config (assoc-in config [:metabase/modules 'outer.a :module-exports] #{'outer.a.leaf})]+ (is (nil? (modules/usage-error config 'outer 'metabase.outer.a.leaf.foo)))+ (is (nil? (modules/usage-error config 'outer.b 'metabase.outer.a.leaf.foo)))+ (is (nil? (modules/usage-error config 'outer.b.deep 'metabase.outer.a.leaf.foo)))+ (testing "but not to a module outside the `outer` subtree"+ (let [config (assoc-in config [:metabase/modules 'unrelated] {:uses :any, :api :any})]+ (is (= (str "Module outer.a.leaf is nested and not exported by its ancestors; "+ "unrelated may not use it. Add outer.a to outer's :module-exports, "+ "or move the caller into the outer subtree. "+ "[:metabase/modules outer :module-exports]")+ (modules/usage-error config 'unrelated 'metabase.outer.a.leaf.foo))))))))))++(deftest ^:parallel usage-error-uses-any-rest-module-test+ (testing "`:uses :any` does not allow domain modules to depend on REST modules"+ (let [config {:metabase/modules {'actions {:api :any}+ 'actions.rest {:ns-prefix "metabase.actions-rest"+ :api :any}+ 'caller {:uses :any+ :api :any}}}]+ (is (= (str "Do not use -rest modules (actions.rest) in non-rest modules (caller) "+ "-- move things from actions.rest to actions if needed")+ (modules/usage-error config 'caller 'metabase.actions-rest.api))))))++(deftest ^:parallel usage-error-rest-module-exceptions-test+ (testing "REST modules, route aggregators, and core initializers may use REST modules"+ (let [config {:metabase/modules {'actions {:module-exports #{'actions.rest}+ :api :any}+ 'actions.rest {:ns-prefix "metabase.actions-rest"+ :api :any}+ 'questions.rest {:ns-prefix "metabase.questions-rest"+ :uses :any+ :api :any}+ 'api-routes {:uses :any+ :api :any}+ 'core {:uses :any+ :api :any}}}]+ (are [caller] (nil? (modules/usage-error config caller 'metabase.actions-rest.api))+ 'questions.rest+ 'api-routes+ 'core))))++;;;; -------------------------------------------------------------------------+;;;; Enterprise companion modules+;;;;+;;;; A declared `enterprise/X` sits under OSS `X` and is exported implicitly.+;;;; Without OSS `X`, it remains top-level.+;;;; -------------------------------------------------------------------------++(deftest ^:parallel usage-error-enterprise-shorthand-allows-cross-subtree-access-test+ (testing "enterprise/core can require enterprise/cache because cache exports it implicitly"+ (let [config {:metabase/modules {'core {:api :any}+ 'cache {:api :any}+ 'enterprise/core {:api :any+ :uses #{'enterprise/cache}}+ 'enterprise/cache {:api :any+ :uses #{}}}}]+ (is (nil? (modules/usage-error config 'enterprise/core 'metabase-enterprise.cache.core))))))++(deftest ^:parallel usage-error-enterprise-shorthand-same-subtree-access-test+ (testing "OSS X and enterprise/X share X's subtree"+ (let [config {:metabase/modules {'cache {:api :any+ :uses #{'enterprise/cache}}+ 'enterprise/cache {:api :any+ :uses #{'cache}}}}]+ (testing "OSS cache → enterprise/cache"+ (is (nil? (modules/usage-error config 'cache 'metabase-enterprise.cache.core))))+ (testing "enterprise/cache → OSS cache"+ (is (nil? (modules/usage-error config 'enterprise/cache 'metabase.cache.core)))))))++(deftest ^:parallel usage-error-enterprise-without-oss-counterpart-stays-top-level-test+ (testing "enterprise/X without a declared OSS X is top-level, so anyone may name it"+ (let [config {:metabase/modules {'enterprise/sandbox {:api :any+ :uses #{}}+ 'unrelated {:api :any+ :uses #{'enterprise/sandbox}}}}]+ (is (nil? (modules/usage-error config 'unrelated 'metabase-enterprise.sandbox.core))))))++(deftest ^:parallel usage-error-backwards-compat-flat-config-test+ (testing "a config without nested modules keeps flat-module behavior"+ (let [config {:metabase/modules {'lib {:api #{'metabase.lib.core+ 'metabase.lib.schema.foo}+ :uses #{}}+ 'query-processor {:api :any+ :uses #{'lib}}}}]+ (testing "flat :uses lib is sufficient for namespaces in lib's api"+ (is (nil? (modules/usage-error config 'query-processor 'metabase.lib.core)))+ (is (nil? (modules/usage-error config 'query-processor 'metabase.lib.schema.foo))))+ (testing "namespaces not in :api are denied"+ (is (some? (modules/usage-error config 'query-processor 'metabase.lib.internal)))))))diff --git a/test/metabase/premium_features/token_check_test.clj b/test/metabase/premium_features/token_check_test.cljindex a08aeb4e292..a5d4dba36ce 100644--- a/test/metabase/premium_features/token_check_test.clj+++ b/test/metabase/premium_features/token_check_test.clj@@ -23,12 +23,21 @@(use-fixtures :once (fixtures/initialize :db))+(defn- fresh-checker+ "A token checker with production TTLs and no circuit breaker.+ The global checker's breaker is shared across the JVM, and one tripped by an earlier test would fail every check here."+ []+ (binding [token-check/*customize-checker* true]+ (token-check/make-checker {:local-ttl (t/seconds 5)+ :soft-ttl (t/hours 12)+ :hard-ttl (t/hours 36)})))+(deftest log-tests(let [token (tu/random-token)print-token (apply str (concat (take 4 token) "..." (take-last 4 token)))](testing "Do not log the token (#18249)"(mt/with-log-messages-for-level [messages :info]- (token-check/check-token token)+ (token-check/check-token (fresh-checker) token)(let [logs (mapv :message (messages))](is (every? (complement #(re-find (re-pattern token) %)) logs))(is (= 1 (count (filter #(re-find (re-pattern print-token) %) logs)))))))))@@ -40,8 +49,8 @@(binding [http/request (fn [& _](swap! call-count inc){:status 400 :body "{\"valid\": false, \"status\": \"fake\"}"})]- (token-check/-clear-cache! token-check/token-checker)- (dotimes [_ 10] (token-check/check-token token))+ (let [checker (fresh-checker)]+ (dotimes [_ 10] (token-check/check-token checker token)))(is (= 1 @call-count))))))(deftest fetch-token-does-not-cache-exceptions@@ -71,16 +80,17 @@(deftest not-found-test(mt/with-log-level :fatal(is (=? {:valid false, :status "Token does not exist."}- (token-check/check-token (tu/random-token))))))+ (token-check/check-token (fresh-checker) (tu/random-token))))))(deftest fetch-token-does-not-call-db-when-cached(testing "No DB calls are made when checking token status if the status is in local cache"(let [token (tu/random-token)- _ (token-check/check-token token)+ checker (fresh-checker)+ _ (token-check/check-token checker token);; The local cache has a 5s TTL, so repeated checks within that window should not hit the DB.call-counts (repeatedly 3 (fn [](t2/with-call-count [call-count]- (token-check/check-token token)+ (token-check/check-token checker token)(call-count))))];; At least some of these should be zero (served from local in-memory cache)(is (some zero? call-counts)))))@@ -303,7 +313,7 @@(testing "If a `premium-embedding-token` has been set, the `token-status` setting should return the responsefrom the store.metabase.com endpoint for that token."(is (= {:valid false, :status "Token does not exist.", :canonical? true}- (token-check/check-token (tu/random-token)))))+ (token-check/check-token (fresh-checker) (tu/random-token)))))(testing "If premium-embedding-token is nil, the token-status setting should also be nil."(mt/with-temporary-setting-values [premium-embedding-token nil](is (nil? (premium-features/token-status))))))diff --git a/test/metabase/util/log_test.clj b/test/metabase/util/log_test.cljindex 5b872ccbd42..f6f86ac1001 100644--- a/test/metabase/util/log_test.clj+++ b/test/metabase/util/log_test.clj@@ -100,3 +100,15 @@(is (= [:ok nil](vec (for [_ (range 2)](log/throttle 60000 :ok))))))))++(deftest ^:parallel nested-module-team-attribution-test+ (testing "nested modules inherit their nearest configured team"+ (is (= "Querying Platform" (log/ns->team* 'metabase.lib.schema.expression)))+ (is (= "UX West" (log/ns->team* 'metabase-enterprise.analytics.stats))))+ (testing "nested modules can override their ancestor's team"+ (is (= "Metabot" (log/ns->team* 'metabase.agent-lib.core))))+ (testing "module-level test namespaces resolve like their source namespace"+ (let [prefix->module {"metabase.lib" 'lib+ "metabase.lib.schema" 'lib.schema}]+ (is (= 'lib.schema+ (#'log/module-for-ns prefix->module 'metabase.lib.schema-test))))))