The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Improve model encapsulation with serialization bypass (#71553)
Chris Truter3c968befffa3
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 2092d13ead0..f7229d463a0 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -53,7 +53,7 @@;; ## `model-exports`:;;;; Set of `:model/X` keywords defined by this module that it allows others to reference. By default all are private.-;; If you know what you're doing, we support an :all keyword, but let's prefer explicit sets for now.+;; If you know what you're doing, we support an :any keyword, but let's prefer explicit sets for now.;;;; enterprise/workspaces;; {...@@ -62,12 +62,25 @@;; ## `model-imports`:;;;; Set of `:model/X` keywords that this module is allowed to reference from other modules. Defaults to none.-;; If you know what you're doing, we support an :all keyword, but let's prefer explicit sets for now.+;; If you know what you're doing, we support an :any keyword, but let's prefer explicit sets for now.;;;; transforms-base;; {...;; :model-imports #{:model/Database :model/Field :model/Table :model/Transform}};;+;; ### `:bypass`+;;+;; Some modules (like `cmd` and `enterprise/serialization`) are inherently cross-cutting and import nearly+;; every model. For these, use `:model-imports :bypass` instead of an explicit set:+;;+;; cmd+;; {...+;; :model-imports :bypass}+;;+;; A bypassed module can reference any model — even ones that are not exported. Models that are *only*+;; referenced outside their home module by bypassed modules do not need to appear in `:model-exports`+;; (in fact, they must not — the staleness test will flag them).+;;;; ## Tips;;;; PRO TIP: Check out the [[dev.deps-graph]] namespace for helpful tools to see where a module is used externally, and@@ -97,7 +110,7 @@settingsutilwarehouses}- :model-exports #{:model/Action :model/HTTPAction :model/ImplicitAction :model/QueryAction}+ :model-exports #{:model/Action}:model-imports #{:model/Card:model/Collection:model/Dashboard@@ -133,7 +146,6 @@eventsmodelsutil}- :model-exports #{:model/RecentViews}:model-imports #{:model/Card:model/Collection:model/Dashboard@@ -290,7 +302,11 @@connection-pooltaskutil}- :model-imports #{:model/Database :model/Field :model/QueryCache :model/Secret :model/Setting}}+ :model-imports #{:model/Database+ :model/Field+ :model/QueryCache+ :model/Secret+ :model/Setting}}appearance{:team "UX West"@@ -372,12 +388,11 @@collectionsqueriesutil}- :model-exports #{:model/BookmarkOrdering- :model/CardBookmark- :model/CollectionBookmark- :model/DashboardBookmark- :model/DocumentBookmark}- :model-imports #{:model/Card :model/Collection :model/Dashboard :model/Document}}+ :model-exports #{:model/CardBookmark :model/CollectionBookmark :model/DashboardBookmark}+ :model-imports #{:model/Card+ :model/Collection+ :model/Dashboard+ :model/Document}}bug-reporting{:team "UX West"@@ -410,7 +425,10 @@settingsutil}:model-exports #{:model/CacheConfig :model/QueryCache}- :model-imports #{:model/Card :model/Dashboard :model/DashboardCard :model/Database}}+ :model-imports #{:model/Card+ :model/Dashboard+ :model/DashboardCard+ :model/Database}};; TODO -- way too many API namespaces.channel@@ -452,7 +470,10 @@typesutil}:model-exports #{:model/Channel :model/ChannelTemplate}- :model-imports #{:model/Dashboard :model/Database :model/PulseChannel :model/User}}+ :model-imports #{:model/Dashboard+ :model/Database+ :model/PulseChannel+ :model/User}}classloader{:team "DevEx"@@ -500,82 +521,7 @@searchsettingsutil}- :model-imports #{:model/Action- :model/ApiKey- :model/ApplicationPermissionsRevision- :model/AuditLog- :model/AuthIdentity- :model/BookmarkOrdering- :model/Card- :model/CardBookmark- :model/Channel- :model/ChannelTemplate- :model/Collection- :model/CollectionBookmark- :model/CollectionPermissionGraphRevision- :model/Comment- :model/CommentReaction- :model/ConnectionImpersonation- :model/Dashboard- :model/DashboardBookmark- :model/DashboardCard- :model/DashboardCardSeries- :model/DashboardTab- :model/DataPermissions- :model/Database- :model/Dimension- :model/Document- :model/DocumentBookmark- :model/Field- :model/FieldUserSettings- :model/FieldValues- :model/Glossary- :model/HTTPAction- :model/ImplicitAction- :model/LoginHistory- :model/Measure- :model/Metabot- :model/MetabotConversation- :model/MetabotMessage- :model/MetabotPrompt- :model/ModelIndex- :model/ModelIndexValue- :model/ModerationReview- :model/NativeQuerySnippet- :model/Notification- :model/NotificationCard- :model/NotificationHandler- :model/NotificationRecipient- :model/NotificationSubscription- :model/OAuthAccessToken- :model/OAuthAuthorizationCode- :model/OAuthClient- :model/OAuthRefreshToken- :model/ParameterCard- :model/Permissions- :model/PermissionsGroup- :model/PermissionsGroupMembership- :model/PermissionsRevision- :model/PersistedInfo- :model/Pulse- :model/PulseCard- :model/PulseChannel- :model/PulseChannelRecipient- :model/QueryAction- :model/RecentViews- :model/Revision- :model/Sandbox- :model/Secret- :model/Segment- :model/Session- :model/Setting- :model/Table- :model/Tenant- :model/Timeline- :model/TimelineEvent- :model/User- :model/UserParameterValue- :model/ViewLog}}+ :model-imports :bypass}collections{:team "UX West"@@ -656,7 +602,7 @@requestusersutil}- :model-exports #{:model/Comment :model/CommentReaction}+ :model-exports #{:model/Comment}:model-imports #{:model/Document :model/User}};; technically this 'uses' `enterprise/core` and `test` since it tries to load them to see if they exists so we know@@ -787,7 +733,11 @@syncutilwarehouse-schema}- :model-imports #{:model/Database :model/Dimension :model/Field :model/FieldValues :model/Table}}+ :model-imports #{:model/Database+ :model/Dimension+ :model/Field+ :model/FieldValues+ :model/Table}}database-routing{:team "UX West"@@ -817,7 +767,11 @@utilview-log}:model-exports #{:model/Document}- :model-imports #{:model/Card :model/Collection :model/Dashboard :model/Table :model/User}}+ :model-imports #{:model/Card+ :model/Collection+ :model/Dashboard+ :model/Table+ :model/User}}driver{:team "Querying Platform"@@ -1049,7 +1003,11 @@metabase.internal-stats.metabot}:uses #{app-dbmodels}- :model-imports #{:model/Card :model/Dashboard :model/MetabotMessage :model/QueryExecution :model/User}}+ :model-imports #{:model/Card+ :model/Dashboard+ :model/MetabotMessage+ :model/QueryExecution+ :model/User}}legacy-mbql{:team "Querying Platform"@@ -1253,10 +1211,7 @@transforms-baseutilwarehouses}- :model-exports #{:model/Metabot- :model/MetabotConversation- :model/MetabotMessage- :model/MetabotPrompt}+ :model-exports #{:model/MetabotMessage}:model-imports #{:model/Card:model/Collection:model/Dashboard@@ -1404,11 +1359,7 @@tracingusersutil}- :model-exports #{:model/Notification- :model/NotificationCard- :model/NotificationHandler- :model/NotificationRecipient- :model/NotificationSubscription}+ :model-exports #{:model/Notification :model/NotificationHandler :model/NotificationRecipient}:model-imports #{:model/Card:model/Channel:model/ChannelTemplate@@ -1427,11 +1378,7 @@modelssettingssystem- util}- :model-exports #{:model/OAuthAccessToken- :model/OAuthAuthorizationCode- :model/OAuthClient- :model/OAuthRefreshToken}}+ util}};; TODO -- too many API namespacesparameters@@ -1456,7 +1403,11 @@utilwarehouse-schemawarehouses}- :model-imports #{:model/Card :model/Dashboard :model/Field :model/FieldValues :model/Table}}+ :model-imports #{:model/Card+ :model/Dashboard+ :model/Field+ :model/FieldValues+ :model/Table}}permissions{:team "UX West"@@ -1484,7 +1435,10 @@:model/PermissionsGroup:model/PermissionsGroupMembership:model/PermissionsRevision}- :model-imports #{:model/Collection :model/Database :model/Table :model/User}}+ :model-imports #{:model/Collection+ :model/Database+ :model/Table+ :model/User}}permissions-rest{:team "UX West"@@ -1552,7 +1506,10 @@settingstaskutil}- :model-imports #{:model/Card :model/Dashboard :model/Database :model/User}}+ :model-imports #{:model/Card+ :model/Dashboard+ :model/Database+ :model/User}}public-sharing{:team "UX West"@@ -1610,7 +1567,10 @@utilenterprise/advanced-permissionsenterprise/sandbox}- :model-exports #{:model/Pulse :model/PulseCard :model/PulseChannel :model/PulseChannelRecipient}+ :model-exports #{:model/Pulse+ :model/PulseCard+ :model/PulseChannel+ :model/PulseChannelRecipient}:model-imports #{:model/Card:model/Channel:model/Collection@@ -1655,7 +1615,6 @@view-logwarehouse-schema}:model-exports #{:model/Card- :model/ParameterCard:model/Query:model/QueryExecution:model/QueryTable}@@ -1882,7 +1841,11 @@tracingtransformsutil}- :model-imports #{:model/Card :model/Collection :model/Dashboard :model/Table :model/User}}+ :model-imports #{:model/Card+ :model/Collection+ :model/Dashboard+ :model/Table+ :model/User}}secrets{:team "Graphy"@@ -2120,7 +2083,10 @@utilwarehouse-schemawarehouses}- :model-imports #{:model/Database :model/Field :model/FieldValues :model/Table}}+ :model-imports #{:model/Database+ :model/Field+ :model/FieldValues+ :model/Table}}system{:team "UX West"@@ -2200,7 +2166,7 @@eventsmodelsutil}- :model-exports #{:model/Timeline :model/TimelineEvent}+ :model-exports #{:model/Timeline}:model-imports #{:model/Collection :model/User}}tracing@@ -2323,7 +2289,10 @@syncutilwarehouse-schema}- :model-imports #{:model/Card :model/Database :model/Field :model/Table}}+ :model-imports #{:model/Card+ :model/Database+ :model/Field+ :model/Table}}user-key-value{:team "UX West"@@ -2455,7 +2424,6 @@warehouses}:model-exports #{:model/Dimension:model/Field- :model/FieldUserSettings:model/FieldValues:model/Table}:model-imports #{:model/Card@@ -2488,7 +2456,11 @@utilwarehouse-schemaxrays}- :model-imports #{:model/Database :model/Dimension :model/Field :model/FieldValues :model/Table}}+ :model-imports #{:model/Database+ :model/Dimension+ :model/Field+ :model/FieldValues+ :model/Table}}warehouses{:team "UX West"@@ -2510,7 +2482,10 @@syncutil}:model-exports #{:model/Database}- :model-imports #{:model/Card :model/Field :model/Table :model/User}}+ :model-imports #{:model/Card+ :model/Field+ :model/Table+ :model/User}}warehouses-rest{:team "UX West"@@ -2540,7 +2515,11 @@warehouse-schemawarehousesenterprise/advanced-permissions}- :model-imports #{:model/Card :model/Collection :model/Database :model/Field :model/Table}}+ :model-imports #{:model/Card+ :model/Collection+ :model/Database+ :model/Field+ :model/Table}}workspaces{:team "Metabot"@@ -2594,7 +2573,10 @@query-processorutilwarehouse-schema}- :model-imports #{:model/Database :model/Field :model/FieldValues :model/Table}}+ :model-imports #{:model/Database+ :model/Field+ :model/FieldValues+ :model/Table}}enterprise/advanced-config{:team "UX West"@@ -2801,7 +2783,10 @@permissionspremium-featuresutil}- :model-imports #{:model/Collection :model/Dimension :model/Field :model/Table}}+ :model-imports #{:model/Collection+ :model/Dimension+ :model/Field+ :model/Table}}enterprise/database-replication{:team "Cloud"@@ -3156,7 +3141,7 @@serversetuputil}- :model-imports #{:model/Card :model/Collection :model/Dashboard :model/User}}+ :model-imports :bypass}enterprise/snippet-collections{:team "Graphy"@@ -3254,7 +3239,10 @@transformstransforms-baseutil}- :model-imports #{:model/Database :model/Field :model/Table :model/Transform}}+ :model-imports #{:model/Database+ :model/Field+ :model/Table+ :model/Transform}}enterprise/transforms-python{:team "Graphy"@@ -3278,7 +3266,10 @@transforms-baseutil}:model-exports #{:model/PythonLibrary}- :model-imports #{:model/Database :model/Field :model/Table :model/TransformRun}}+ :model-imports #{:model/Database+ :model/Field+ :model/Table+ :model/TransformRun}}enterprise/upload-management{:team "UX West"diff --git a/dev/notes/model-boundary-enforcement.md b/dev/notes/model-boundary-enforcement.mdindex fd8008affb9..a5263be519b 100644--- a/dev/notes/model-boundary-enforcement.md+++ b/dev/notes/model-boundary-enforcement.md@@ -15,9 +15,13 @@ Two optional config keys per module in| `:model-imports` | `#{}` | Set of `:model/X` keywords **this module** may use from other modules. |When omitted, no models are exported/imported (closed by default). `:any`-opens all restrictions. Every module has explicit `:model-exports` and-`:model-imports` in `config.edn` — the defaults apply only to new modules-that haven't been configured yet.+opens all restrictions.++`:model-imports` also supports `:bypass` for cross-cutting modules that+reference nearly every model (e.g. `cmd`, `enterprise/serialization`). A+bypassed module can reference any model — even unexported ones. Models that+are *only* referenced by bypass modules do not need `:model-exports` entries+(the staleness test will flag them if added).## Why it exists@@ -120,6 +124,24 @@ The module can only reference these models from other modules. Any other`:model/X` keyword in the module's source files will cause a test failure.Unknown models (not defined anywhere) are also violations.+### Bypassing import restrictions++Some modules are inherently cross-cutting and import nearly every model. Use+`:bypass` instead of an explicit set:++```clojure+cmd+{:team "UX West"+ :model-imports :bypass+ ...}+```++A bypassed module can reference any model, even unexported ones. This means+bypass modules don't drive `:model-exports` — if a model is *only* used+outside its home module by bypass modules, it doesn't need to be exported.++Use sparingly. Prefer explicit sets where feasible.+### Verifying changes```bash@@ -143,8 +165,8 @@ externally, imports not referenced in the module's source files).Use the dev namespace to see what the config should contain:```clojure-(dev.model-boundary-config/compute-and-print!) ;; print all exports/imports(dev.model-boundary-config/compute-model-boundaries) ;; returns data+(dev.model-boundary-config/update-config!) ;; rewrite config.edn```## Limitations and future directionsdiff --git a/dev/src/dev/deps_graph.clj b/dev/src/dev/deps_graph.cljindex a97223e4168..90269c40d61 100644--- a/dev/src/dev/deps_graph.clj+++ b/dev/src/dev/deps_graph.clj@@ -826,25 +826,31 @@#{'metabase.models.resolution})(defn- model-reference-violations- "Return violation types for a single model reference. Pure function — no IO."+ "Return violation types for a single model reference. Pure function — no IO.++ When `model-imports` is `:bypass`, the using module is exempt from all model boundary checks —+ it may reference any model regardless of whether it is exported."[model defining-mod model-exports model-imports]- (cond-> []- (nil? defining-mod)- (conj :unknown-model)+ (if (= model-imports :bypass)+ []+ (cond-> []+ (nil? defining-mod)+ (conj :unknown-model)- (and defining-mod- (not= model-exports :any)- (not (contains? model-exports model)))- (conj :not-exported)+ (and defining-mod+ (not= model-exports :any)+ (not (contains? model-exports model)))+ (conj :not-exported)- (and defining-mod- (not= model-imports :any)- (not (contains? model-imports model)))- (conj :not-imported)))+ (and defining-mod+ (not= model-imports :any)+ (not (contains? model-imports model)))+ (conj :not-imported))))(defn model-references-by-module"Scan all source files and build a map of `{module => #{:model/X ...}}` — the set of model keywords- referenced in each module's source files. Exempt namespaces (e.g. `metabase.models.resolution`) are excluded."+ referenced in each module's source files. Exempt namespaces (e.g. `metabase.models.resolution`) are excluded.+ Includes all modules (including bypass modules) — callers filter as needed."[](reduce(fn [acc file]@@ -872,6 +878,9 @@2. The using module's `:model-imports` allows the model (`:any` or set containing it)3. The model's definition exists somewhere (`:unknown-model` is always a violation)+ Modules with `:model-imports :bypass` are exempt from all checks — they may reference any model,+ even unexported ones.+Returns a sequence of violation maps with `:file`, `:module`, `:model`, `:defining-module`, `:violation-type`."([](model-boundary-violations (kondo-config)))diff --git a/dev/src/dev/model_boundary_config.clj b/dev/src/dev/model_boundary_config.cljindex ed1394e0a45..b33ea1c277b 100644--- a/dev/src/dev/model_boundary_config.clj+++ b/dev/src/dev/model_boundary_config.clj@@ -5,9 +5,16 @@(dev.model-boundary-config/compute-model-boundaries);; => {:model-exports {module => #{:model/X ...}}- ;; :model-imports {module => #{:model/X ...}}}"+ ;; :model-imports {module => #{:model/X ...}}}++ To update config.edn with computed model boundaries:++ (dev.model-boundary-config/update-config!)"(:require- [dev.deps-graph :as deps-graph]))+ [clojure.string :as str]+ [dev.deps-graph :as deps-graph]+ [rewrite-clj.parser :as r.parser]+ [rewrite-clj.zip :as z]))(set! *warn-on-reflection* true)@@ -17,19 +24,31 @@Returns `{:model-exports {module => sorted-set-of-models}:model-imports {module => sorted-set-of-models}}`- `:model-exports` for module M = models owned by M that are referenced by at least one other module.- `:model-imports` for module M = models owned by other modules that M references."+ `:model-exports` for module M = models owned by M that are referenced by at least one non-bypass module.+ `:model-imports` for module M = models owned by other modules that M references.++ Modules with `:model-imports :bypass` are excluded: they don't need computed imports, and their+ references don't drive exports (models used only by bypass modules need not be exported)."[]- (let [ownership (deps-graph/model-ownership)+ (let [config (deps-graph/kondo-config)+ ownership (deps-graph/model-ownership)module-refs (deps-graph/model-references-by-module)- all-modules (set (keys (deps-graph/kondo-config)))+ all-modules (set (keys config))+ bypass-modules (into #{}+ (keep (fn [[mod mod-config]]+ (when (= (:model-imports mod-config) :bypass)+ mod)))+ config);; {:model/X => #{modules that reference it}} — inverted index for fast export lookups+ ;; Only non-bypass modules drive exports.model->referencing-modules(reduce-kv (fn [acc mod models]- (reduce (fn [acc model]- (update acc model (fnil conj #{}) mod))- acc- models))+ (if (contains? bypass-modules mod)+ acc+ (reduce (fn [acc model]+ (update acc model (fnil conj #{}) mod))+ acc+ models))){}module-refs)]{:model-exports@@ -45,6 +64,7 @@:model-imports(into (sorted-map)(for [mod all-modules+ :when (not (contains? bypass-modules mod)):let [imported (into (sorted-set)(for [model (get module-refs mod):let [defining-mod (get ownership model)]@@ -52,3 +72,70 @@model))]:when (seq imported)][mod imported]))}))++(def ^:private config-path ".clj-kondo/config/modules/config.edn")++(def ^:private set-indent+ "Indentation for multi-line model sets — aligns with existing config.edn style."+ (apply str (repeat 20 \space)))++(defn- model-set-str+ "Build a string representation of a sorted set of model keywords.+ Short sets (<=3 items) stay on one line; longer sets get one item per line."+ [models]+ (let [sorted (sort models)]+ (if (<= (count sorted) 3)+ (str "#{" (str/join " " sorted) "}")+ (str "#{" (str/join (str "\n" set-indent) sorted) "}"))))++(defn- find-module-config+ "Navigate from `root` to the config map for `module-sym`. Returns a zipper or nil."+ [root module-sym]+ (let [mods-zloc (-> (z/of-node root)+ (z/find-value z/next :metabase/modules)+ z/right)]+ (z/find-value (z/down mods-zloc) z/right module-sym)))++(defn- find-key-value+ "Find the value zipper for keyword `k` inside a map zipper, or nil."+ [map-zloc k]+ (some-> (z/find-value (z/down map-zloc) z/right k)+ z/right))++(defn update-config!+ "Compute model boundaries and update config.edn with `:model-exports` and `:model-imports` for all modules.++ Only updates keys that already exist in the config — does not add new keys.+ Uses rewrite-clj to manipulate the AST directly, preserving formatting."+ []+ (let [boundaries (compute-model-boundaries)+ root-zloc (z/of-node (r.parser/parse-file-all config-path))+ modules-zloc (-> root-zloc+ (z/find-value z/next :metabase/modules)+ z/right)+ module-syms (loop [zloc (z/down modules-zloc), acc []]+ (if-not zloc+ acc+ (recur (some-> zloc z/right z/right)+ (conj acc (z/sexpr zloc)))))+ updated-root+ (reduce+ (fn [root module-sym]+ (reduce+ (fn [root config-key]+ (let [mod-cfg (some-> (find-module-config root module-sym) z/right)+ val-zloc (when mod-cfg (find-key-value mod-cfg config-key))]+ ;; Only update keys that already exist and are sets (skip :bypass, :any, etc.)+ (if (and val-zloc (set? (z/sexpr val-zloc)))+ (let [computed (get-in boundaries [config-key module-sym] #{})]+ (if (seq computed)+ (z/root (z/replace val-zloc (r.parser/parse-string (model-set-str computed))))+ ;; Empty — remove the key-value pair+ (-> val-zloc z/remove z/remove z/root)))+ root)))+ root+ [:model-exports :model-imports]))+ (z/root root-zloc)+ module-syms)]+ (spit config-path (z/root-string (z/of-node updated-root)))+ (println "Updated" config-path "with :model-exports and :model-imports.")))diff --git a/test/metabase/core/modules_test.clj b/test/metabase/core/modules_test.cljindex 8b4461a64f1..d4d14747875 100644--- a/test/metabase/core/modules_test.clj+++ b/test/metabase/core/modules_test.clj@@ -199,15 +199,12 @@file module model (or defining-module "unknown") (name violation-type))(is (nil? violation-type)))))(testing ":model-exports and :model-imports reference valid models"- (doseq [[module module-config] config]- (when (set? (:model-exports module-config))- (doseq [model (:model-exports module-config)]- (testing (format "\n'%s' :model-exports %s should be a known model" module model)- (is (contains? known-models model)))))- (when (set? (:model-imports module-config))- (doseq [model (:model-imports module-config)]- (testing (format "\n'%s' :model-imports %s should be a known model" module model)- (is (contains? known-models model)))))))+ (doseq [[module module-config] config+ config-key [:model-exports :model-imports]+ :when (set? (get module-config config-key))+ model (get module-config config-key)]+ (testing (format "\n'%s' %s %s should be a known model" module config-key model)+ (is (contains? known-models model)))))(testing ":model-exports only lists models owned by the module"(doseq [[module module-config] config:when (set? (:model-exports module-config))