The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix N+1 query patterns with permissions (#78919)
Alexander Polyankin70c3f15684d4
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex cdbf541fd4b..d5f5782f73b 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -146,6 +146,7 @@configeventsmodels+ permissionsutil}:model-imports #{:model/Card:model/Collection@@ -2519,6 +2520,7 @@metabotmetricsmodels+ permissionssystemtypesutil}diff --git a/enterprise/backend/src/metabase_enterprise/advanced_permissions/common.clj b/enterprise/backend/src/metabase_enterprise/advanced_permissions/common.cljindex a90b5e78155..3bfab6bec2a 100644--- a/enterprise/backend/src/metabase_enterprise/advanced_permissions/common.clj+++ b/enterprise/backend/src/metabase_enterprise/advanced_permissions/common.clj@@ -106,15 +106,17 @@(empty tables):else- (filter- (fn [{table-id :id db-id :db_id}]- (perms/user-has-permission-for-table?- api/*current-user-id*- :perms/manage-table-metadata- :yes- db-id- table-id))- tables)))+ (do+ (perms/prime-db-perms-cache (into #{} (map :db_id) tables))+ (filter+ (fn [{table-id :id db-id :db_id}]+ (perms/user-has-permission-for-table?+ api/*current-user-id*+ :perms/manage-table-metadata+ :yes+ db-id+ table-id))+ tables))))(defn filter-schema-by-data-model-perms"Given a list of schema, remove the ones for which `*current-user*` does not have data model editing permissions."diff --git a/enterprise/backend/src/metabase_enterprise/data_studio/api/table.clj b/enterprise/backend/src/metabase_enterprise/data_studio/api/table.cljindex 5ee7159af4f..654139440d1 100644--- a/enterprise/backend/src/metabase_enterprise/data_studio/api/table.clj+++ b/enterprise/backend/src/metabase_enterprise/data_studio/api/table.clj@@ -52,7 +52,13 @@(if (empty? tables)#{}(let [input-table-id (keyword (name input-field) "table_id")- output-table-id (keyword (name output-field) "table_id")]+ output-table-id (keyword (name output-field) "table_id")+ not-in-tables (if (map? tables)+ [:not [:exists (-> tables+ (assoc :select [1])+ (update :where (fn [where]+ [:and where [:= :id output-table-id]])))]]+ [:not [:in output-table-id tables]])](into #{} (map :table_id)(t2/reducible-query {:select [[output-table-id :table_id]]:from [[(t2/table-name :model/Dimension) :dim]]@@ -63,7 +69,7 @@:where [:and[:= :dim.type "external"][:in input-table-id tables]- [:not [:in output-table-id tables]]]})))))+ not-in-tables]})))))(defn- upstream-table-ids"Given a table selector (set of IDs or subquery), find all tables that these tables depend ondiff --git a/enterprise/backend/src/metabase_enterprise/embedding_hub/api.clj b/enterprise/backend/src/metabase_enterprise/embedding_hub/api.cljindex 247ee0f9990..4148799e308 100644--- a/enterprise/backend/src/metabase_enterprise/embedding_hub/api.clj+++ b/enterprise/backend/src/metabase_enterprise/embedding_hub/api.clj@@ -48,9 +48,11 @@[:or[:and[:!= :collection_id (:id (audit/default-audit-collection))]- [:not-in :collection_id {:select :id- :from [(t2/table-name :model/Collection)]- :where [:= :is_sample true]}]]+ [:not [:exists {:select [1]+ :from [[(t2/table-name :model/Collection) :sample_coll]]+ :where [:and+ [:= :sample_coll.is_sample true]+ [:= :sample_coll.id :report_card.collection_id]]}]]][:is :collection_id nil]]]}))(defn- has-user-created-tenants? []diff --git a/src/metabase/activity_feed/models/recent_views.clj b/src/metabase/activity_feed/models/recent_views.cljindex 998ff87e7c3..a5d95d2fb78 100644--- a/src/metabase/activity_feed/models/recent_views.clj+++ b/src/metabase/activity_feed/models/recent_views.clj@@ -38,6 +38,7 @@[metabase.collections.models.collection.root :as root][metabase.config.core :as config][metabase.models.interface :as mi]+ [metabase.permissions.core :as perms][metabase.util :as u][metabase.util.honey-sql-2 :as h2x][metabase.util.log :as log]@@ -438,28 +439,29 @@(defn- table-recents"Query to select recent table data"[table-ids]- (t2/select :model/Table- {:select [:t.id :t.name :t.description- :t.display_name :t.active :t.visibility_type :t.schema- [:db.name :database-name]- [:db.id :db_id]- [:db.initial_sync_status :initial-sync-status]]- :from [[:metabase_table :t]]- :where (let [base-condition [:or- [:= :visibility_type nil]- [:!= :visibility_type "hidden"]]]- (if (seq table-ids)- [:and base-condition [:in :t.id table-ids]]- base-condition))- :left-join [[:metabase_database :db]- [:= :db.id :t.db_id]]}))+ (if-not (seq table-ids)+ []+ (t2/select :model/Table+ {:select [:t.id :t.name :t.description+ :t.display_name :t.active :t.visibility_type :t.schema+ [:db.name :database-name]+ [:db.id :db_id]+ [:db.initial_sync_status :initial-sync-status]]+ :from [[:metabase_table :t]]+ :where [:and+ [:or+ [:= :visibility_type nil]+ [:!= :visibility_type "hidden"]]+ [:in :t.id table-ids]]+ :left-join [[:metabase_database :db]+ [:= :db.id :t.db_id]]})))(defmethod fill-recent-view-info :table [{:keys [_model model_id timestamp model_object]}](let [table model_object](when (and (not= "hidden" (:visibility_type table))(:database-name table)(:active table)- (mi/can-read? :model/Table model_id))+ (mi/can-read? table)){:id model_id:name (:name table):description (:description table)@@ -576,11 +578,13 @@table-ids :tabledocument-ids :document} (as-> views views(group-by (comp keyword :model) views)- (update-vals views #(mapv :model_id %)))]+ (update-vals views #(mapv :model_id %)))+ tables (table-recents table-ids)]+ (perms/prime-db-perms-cache (into #{} (keep :db_id) tables)){:card (m/index-by :id (card-recents card-ids)):dashboard (m/index-by :id (dashboard-recents dashboard-ids)):collection (m/index-by :id (collection-recents collection-ids))- :table (m/index-by :id (table-recents table-ids))+ :table (m/index-by :id tables):document (m/index-by :id (document-recents document-ids))}))(def ^:private ItemValidator (mr/validator Item))diff --git a/src/metabase/data_studio/api/table.clj b/src/metabase/data_studio/api/table.cljindex dd9dd251b47..8f6269a233a 100644--- a/src/metabase/data_studio/api/table.clj+++ b/src/metabase/data_studio/api/table.clj@@ -55,7 +55,13 @@(if (empty? tables)#{}(let [input-table-id (keyword (name input-field) "table_id")- output-table-id (keyword (name output-field) "table_id")]+ output-table-id (keyword (name output-field) "table_id")+ not-in-tables (if (map? tables)+ [:not [:exists (-> tables+ (assoc :select [1])+ (update :where (fn [where]+ [:and where [:= :id output-table-id]])))]]+ [:not [:in output-table-id tables]])](into #{} (map :table_id)(t2/reducible-query {:select [[output-table-id :table_id]]:from [[(t2/table-name :model/Dimension) :dim]]@@ -66,7 +72,7 @@:where [:and[:= :dim.type "external"][:in input-table-id tables]- [:not [:in output-table-id tables]]]})))))+ not-in-tables]})))))(defn- upstream-table-ids"Given a table selector (set of IDs or subquery), find all tables that these tables depend ondiff --git a/src/metabase/measures/api.clj b/src/metabase/measures/api.cljindex ab77edea3f8..6ebae0ddb43 100644--- a/src/metabase/measures/api.clj+++ b/src/metabase/measures/api.clj@@ -8,6 +8,7 @@[metabase.lib.core :as lib][metabase.metrics.core :as metrics][metabase.models.interface :as mi]+ [metabase.permissions.core :as perms][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.malli :as mu]@@ -103,9 +104,12 @@(api.macros/defendpoint :get "/" :- [:sequential ::measure]"Fetch *all* `Measures`."[]- (as-> (t2/select :model/Measure, :archived false, {:order-by [[:%lower.name :asc]]}) measures- (filter mi/can-read? measures)- (t2/hydrate measures :creator :definition_description)))+ (let [measures (t2/select :model/Measure, :archived false, {:order-by [[:%lower.name :asc]]})+ table-ids (into #{} (keep :table_id) measures)]+ (when (seq table-ids)+ (perms/prime-db-perms-cache (t2/select-fn-set :db_id :model/Table :id [:in table-ids])))+ (-> (filterv mi/can-read? measures)+ (t2/hydrate :creator :definition_description))))(defn- write-check-and-update-measure!"Check whether current user has write permissions, then update Measure with values in `body`. Publishes appropriatediff --git a/src/metabase/permissions/core.clj b/src/metabase/permissions/core.cljindex 32b47d03fbb..a2f3f1bf784 100644--- a/src/metabase/permissions/core.clj+++ b/src/metabase/permissions/core.clj@@ -51,7 +51,7 @@most-permissive-database-permission-for-usernative-download-permission-for-userpermissions-for-user- prime-db-cache+ prime-db-perms-cachesandboxes-for-userschema-permission-for-userset-database-permission!diff --git a/src/metabase/permissions/models/data_permissions.clj b/src/metabase/permissions/models/data_permissions.cljindex a3cd3d89f5d..69f0b08184b 100644--- a/src/metabase/permissions/models/data_permissions.clj+++ b/src/metabase/permissions/models/data_permissions.clj@@ -16,10 +16,8 @@[metabase.util.malli :as mu][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]- [metabase.util.memoize :as u.memo][methodical.core :as methodical]- [toucan2.core :as t2]- [toucan2.realize :as t2.realize])+ [toucan2.core :as t2])(:import(clojure.lang PersistentVector)))@@ -144,210 +142,6 @@[perm-type perm-value](validate-perm-value! perm-type perm-value))-;;; ---------------------------------------- Caching --------------------------------------------------------------------(defn- relevant-permissions-for-user-and-dbs- "Returns a reducible of all relevant permission rows for the user, excluding permissions for deactivated tables, for- the given sequence of database ids. Reducing it folds rows one at a time instead of realizing the full result set- (see [[prime-db-cache]]). Rows are raw realized maps, not model instances, so `:perm_type` and `:perm_value` are- strings rather than keywords. The per-row [[t2.realize/realize]] is a performance requirement, not a convenience:- key access on unrealized result-set rows goes through toucan2's deferred-row machinery on every lookup, which- benchmarked ~15x slower than realizing each row once and reading plain map keys."- [user-id db-ids]- (eduction- (map t2.realize/realize)- (t2/reducible-query- {:select [:p.group_id :p.perm_type :p.db_id :p.table_id :p.schema_name :p.perm_value]- :from [[(t2/table-name :model/PermissionsGroupMembership) :pgm]]- :join [[(t2/table-name :model/PermissionsGroup) :pg] [:= :pg.id :pgm.group_id]- [(t2/table-name :model/DataPermissions) :p] [:= :p.group_id :pg.id]]- :left-join [[(t2/table-name :model/Table) :mt] [:= :mt.id :p.table_id]]- :where [:and- [:= :pgm.user_id user-id]- [:in :p.db_id db-ids]- [:or- [:= :p.table_id nil]- [:= :mt.active true]]]})))--(defn- relevant-permissions-for-user-perm-and-db- "Returns all relevant rows for a given user, permission type, and db_id, excluding permissions for deactivated- tables. Rows are raw realized maps rather than model instances -- much cheaper per row, see- [[relevant-permissions-for-user-and-dbs]] -- so `:perm_value` is a string; [[rows->cache-entry]] normalizes it."- [user-id perm-type db-id]- (t2/query {:select [:p.group_id :p.perm_type :p.db_id :p.table_id :p.schema_name :p.perm_value]- :from [[(t2/table-name :model/PermissionsGroupMembership) :pgm]]- :join [[(t2/table-name :model/PermissionsGroup) :pg] [:= :pg.id :pgm.group_id]- [(t2/table-name :model/DataPermissions) :p] [:= :p.group_id :pg.id]]- :left-join [[(t2/table-name :model/Table) :mt] [:= :mt.id :p.table_id]]- :where [:and- [:= :pgm.user_id user-id]- [:= :p.perm_type (u/qualified-name perm-type)]- [:= :p.db_id db-id]- [:or- [:= :p.table_id nil]- [:= :mt.active true]]]}))--(def ^:dynamic *permissions-for-user*- "A dynamically-bound atom containing a cache of data permissions that have been fetched so far for the current user.- Keys are:- - :db-ids -> A set of the IDs of databases which have already been fetched.- - :intern -> The interner used to dedupe entry values; created on first prime so sharing spans every prime- in the request.- - :perms -> A map of permissions with the structure `{user-id {perm-type {db-id entry}}}` so that we NEVER- accidentally use the cache of the wrong user. Each `entry` is a compact map-- {:groups {group-id #{perm-value}} ; db-level rows- :tables {table-id {group-id #{{:schema schema-name ; table-level rows- :value perm-value}}}}}-- The sets almost always hold exactly one value; see [[rows->cache-entry]] for why every distinct- value of duplicate rows is preserved rather than one row winning.-- On instances with many databases these maps are typically identical across most of them, so- they are interned at load time: equal values share one instance. This keeps the cache size- proportional to the number of *distinct* permission profiles rather than `groups * databases`- (see #76077 for how large the raw row count can get). Concretely, on an instance where the- user's groups have the same db-level grants almost everywhere plus one database with- table-level rows:-- {1- {:perms/view-data- {10 {:groups {1 #{:unrestricted}, 2 #{:blocked}} :tables {}} ; ─┐ interning makes- 11 {:groups {1 #{:unrestricted}, 2 #{:blocked}} :tables {}} ; ├ these :groups maps- 12 {:groups {1 #{:unrestricted}, 2 #{:blocked}} :tables {}} ; ─┘ identical- 13 {:groups {1 #{:unrestricted}}- :tables- {100 {2 #{{:schema \"public\" :value :blocked}}} ; ─┬ identical, and their- 101 {2 #{{:schema \"public\" :value :blocked}}} ; ─┘ inner :schema/:value- 102 {2 #{{:schema \"reporting\" :value :blocked}}}}}}}} ; maps also identical-- Each annotated group prints as N equal maps but is held as ONE shared instance plus N- pointers — 1,000 databases with the same profile cost one :groups map, not 1,000. The sharing- is established at construction by [[rows->cache-entry]] and is invisible to readers.-- When checking permissions, if a DB has not been fetched, it will be added to the cache before the check returns."- (atom {:db-ids #{} :perms {}}))--(defn- cache-interner- "A nil-tolerant [[u.memo/fast-interner]]."- []- (let [intern! (u.memo/fast-interner)]- (fn [v]- (when (some? v)- (intern! v)))))--(defn- add-row-to-cache-entry- "Fold a single data_permissions row into an in-progress cache entry (see [[rows->cache-entry]] for the semantics).- `perm_value` may be a raw result-set string or an already-transformed keyword depending on the fetch path;- `keyword` normalizes both."- [interner entry {:keys [table_id schema_name group_id perm_value]}]- (let [perm-value (keyword perm_value)]- (if table_id- (update-in entry [:tables table_id group_id] (fnil conj #{})- (interner {:schema schema_name :value perm-value}))- (update-in entry [:groups group_id] (fnil conj #{}) perm-value))))--(defn- finalize-cache-entry- "Intern the shareable pieces of a folded cache entry: the whole :groups map, and each table's group-id->values map."- [interner {:keys [groups tables] :or {groups {} tables {}}}]- {:groups (interner groups)- :tables (update-vals tables interner)})--(defn- rows->cache-entry- "Build the compact cache entry (see [[*permissions-for-user*]]) for one (perm-type, db-id) bucket of- data_permissions rows. Returns nil when there are no rows.-- The `interner` calls are a semantic no-op: they never change what any lookup returns, only object identity,- collapsing =-equal values to one shared instance so that the same profile repeated across many databases (or the- same :schema/:value map across many tables) is stored once — see the example on [[*permissions-for-user*]].-- Values accumulate into sets rather than a single value per key: data_permissions has no unique constraint on- (group_id, perm_type, db_id, table_id), and its delete-then-insert write paths take different cluster locks, so- racing writers can — and in production data do — leave multiple rows for the same logical key. The raw rows fed- every duplicate's value into the consumers' coalescing logic, which resolved conflicts deterministically; a- single-value map would instead keep whichever row the unordered SELECT returned last, making permission results- flip with JDBC row order (e.g. duplicate download-results rows {:no, :one-million-rows} granting downloads that- coalescing would deny)."- [interner rows]- (when (seq rows)- (finalize-cache-entry interner (reduce (partial add-row-to-cache-entry interner) {} rows))))--(defn prime-db-cache- "Prime the permissions cache for a given user and database IDs.- This can be called directly prior to checking the permissions for a large number of databases to improve performance"- [db-ids]- (let [{cached-db-ids :db-ids interner :intern perms :perms} @*permissions-for-user*- filtered-ids (remove cached-db-ids db-ids)]- (when (seq filtered-ids)- (let [user-id api/*current-user-id*- interner (or interner (cache-interner))- folded (reduce (fn [m {:keys [perm_type db_id] :as row}]- (update-in m [(keyword perm_type) db_id]- (fnil #(add-row-to-cache-entry interner % row) {})))- {}- (relevant-permissions-for-user-and-dbs user-id filtered-ids))- new-by-type (update-vals folded- (fn [db-id->entry]- (update-vals db-id->entry #(finalize-cache-entry interner %))))]- (reset! *permissions-for-user*- {:db-ids (into cached-db-ids db-ids)- :intern interner- :perms (update perms user-id #(merge-with merge % new-by-type))})))))--(defenterprise enforced-sandboxes-for-user- "Given a user-id, returns the set of sandboxes that should be enforced for the provided user ID. This result is- cached for the duration of a request. Empty on OSS instances."- metabase-enterprise.sandbox.api.util- [_user-id]- #{})--(def ^:dynamic *sandboxes-for-user*- "Filled by `enforced-sandboxes-for-user`. Empty on OSS instances, or EE instances without the `sandboxes` feature."- (delay nil))--(defn sandboxes-for-user- "Derefs the *sandboxes-for-user* dynamic variable so it can be contained to this namespace"- []- @*sandboxes-for-user*)--(defmacro with-relevant-permissions-for-user- "Populates the `*permissions-for-user*` and `*sandboxes-for-user*` dynamic vars for use by the cache-aware functions- in this namespace."- [user-id & body]- `(binding [*permissions-for-user* (atom {:db-ids #{} :perms {}})- *sandboxes-for-user* (delay (enforced-sandboxes-for-user ~user-id))]- ~@body))--(def ^:dynamic *use-perms-cache?*- "Bind to `false` to intentionally bypass the permissions cache and fetch data straight from the DB."- true)--(defmacro disable-perms-cache- "Intentionally bypass the permissions cache and fetch data straight from the DB."- {:style/indent 0}- [& body]- `(binding [*use-perms-cache?* false]- ~@body))--(defn- use-cache?- [user-id]- (and *use-perms-cache?*- (= user-id api/*current-user-id*)))--(defn- get-permissions- "Returns the compact cache entry (see [[*permissions-for-user*]]) for the given user, perm type and database, or- nil when the user has no permissions rows for it."- [user-id perm-type db-id]- (if (use-cache? user-id)- (do ; Use the cache if we can- (prime-db-cache [db-id])- (get-in (:perms @*permissions-for-user*) [user-id perm-type db-id]))- ;; If we're checking permissions for a *different* user than ourselves, fetch it straight from the DB. The- ;; entry is read once and discarded, so there is nothing to gain from interning.- (rows->cache-entry identity (relevant-permissions-for-user-perm-and-db user-id perm-type db-id))))--;;; ---------------------------------------- Fetching a user's permissions ---------------------------------------------(defmulti coalesce"Coalesce a set of permission values into a single value. This is used to determine the permission to enforce for auser in multiple groups with conflicting permissions. By default, this returns the *most* permissive value that the@@ -382,6 +176,110 @@(let [ordered-values (-> permissions.schema/data-permissions perm-type :values reverse)](first (filter (set perm-values) ordered-values))))+;;; ---------------------------------------- Caching ------------------------------------------------------------------++;;; Every permission-checking function below has its own request-scoped cache — one atom, one hand-written aggregate+;;; query, one cache structure that fits exactly the question the function answers. All the atoms are bound by+;;; [[with-relevant-permissions-for-user]] and key `:perms` by user ID so we NEVER accidentally use the cache of the+;;; wrong user.+;;;+;;; All coalescing happens in SQL: each permission value is mapped to its rank (see [[value-rank-case]]) and rows+;;; are aggregated with GROUP BY, so result sets scale with tables/schemas/databases rather than with raw+;;; data_permissions row counts (dbs x groups x tables x duplicate values). A bucket's (min, max) rank pair is+;;; enough to reconstruct [[coalesce]] exactly (see [[ranks->most-permissive-value]]), and the+;;; most-restrictive-within-a-group-then-most-permissive-across-groups questions nest two levels of GROUP BY, so+;;; even group identity collapses.+;;;+;;; All caches are point-in-time snapshots for the request: nothing invalidates them when data_permissions is+;;; written, so code that mutates permissions and re-checks them within the same request scope reads pre-write+;;; answers, and the caches may observe a concurrent write at different first-load moments.++(defn- perm-rows-query-base+ "The FROM/JOIN/WHERE shared by every cache load: one user's groups' rows for every permission type, excluding rows+ for deactivated tables. All caches must select from this same row set so they can never answer the same permission+ question differently — change the row set here, not in one of the queries. `db-ids` of nil means every database."+ [user-id db-ids]+ {:from [[(t2/table-name :model/PermissionsGroupMembership) :pgm]]+ :join [[(t2/table-name :model/PermissionsGroup) :pg] [:= :pg.id :pgm.group_id]+ [(t2/table-name :model/DataPermissions) :p] [:= :p.group_id :pg.id]]+ :left-join [[(t2/table-name :model/Table) :mt] [:= :mt.id :p.table_id]]+ :where [:and+ [:= :pgm.user_id user-id]+ (when (seq db-ids)+ [:in :p.db_id db-ids])+ [:or+ [:= :p.table_id nil]+ [:= :mt.active true]]]})++(def ^:private value-rank-case+ "A HoneySQL CASE expression mapping a data_permissions row's (perm_type, perm_value) to the value's rank in its+ permission type's ordering — 0 = most permissive. MIN/MAX aggregates of this rank are what let SQL collapse+ groups, tables and duplicate values."+ (into [:case]+ cat+ (for [[perm-type {:keys [values]}] permissions.schema/data-permissions+ [i v] (map-indexed vector values)]+ [[:and+ [:= :p.perm_type (u/qualified-name perm-type)]+ [:= :p.perm_value (u/qualified-name v)]]+ [:inline i]])))++(defn- ranks->most-permissive-value+ "Reconstructs [[coalesce]] over a bucket of values from the bucket's `[min-rank max-rank]` pair. Verified+ exhaustively against [[coalesce]] for every permission type and value subset: the most permissive value is the one+ at the minimum rank, except for :perms/view-data's special rule where a :blocked row overrides+ :legacy-no-self-service (but not :unrestricted). Returns nil for a nil pair (no rows), matching what coalescing an+ empty value set returns for callers that fall back to the least permissive value."+ [perm-type pair]+ (when-let [[mn mx] pair]+ (let [^PersistentVector values (-> permissions.schema/data-permissions perm-type :values)]+ (if (and (= perm-type :perms/view-data)+ (= (nth values mx) :blocked)+ (not= (nth values mn) :unrestricted))+ :blocked+ (nth values mn)))))++(defn- combine-rank-pairs+ "Merges `[min max]` rank pairs — equivalent to coalescing the union of the buckets they summarize."+ [& pairs]+ (when-let [pairs (seq (remove nil? pairs))]+ [(apply min (map first pairs))+ (apply max (map second pairs))]))++(def ^:dynamic *use-perms-cache?*+ "Bind to `false` to intentionally bypass the permissions caches and fetch data straight from the DB."+ true)++(defmacro disable-perms-cache+ "Intentionally bypass the permissions caches and fetch data straight from the DB."+ {:style/indent 0}+ [& body]+ `(binding [*use-perms-cache?* false]+ ~@body))++(defn- use-cache?+ [user-id]+ (and *use-perms-cache?*+ (= user-id api/*current-user-id*)))++(defn- cached-db-perms+ "The shared read-through pattern for the per-database caches below: returns the per-user `{perm-type {db-id v}}`+ map from `cache-atom` (shaped `{:db-ids #{} :perms {user-id ...}}`), loading missing `db-ids` with+ `(load-fn user-id db-ids)` first. Loads straight through without caching when the request cache doesn't apply."+ [cache-atom load-fn user-id db-ids]+ (if (use-cache? user-id)+ (do+ (let [missing-db-ids (into [] (remove (:db-ids @cache-atom)) db-ids)]+ (when (seq missing-db-ids)+ (let [loaded (load-fn user-id missing-db-ids)]+ (swap! cache-atom+ (fn [cache]+ (-> cache+ (update :db-ids into missing-db-ids)+ (update-in [:perms user-id] #(merge-with merge % loaded))))))))+ (get-in @cache-atom [:perms user-id]))+ (load-fn user-id db-ids)))+(defn is-superuser?"Returns true if the given user ID is a superuser. Avoids a DB query when checking the current user."[user-id]@@ -396,26 +294,28 @@api/*is-data-analyst?*(t2/select-one-fn :is_data_analyst :model/User :id user-id)))-(mu/defn database-permission-for-user :- ::permissions.schema/data-permission-value- "Returns the effective permission value for a given user, permission type, and database ID. If the user has- multiple permissions for the given type in different groups, they are coalesced into a single value."- [user-id perm-type database-id]- (when (not= :model/Database (model-by-perm-type perm-type))- (throw (ex-info (tru "Permission type {0} is a table-level permission." perm-type)- {perm-type (permissions.schema/data-permissions perm-type)})))- (if (is-superuser? user-id)- (most-permissive-value perm-type)- (let [perm-values (into #{} cat (vals (:groups (get-permissions user-id perm-type database-id))))]- (or (coalesce perm-type perm-values)- (least-permissive-value perm-type)))))+;;; ---------------------------------- table-permission-for-user -------------------------------------------------------(mu/defn user-has-permission-for-database? :- :boolean- "Returns a Boolean indicating whether the user has the specified permission value for the given database ID and table ID,- or a more permissive value."- [user-id perm-type perm-value database-id]- (at-least-as-permissive? perm-type- (database-permission-for-user user-id perm-type database-id)- perm-value))+(def ^:dynamic *table-permission-cache*+ "Request cache for [[table-permission-for-user]]: `{:db-ids #{} :perms {user-id {perm-type {db-id entry}}}}` where+ each entry is `{:db-value v, :tables {table-id v}}` — one coalesced value per table with table-granular rows, plus+ the coalesced db-level value. Loaded per database, all permission types at once, one row per (type, db, table)."+ (atom {:db-ids #{} :perms {}}))++(defn- load-table-permission-perms+ [user-id db-ids]+ (reduce (fn [m {:keys [perm_type db_id table_id mn mx]}]+ (let [perm-type (keyword perm_type)+ value (ranks->most-permissive-value perm-type [mn mx])]+ (if table_id+ (assoc-in m [perm-type db_id :tables table_id] value)+ (assoc-in m [perm-type db_id :db-value] value))))+ {}+ (t2/query (assoc (perm-rows-query-base user-id db-ids)+ :select [:p.perm_type :p.db_id :p.table_id+ [[:min value-rank-case] :mn]+ [[:max value-rank-case] :mx]]+ :group-by [:p.perm_type :p.db_id :p.table_id]))))(def ^:dynamic *additional-table-permissions*"See the `with-additional-table-permission` macro below."@@ -438,36 +338,6 @@(defn- get-additional-table-permission! [{:keys [db-id table-id]} perm-type](get-in *additional-table-permissions* [db-id table-id perm-type]))-(mu/defn table-permission-for-groups :- ::permissions.schema/data-permission-value- "Returns the effective permission value provided by a set of *group-ids*, for a provided permission type, database- ID, and table ID."- [group-ids perm-type database-id table-id]- (when (not= :model/Table (model-by-perm-type perm-type))- (throw (ex-info (tru "Permission type {0} is not a table-level permission." perm-type)- {perm-type (permissions.schema/data-permissions perm-type)})))- (let [perm-values (t2/select-fn-set :value- :model/DataPermissions- {:select [[:p.perm_value :value]]- :from [[:data_permissions :p]]- :where [:and- [:in :p.group_id group-ids]- [:= :p.perm_type (u/qualified-name perm-type)]- [:= :p.db_id database-id]- [:or- [:= :table_id table-id]- [:= :table_id nil]]]})]- (or (coalesce perm-type (conj perm-values (get-additional-table-permission! {:db-id database-id :table-id table-id}- perm-type)))- (least-permissive-value perm-type))))--(mu/defn groups-have-permission-for-table? :- :boolean- "Returns a Boolean indicating whether the provided groups grant the specified permission level or higher for the given- table ID, or a more permissive value. (i.e. if a user is in all of these groups, would they have this permission?)"- [group-ids perm-type perm-value database-id table-id]- (at-least-as-permissive? perm-type- (table-permission-for-groups group-ids perm-type database-id table-id)- perm-value))-(mu/defn table-permission-for-user :- ::permissions.schema/data-permission-value"Returns the effective permission value for a given user, permission type, and database ID, and table ID. If the userhas multiple permissions for the given type in different groups, they are coalesced into a single value."@@ -484,11 +354,16 @@(most-permissive-value perm-type):else- (let [{:keys [groups tables]} (get-permissions user-id perm-type database-id)- perm-values (-> (into #{} cat (vals groups))- (into (comp cat (map :value)) (vals (get tables table-id))))- table-perm (coalesce perm-type (conj perm-values (get-additional-table-permission! {:db-id database-id :table-id table-id}- perm-type)))]+ (let [entry (get-in (cached-db-perms *table-permission-cache* load-table-permission-perms+ user-id [database-id])+ [perm-type database-id])+ table-perm (coalesce perm-type+ (into #{}+ (remove nil?)+ [(:db-value entry)+ (get (:tables entry) table-id)+ (get-additional-table-permission! {:db-id database-id :table-id table-id}+ perm-type)]))](or (when-not (= table-perm (least-permissive-value perm-type))table-perm)(when (pos-int? table-id)@@ -503,36 +378,92 @@(table-permission-for-user user-id perm-type database-id table-id)perm-value))-(defn- most-restrictive-per-group- "Given a perm-type and a collection of maps that look like `{:group-id 1 :value :permission-value}`, returns a set- containing the most restrictive permission value in each group."- [perm-type perms]- (->> perms- (group-by :group-id)- (m/map-vals (fn [ps]- (->> ps (map :value) set (coalesce-most-restrictive perm-type))))- vals- set))--(defn- entry-group-value-pairs- "Returns `{:group-id g :value v}` maps for every db-level row in a cache entry, plus every table-level row whose- schema-name satisfies `schema-match?`."- [{:keys [groups tables]} schema-match?]- (into (reduce-kv (fn [acc group-id values]- (reduce #(conj %1 {:group-id group-id :value %2}) acc values))- []- groups)- (for [[_table-id group-id->rows] tables- [group-id rows] group-id->rows- {:keys [schema value]} rows- :when (schema-match? schema)]- {:group-id group-id :value value})))+;;; -------------------------------- database-permission-for-user ------------------------------------------------------(mu/defn full-schema-permission-for-user :- ::permissions.schema/data-permission-value+(def ^:dynamic *database-permission-cache*+ "Request cache for [[database-permission-for-user]]: `{:db-ids #{} :perms {user-id {perm-type {db-id value}}}}` —+ the coalesced db-level value. Loaded per database, all permission types at once, one row per (type, db)."+ (atom {:db-ids #{} :perms {}}))++(defn- load-database-permission-perms+ [user-id db-ids]+ (reduce (fn [m {:keys [perm_type db_id mn mx]}]+ (let [perm-type (keyword perm_type)]+ (assoc-in m [perm-type db_id] (ranks->most-permissive-value perm-type [mn mx]))))+ {}+ (t2/query (-> (perm-rows-query-base user-id db-ids)+ (assoc :select [:p.perm_type :p.db_id+ [[:min value-rank-case] :mn]+ [[:max value-rank-case] :mx]]+ :group-by [:p.perm_type :p.db_id])+ (update :where conj [:= :p.table_id nil])))))++(mu/defn database-permission-for-user :- ::permissions.schema/data-permission-value+ "Returns the effective permission value for a given user, permission type, and database ID. If the user has+ multiple permissions for the given type in different groups, they are coalesced into a single value."+ [user-id perm-type database-id]+ (when (not= :model/Database (model-by-perm-type perm-type))+ (throw (ex-info (tru "Permission type {0} is a table-level permission." perm-type)+ {perm-type (permissions.schema/data-permissions perm-type)})))+ (if (is-superuser? user-id)+ (most-permissive-value perm-type)+ (or (get-in (cached-db-perms *database-permission-cache* load-database-permission-perms+ user-id [database-id])+ [perm-type database-id])+ (least-permissive-value perm-type))))++(mu/defn user-has-permission-for-database? :- :boolean+ "Returns a Boolean indicating whether the user has the specified permission value for the given database ID and table ID,+ or a more permissive value."+ [user-id perm-type perm-value database-id]+ (at-least-as-permissive? perm-type+ (database-permission-for-user user-id perm-type database-id)+ perm-value))++;;; --------------------------------- schema-permission-for-user ------------------------------------------------------++(def ^:dynamic *schema-permission-cache*+ "Request cache for [[schema-permission-for-user]]: `{:db-ids #{} :perms {user-id {perm-type {db-id entry}}}}`+ where each entry is `{:default v, :schemas {schema v}}` — per schema, the coalesced value of the schema's table+ rows combined with the db-level rows; `:default` (the db-level value alone) answers schemas with no rows of their+ own. Schema names are normalized (nil = \"\"). Loaded per database, all permission types at once."+ (atom {:db-ids #{} :perms {}}))++(defn- load-schema-permission-perms+ [user-id db-ids]+ (let [table-level-case [:case [:= :p.table_id nil] [:inline 0] :else [:inline 1]]+ folded (reduce (fn [m {:keys [perm_type db_id schema_name table_level mn mx]}]+ (if (pos? table_level)+ (update-in m [(keyword perm_type) db_id :schemas (or schema_name "")]+ combine-rank-pairs [mn mx])+ (assoc-in m [(keyword perm_type) db_id :db-level] [mn mx])))+ {}+ (t2/query (assoc (perm-rows-query-base user-id db-ids)+ :select [:p.perm_type :p.db_id :p.schema_name+ [table-level-case :table_level]+ [[:min value-rank-case] :mn]+ [[:max value-rank-case] :mx]]+ :group-by [:p.perm_type :p.db_id :p.schema_name table-level-case])))]+ (into {}+ (map (fn [[perm-type db-id->folded]]+ [perm-type+ (update-vals db-id->folded+ (fn [{:keys [db-level schemas]}]+ {:default (ranks->most-permissive-value perm-type db-level)+ :schemas (update-vals schemas+ #(ranks->most-permissive-value+ perm-type+ (combine-rank-pairs db-level %)))}))]))+ folded)))++(mu/defn schema-permission-for-user :- ::permissions.schema/data-permission-value"Returns the effective *schema-level* permission value for a given user, permission type, and database ID, andschema name. If the user has multiple permissions for the given type in different groups, they are coalesced into a- single value. The schema-level permission is the *most* restrictive table-level permission within that schema."- [user-id perm-type database-id schema-name]+ single value. The schema-level permission is the *least* restrictive table-level permission within that schema.++ Schema names are compared with nil and the empty string treated as equivalent, matching how `database-schemas`+ presents these databases to the API."+ [user-id perm-type database-id schema-name :- [:maybe :string]](when (not= :model/Table (model-by-perm-type perm-type))(throw (ex-info (tru "Permission type {0} is not a table-level permission." perm-type){perm-type (permissions.schema/data-permissions perm-type)})))@@ -545,16 +476,44 @@(most-permissive-value perm-type):else- ;; The schema-level permission is the most-restrictive table-level permission within a schema. So for each group,- ;; select the most-restrictive table-level permission. Then use normal coalesce logic to select the *least*- ;; restrictive group permission.- (let [perm-values (most-restrictive-per-group- perm-type- (entry-group-value-pairs (get-permissions user-id perm-type database-id)- #(= % schema-name)))]- (or (coalesce perm-type perm-values)+ (let [{:keys [default schemas]} (get-in (cached-db-perms *schema-permission-cache* load-schema-permission-perms+ user-id [database-id])+ [perm-type database-id])]+ (or (get schemas (or schema-name ""))+ default(least-permissive-value perm-type)))))+(mu/defn user-has-permission-for-schema? :- :boolean+ "Returns a Boolean indicating whether the user has the specified permission value for the given database ID and schema,+ or a more permissive value."+ [user-id perm-type perm-value database-id schema]+ (at-least-as-permissive? perm-type+ (schema-permission-for-user user-id perm-type database-id schema)+ perm-value))++;;; --------------------------------- full-db-permission-for-user -----------------------------------------------------++(def ^:dynamic *full-db-permission-cache*+ "Request cache for [[full-db-permission-for-user]]: `{:db-ids #{} :perms {user-id {perm-type {db-id value}}}}` —+ per database, each group's most restrictive value across the whole database, coalesced most-permissively across+ groups. Group identity collapses inside the nested query. Loaded per database, all permission types at once."+ (atom {:db-ids #{} :perms {}}))++(defn- load-full-db-permission-perms+ [user-id db-ids]+ (reduce (fn [m {:keys [perm_type db_id mn mx]}]+ (let [perm-type (keyword perm_type)]+ (assoc-in m [perm-type db_id] (ranks->most-permissive-value perm-type [mn mx]))))+ {}+ (t2/query {:select [:i.perm_type :i.db_id+ [[:min :i.gmax] :mn] [[:max :i.gmax] :mx]]+ :from [[(assoc (perm-rows-query-base user-id db-ids)+ :select [:p.perm_type :p.db_id :p.group_id+ [[:max value-rank-case] :gmax]]+ :group-by [:p.perm_type :p.db_id :p.group_id])+ :i]]+ :group-by [:i.perm_type :i.db_id]})))+(mu/defn full-db-permission-for-user :- ::permissions.schema/data-permission-value"Returns the effective *db-level* permission value for a given user, permission type, and database ID. If the userhas multiple permissions for the given type in different groups, they are coalesced into a single value. The@@ -572,24 +531,78 @@(most-permissive-value perm-type):else- ;; The DB-level permission is the most-restrictive table-level permission within a DB. So for each group, select the- ;; most-restrictive table-level permission. Then use normal coalesce logic to select the *least* restrictive group- ;; permission.- (let [perm-values (most-restrictive-per-group- perm-type- (entry-group-value-pairs (get-permissions user-id perm-type database-id)- (constantly true)))]- (or (coalesce perm-type perm-values)- (least-permissive-value perm-type)))))+ (or (get-in (cached-db-perms *full-db-permission-cache* load-full-db-permission-perms+ user-id [database-id])+ [perm-type database-id])+ (least-permissive-value perm-type))))-(mu/defn schema-permission-for-user :- ::permissions.schema/data-permission-value+(mu/defn native-download-permission-for-user :- ::permissions.schema/data-permission-value+ "Returns the effective download permission value for a given user and database ID, for native queries on the database.+ For each group, the native download permission for a database is equal to the lowest permission level of any table in+ the database — exactly [[full-db-permission-for-user]] for :perms/download-results, whose cache it shares."+ [user-id :- ::lib.schema.id/user+ database-id :- ::lib.schema.id/database]+ (full-db-permission-for-user user-id :perms/download-results database-id))++;;; -------------------------------- full-schema-permission-for-user --------------------------------------------------++(def ^:dynamic *full-schema-permission-cache*+ "Request cache for [[full-schema-permission-for-user]]: `{:db-ids #{} :perms {user-id {perm-type {db-id entry}}}}`+ where each entry is `{:default v, :schemas {schema v}}` — per schema, each group's most restrictive value across+ the schema's table rows unioned with the group's db-level rows, coalesced most-permissively across groups (the+ nested query expands db-level rows into every schema to keep the per-group pairing). `:default` — the same+ aggregation over db-level rows alone — answers schemas with no rows of their own. Schema names are exact. Loaded+ per database, all permission types at once."+ (atom {:db-ids #{} :perms {}}))++(defn- load-full-schema-permission-perms+ [user-id db-ids]+ (let [base (perm-rows-query-base user-id db-ids)+ with-where (fn [q extra-clause]+ (cond-> q extra-clause (update :where conj extra-clause)))+ branch (fn [bucket schema-expr extra-clause]+ (-> base+ (assoc :select [:p.perm_type :p.db_id [schema-expr :schema_name]+ [[:inline bucket] :bucket] :p.group_id+ [value-rank-case :r]])+ (with-where extra-clause)))+ schemas-sub {:select-distinct [[:p.perm_type :perm_type] [:p.db_id :db_id] [:p.schema_name :schema_name]]+ :from (:from base)+ :join (:join base)+ :left-join (:left-join base)+ :where (conj (:where base) [:not= :p.table_id nil])}+ expanded (-> base+ (update :join conj [schemas-sub :s] [:and+ [:= :s.perm_type :p.perm_type]+ [:= :s.db_id :p.db_id]])+ (assoc :select [:p.perm_type :p.db_id [:s.schema_name :schema_name]+ [[:inline "schema"] :bucket] :p.group_id+ [value-rank-case :r]])+ (update :where conj [:= :p.table_id nil]))]+ (reduce (fn [m {:keys [perm_type db_id schema_name bucket mn mx]}]+ (let [perm-type (keyword perm_type)+ value (ranks->most-permissive-value perm-type [mn mx])]+ (case bucket+ "schema" (assoc-in m [perm-type db_id :schemas schema_name] value)+ "db" (assoc-in m [perm-type db_id :default] value))))+ {}+ (t2/query {:select [:i.perm_type :i.db_id :i.schema_name :i.bucket+ [[:min :i.gmax] :mn] [[:max :i.gmax] :mx]]+ :from [[{:select [:u.perm_type :u.db_id :u.schema_name :u.bucket :u.group_id+ [[:max :u.r] :gmax]]+ :from [[{:union-all [(branch "schema" :p.schema_name [:not= :p.table_id nil])+ expanded+ (branch "db" nil [:= :p.table_id nil])]}+ :u]]+ :group-by [:u.perm_type :u.db_id :u.schema_name :u.bucket :u.group_id]}+ :i]]+ :group-by [:i.perm_type :i.db_id :i.schema_name :i.bucket]}))))++(mu/defn full-schema-permission-for-user :- ::permissions.schema/data-permission-value"Returns the effective *schema-level* permission value for a given user, permission type, and database ID, andschema name. If the user has multiple permissions for the given type in different groups, they are coalesced into a- single value. The schema-level permission is the *least* restrictive table-level permission within that schema.-- Schema names are compared with nil and the empty string treated as equivalent, matching how `database-schemas`- presents these databases to the API."- [user-id perm-type database-id schema-name :- [:maybe :string]]+ single value. The schema-level permission is the *most* restrictive table-level permission within that schema."+ [user-id perm-type database-id schema-name](when (not= :model/Table (model-by-perm-type perm-type))(throw (ex-info (tru "Permission type {0} is not a table-level permission." perm-type){perm-type (permissions.schema/data-permissions perm-type)})))@@ -602,28 +615,53 @@(most-permissive-value perm-type):else- ;; The schema-level permission is the most-restrictive table-level permission within a schema. So for each group,- ;; select the most-restrictive table-level permission. Then use normal coalesce logic to select the *least*- ;; restrictive group permission.- (let [schema-name (or schema-name "")- perm-values (into #{}- (map :value)- (entry-group-value-pairs (get-permissions user-id perm-type database-id)- #(= (or % "") schema-name)))]- (or (coalesce perm-type perm-values)+ (let [{:keys [default schemas]} (get-in (cached-db-perms *full-schema-permission-cache*+ load-full-schema-permission-perms+ user-id [database-id])+ [perm-type database-id])]+ (or (get schemas schema-name)+ default(least-permissive-value perm-type)))))-(mu/defn user-has-permission-for-schema? :- :boolean- "Returns a Boolean indicating whether the user has the specified permission value for the given database ID and schema,- or a more permissive value."- [user-id perm-type perm-value database-id schema]- (at-least-as-permissive? perm-type- (schema-permission-for-user user-id perm-type database-id schema)- perm-value))+;;; ------------------- most-permissive-database-permission-for-user / user-has-any-perms-of-type? --------------------++(def ^:dynamic *most-permissive-permission-cache*+ "Request cache for [[most-permissive-database-permission-for-user]] and [[user-has-any-perms-of-type?]] — the+ cross-database questions: `{user-id {perm-type {db-id value}}}`, one coalesced value per database, loaded for+ *every* database in one aggregated query per permission type. Loading per-row permission data for every database+ instead stops scaling once instances have thousands of databases and groups."+ (atom {}))++(defn- load-most-permissive-db-perms+ [user-id perm-type]+ (reduce (fn [m {:keys [db_id mn mx]}]+ (assoc m db_id (ranks->most-permissive-value perm-type [mn mx])))+ {}+ (t2/query (-> (perm-rows-query-base user-id nil)+ (assoc :select [:p.db_id+ [[:min value-rank-case] :mn]+ [[:max value-rank-case] :mx]]+ :group-by [:p.db_id])+ (update :where conj [:= :p.perm_type (u/qualified-name perm-type)])))))++(defn- most-permissive-db-values+ "Returns `{db-id most-permissive-value}` for `perm-type` across every database, memoized per request in+ [[*most-permissive-permission-cache*]]. Databases where the user has no permission rows are absent from the map+ (they coalesce to the least permissive value)."+ [user-id perm-type]+ (if (use-cache? user-id)+ (or (get-in @*most-permissive-permission-cache* [user-id perm-type])+ (let [db-id->value (load-most-permissive-db-perms user-id perm-type)]+ (swap! *most-permissive-permission-cache* assoc-in [user-id perm-type] db-id->value)+ db-id->value))+ (load-most-permissive-db-perms user-id perm-type)))(mu/defn most-permissive-database-permission-for-user :- ::permissions.schema/data-permission-value"Similar to checking _partial_ permissions with permissions paths - what is the *most permissive* permission the- user has on any of the tables within this database?"+ user has on any of the tables within this database?++ Backed by its own light per-request cache — one value per database for the permission type — so checks looping+ over every database cost a single query however many permission rows exist."[user-id perm-type database-id](when (not= :model/Table (model-by-perm-type perm-type))(throw (ex-info (tru "Permission type {0} is not a table-level permission." perm-type)@@ -637,46 +675,107 @@(most-permissive-value perm-type):else- (let [perm-values (into #{}- (map :value)- (entry-group-value-pairs (get-permissions user-id perm-type database-id)- (constantly true)))]- (or (coalesce perm-type perm-values)- (least-permissive-value perm-type)))))--(mu/defn native-download-permission-for-user :- ::permissions.schema/data-permission-value- "Returns the effective download permission value for a given user and database ID, for native queries on the database.- For each group, the native download permission for a database is equal to the lowest permission level of any table in- the database."- [user-id :- ::lib.schema.id/user- database-id :- ::lib.schema.id/database]- (if (is-superuser? user-id)- (most-permissive-value :perms/download-results)- (let [pairs (entry-group-value-pairs (get-permissions user-id :perms/download-results database-id)- (constantly true))]- (or (coalesce :perms/download-results- (most-restrictive-per-group :perms/download-results pairs))- (least-permissive-value :perms/download-results)))))+ (or (get (most-permissive-db-values user-id perm-type) database-id)+ (least-permissive-value perm-type))))(mu/defn user-has-any-perms-of-type? :- :boolean"Returns a Boolean indicating whether the user has the highest level of access for the given permission type in any- group, for at least one database or table. Optionally takes `:exclude-db-ids` to exclude specific databases from the check."+ group, for at least one database or table. Optionally takes `:exclude-db-ids` to exclude specific databases from the+ check.++ Answered from [[*most-permissive-permission-cache*]] — one aggregated query, memoized per request — so callers+ that invoke this repeatedly within one request (e.g. once per snippet in a list) cost at most one query, however+ many databases the instance has. Note that permission rows for inactive tables do not count."[user-id perm-type & {:keys [exclude-db-ids]}](or (is-superuser? user-id)(and (= perm-type :perms/manage-table-metadata)(is-data-analyst? user-id))- (let [value (most-permissive-value perm-type)]- (t2/exists? :model/DataPermissions- {:select [[:p.perm_value :value]]- :from [[:permissions_group_membership :pgm]]- :join [[:permissions_group :pg] [:= :pg.id :pgm.group_id]- [:data_permissions :p] [:= :p.group_id :pg.id]]- :where (into [:and- [:= :pgm.user_id user-id]- [:= :p.perm_type (u/qualified-name perm-type)]- [:= :p.perm_value (u/qualified-name value)]]- (when (seq exclude-db-ids)- [[:not-in :p.db_id exclude-db-ids]]))}))))+ (let [value (most-permissive-value perm-type)+ exclude? (set exclude-db-ids)]+ (boolean (some (fn [[db-id db-value]]+ (and (= db-value value)+ (not (exclude? db-id))))+ (most-permissive-db-values user-id perm-type))))))++;;; --------------------------------------- cache assembly ------------------------------------------------------------++(defenterprise enforced-sandboxes-for-user+ "Given a user-id, returns the set of sandboxes that should be enforced for the provided user ID. This result is+ cached for the duration of a request. Empty on OSS instances."+ metabase-enterprise.sandbox.api.util+ [_user-id]+ #{})++(def ^:dynamic *sandboxes-for-user*+ "Filled by `enforced-sandboxes-for-user`. Empty on OSS instances, or EE instances without the `sandboxes` feature."+ (delay nil))++(defn sandboxes-for-user+ "Derefs the *sandboxes-for-user* dynamic variable so it can be contained to this namespace"+ []+ @*sandboxes-for-user*)++(defmacro with-relevant-permissions-for-user+ "Populates the per-function permission caches above (and `*sandboxes-for-user*`) for use by the cache-aware+ functions in this namespace."+ [user-id & body]+ `(binding [*table-permission-cache* (atom {:db-ids #{} :perms {}})+ *database-permission-cache* (atom {:db-ids #{} :perms {}})+ *schema-permission-cache* (atom {:db-ids #{} :perms {}})+ *full-db-permission-cache* (atom {:db-ids #{} :perms {}})+ *full-schema-permission-cache* (atom {:db-ids #{} :perms {}})+ *most-permissive-permission-cache* (atom {})+ *sandboxes-for-user* (delay (enforced-sandboxes-for-user ~user-id))]+ ~@body))++(defn prime-db-perms-cache+ "Eagerly load every per-database permission cache for the given database IDs for the current user. The checks fill+ their caches one database at a time, so call this before checking permissions across many databases to pay a+ handful of small aggregate queries instead of some per database. A no-op for superusers (their checks never read+ the caches) and when the request cache is unavailable."+ [db-ids]+ (when (and (use-cache? api/*current-user-id*)+ (not api/*is-superuser?*)+ (seq db-ids))+ (doseq [[cache-atom load-fn] [[*table-permission-cache* load-table-permission-perms]+ [*database-permission-cache* load-database-permission-perms]+ [*schema-permission-cache* load-schema-permission-perms]+ [*full-db-permission-cache* load-full-db-permission-perms]+ [*full-schema-permission-cache* load-full-schema-permission-perms]]]+ (cached-db-perms cache-atom load-fn api/*current-user-id* db-ids))+ nil))++;;; ---------------------------------------- Fetching a user's permissions --------------------------------------------++(mu/defn table-permission-for-groups :- ::permissions.schema/data-permission-value+ "Returns the effective permission value provided by a set of *group-ids*, for a provided permission type, database+ ID, and table ID."+ [group-ids perm-type database-id table-id]+ (when (not= :model/Table (model-by-perm-type perm-type))+ (throw (ex-info (tru "Permission type {0} is not a table-level permission." perm-type)+ {perm-type (permissions.schema/data-permissions perm-type)})))+ (let [perm-values (t2/select-fn-set :value+ :model/DataPermissions+ {:select [[:p.perm_value :value]]+ :from [[:data_permissions :p]]+ :where [:and+ [:in :p.group_id group-ids]+ [:= :p.perm_type (u/qualified-name perm-type)]+ [:= :p.db_id database-id]+ [:or+ [:= :table_id table-id]+ [:= :table_id nil]]]})]+ (or (coalesce perm-type (conj perm-values (get-additional-table-permission! {:db-id database-id :table-id table-id}+ perm-type)))+ (least-permissive-value perm-type))))++(mu/defn groups-have-permission-for-table? :- :boolean+ "Returns a Boolean indicating whether the provided groups grant the specified permission level or higher for the given+ table ID, or a more permissive value. (i.e. if a user is in all of these groups, would they have this permission?)"+ [group-ids perm-type perm-value database-id table-id]+ (at-least-as-permissive? perm-type+ (table-permission-for-groups group-ids perm-type database-id table-id)+ perm-value))(defn- admin-permission-graph"Returns the graph representing admin permissions for all groups"@@ -760,6 +859,7 @@(assert-value-matches-perm-type perm_type perm_value));; Memoized per application DB. Destination status is immutable after creation, so the cache can't go stale.+(def ^:private destination-db-id?"Whether `db-id` is a destination database — one with `router_database_id` set."(mdb/memoize-for-application-db@@ -1184,9 +1284,11 @@:where [:and[:in :group_id group-ids][:in :perm_type ["perms/create-queries" "perms/download-results"]]- [:not-in :db_id {:select [:id]- :from [(t2/table-name :model/Database)]- :where [:= :is_audit true]}]]}))+ [:not [:exists {:select [1]+ :from [[(t2/table-name :model/Database) :audit_db]]+ :where [:and+ [:= :audit_db.is_audit true]+ [:= :audit_db.id :data_permissions.db_id]]}]]]}));; Group by (group_id, perm_type) → set of valuesperms-by-grp (when all-perms(reduce (fn [acc {:keys [group_id perm_type perm_value]}]diff --git a/src/metabase/permissions/user.clj b/src/metabase/permissions/user.cljindex a170bd8cf3c..c3f873c3e67 100644--- a/src/metabase/permissions/user.clj+++ b/src/metabase/permissions/user.clj@@ -49,7 +49,6 @@based on their create-queries permissions across all databases."[user-id](let [db-ids (t2/select-pks-set :model/Database)- _ (data-perms/prime-db-cache db-ids)create-query-perms (into #{}(map #(data-perms/most-permissive-database-permission-for-useruser-id :perms/create-queries %))diff --git a/src/metabase/permissions_rest/data_permissions/graph.clj b/src/metabase/permissions_rest/data_permissions/graph.cljindex 49cd95f5626..0f6dd347d5b 100644--- a/src/metabase/permissions_rest/data_permissions/graph.clj+++ b/src/metabase/permissions_rest/data_permissions/graph.clj@@ -211,9 +211,8 @@Rows are fetched with a raw query rather than a model select, and realized one at a time: key access on unrealizedresult-set rows goes through toucan2's deferred-row machinery on every lookup, which benchmarked ~15x slower than- realizing each row once and reading plain map keys (see- [[metabase.permissions.models.data-permissions/relevant-permissions-for-user-and-dbs]]). The raw query skips the- model transforms, so `:type` and `:value` arrive as strings and are keywordized here."+ realizing each row once and reading plain map keys. The raw query skips the model transforms, so `:type` and+ `:value` arrive as strings and are keywordized here."[{:keys [group-id group-ids db-id perm-type audit?]}](eduction(map (fn [row]@@ -234,9 +233,11 @@(when group-id [:= :group_id group-id])(when group-ids [:in :group_id group-ids])(when-not audit? [:not= :db_id audit/audit-db-id])- [:not-in :db_id {:select [:id]- :from [(t2/table-name :model/Database)]- :where [:not= :router_database_id nil]}]]+ [:not [:exists {:select [1]+ :from [[(t2/table-name :model/Database) :router_db]]+ :where [:and+ [:not= :router_db.router_database_id nil]+ [:= :router_db.id :db_id]]}]]]:order-by [:group_id :db_id]})))(defn- add-permdiff --git a/src/metabase/pulse/task/send_pulses.clj b/src/metabase/pulse/task/send_pulses.cljindex bbc65e886b8..5c8d9346c1f 100644--- a/src/metabase/pulse/task/send_pulses.clj+++ b/src/metabase/pulse/task/send_pulses.clj@@ -110,11 +110,12 @@(tracing/with-span :tasks "task.pulse.clear-orphan-channels" {:pulse/id pulse-id}(when-let [ids-to-delete (seq(for [channel (t2/select [:model/PulseChannel :id :details :channel_id :channel_type]- :pulse_id pulse-id- :id [:not-in {:select [[:pulse_channel_id :id]]- :from :pulse_channel_recipient- :group-by [:pulse_channel_id]- :having [:>= :%count.* [:raw 1]]}])+ {:where [:and+ [:= :pulse_id pulse-id]+ [:not [:exists {:select [1]+ :from [:pulse_channel_recipient]+ :where [:= :pulse_channel_recipient.pulse_channel_id+ :pulse_channel.id]}]]]}):when (case (:channel_type channel)(empty? (get-in channel [:details :emails]))diff --git a/src/metabase/queries/metadata.clj b/src/metabase/queries/metadata.cljindex e7c1e4ac120..daeed8ef482 100644--- a/src/metabase/queries/metadata.clj+++ b/src/metabase/queries/metadata.clj@@ -40,7 +40,7 @@(defn- get-databases[ids](when (seq ids)- (perms/prime-db-cache ids)+ (perms/prime-db-perms-cache ids)(into [] (comp (filter mi/can-read?)(map #(assoc % :native_permissions (get-native-perms-info %))))(t2/select :model/Database :id [:in ids]))))diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex 31ea460a604..317d015f9fa 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -260,7 +260,7 @@(query-perms/with-card-instances (when (seq source-card-ids)(t2/select-fn->fn :id identity [:model/Card :id :collection_id :card_schema]:id [:in source-card-ids]))- (perms/prime-db-cache (into #{} (map :database_id cards)))+ (perms/prime-db-perms-cache (into #{} (keep :database_id) cards-with-non-empty-queries))(mi/instances-with-hydrated-datacards :can_run_adhoc_query(fn []diff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.cljindex 2819327bb80..d7e27b0dc01 100644--- a/src/metabase/search/appdb/index.clj+++ b/src/metabase/search/appdb/index.clj@@ -117,21 +117,22 @@(defn- orphan-indexes [](map (comp keyword u/lower-case-en :table_name)- (t2/query {:select [:table_name]- :from :information_schema.tables+ (t2/query {:select [:ist.table_name]+ :from [[:information_schema.tables :ist]]:where [:and- [:= :table_schema :%current_schema]+ [:= :ist.table_schema :%current_schema][:or- [:like [:lower :table_name] [:inline "search\\_index\\_\\_%"]]+ [:like [:lower :ist.table_name] [:inline "search\\_index\\_\\_%"]];; legacy table names- [:in [:lower :table_name]+ [:in [:lower :ist.table_name](mapv #(vector :inline %) ["search_index" "search_index_next" "search_index_retired"])]];; Exclude temp tables — they are managed by with-temp-index-table- [:not-like [:lower :table_name] [:inline "%\\_temp"]]- [:not-in [:lower :table_name]- {:select [:%lower.index_name]- :from [(t2/table-name :model/SearchIndexMetadata)]- :where [:= :engine [:inline "appdb"]]}]]})))+ [:not-like [:lower :ist.table_name] [:inline "%\\_temp"]]+ [:not [:exists {:select [1]+ :from [[(t2/table-name :model/SearchIndexMetadata) :sim]]+ :where [:and+ [:= :sim.engine [:inline "appdb"]]+ [:= [:lower :sim.index_name] [:lower :ist.table_name]]]}]]]})))(defn- delete-obsolete-tables! [];; Delete metadata around indexes that are no longer needed.diff --git a/src/metabase/segments/api.clj b/src/metabase/segments/api.cljindex 192efc85024..9fb31776e4d 100644--- a/src/metabase/segments/api.clj+++ b/src/metabase/segments/api.clj@@ -7,6 +7,7 @@[metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.models.interface :as mi]+ [metabase.permissions.core :as perms][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.log :as log]@@ -75,11 +76,14 @@(api.macros/defendpoint :get "/""Fetch *all* `Segments`."[]- (as-> (t2/select :model/Segment- :archived false- {:order-by [[:%lower.name :asc]]}) segments- (filter mi/can-read? segments)- (t2/hydrate segments :creator :definition_description)))+ (let [segments (t2/select :model/Segment+ :archived false+ {:order-by [[:%lower.name :asc]]})+ table-ids (into #{} (keep :table_id) segments)]+ (when (seq table-ids)+ (perms/prime-db-perms-cache (t2/select-fn-set :db_id :model/Table :id [:in table-ids])))+ (-> (filterv mi/can-read? segments)+ (t2/hydrate :creator :definition_description))))(defn- write-check-and-update-segment!"Check whether current user has write permissions, then update Segment with values in `body`. Publishes appropriatediff --git a/src/metabase/transforms/models/transform_run_cancelation.clj b/src/metabase/transforms/models/transform_run_cancelation.cljindex e801158534b..731e12f99c3 100644--- a/src/metabase/transforms/models/transform_run_cancelation.clj+++ b/src/metabase/transforms/models/transform_run_cancelation.clj@@ -38,19 +38,24 @@[](t2/reducible-select :model/TransformRunCancelation))+(defn- no-active-run-clause+ []+ [:not [:exists {:select [1]+ :from [[:transform_run :wr]]+ :where [:and+ [:= :wr.id :transform_run_cancelation.run_id]+ :wr.is_active]}]])+(defn delete-cancelation!"Delete a cancelation once it has been handled."[run-id](t2/delete! :model/TransformRunCancelation- :run_id run-id- :run_id [:not-in {:select :wr.id- :from [[:transform_run :wr]]- :where :wr.is_active}]))+ {:where [:and+ [:= :run_id run-id]+ (no-active-run-clause)]}))(defn delete-old-canceling-runs!"Delete cancelations for runs that are no longer running."[](t2/delete! :model/TransformRunCancelation- :run_id [:not-in {:select :wr.id- :from [[:transform_run :wr]]- :where :wr.is_active}]))+ {:where (no-active-run-clause)}))diff --git a/src/metabase/typed_schemas/api/schema/table.clj b/src/metabase/typed_schemas/api/schema/table.cljindex e22846875cd..4ac3cd42a3a 100644--- a/src/metabase/typed_schemas/api/schema/table.clj+++ b/src/metabase/typed_schemas/api/schema/table.clj@@ -5,6 +5,7 @@[metabase.lib-be.core :as lib-be][metabase.metabot.core :as metabot][metabase.models.interface :as mi]+ [metabase.permissions.core :as perms][metabase.typed-schemas.api.common :as common][metabase.typed-schemas.api.schema.common :as schema.common][metabase.typed-schemas.api.scope :as scope]@@ -37,6 +38,12 @@:tableId (when (integer? table-id) table-id):defaultTemporalBucket (:unit field)))))+(defn- filter-readable-tables+ "Filters tables down to the ones the current user can read."+ [tables]+ (perms/prime-db-perms-cache (into #{} (map :db_id) tables))+ (filter mi/can-read? tables))+(defn select-tables"Returns readable tables, with optional database and table-id scopes.@@ -50,7 +57,7 @@database-ids (conj (scope/database-id-filter-clause database-ids :db_id))table-ids (conj (scope/id-filter-clause table-ids :id))):order-by [[:name :asc] [:id :asc]]})- (filter mi/can-read?))))+ (filter-readable-tables))))(defn select-library-tables"Returns published tables from the library based on the given scope."@@ -61,7 +68,7 @@[:= :is_published true](scope/id-filter-clause data-collection-ids :collection_id)]:order-by [[:name :asc] [:id :asc]]})- (filter mi/can-read?)))+ (filter-readable-tables)))(defn segment-schema"Returns the schema for a segment."diff --git a/src/metabase/users/models/user.clj b/src/metabase/users/models/user.cljindex 3ffdadfc4bd..9e1cc948a8d 100644--- a/src/metabase/users/models/user.clj+++ b/src/metabase/users/models/user.clj@@ -545,6 +545,18 @@[:like :%lower.last_name (wildcard-query query)][:like :%lower.email (wildcard-query query)]])+(defn- table-metadata-perms-exist-clause+ "EXISTS clause, correlated to :core_user.id, testing whether the user is in a group that grants+ manage-table-metadata."+ []+ [:exists {:select [1]+ :from [[:permissions_group_membership :pgm]]+ :join [[:data_permissions :p] [:= :p.group_id :pgm.group_id]]+ :where [:and+ [:= :pgm.user_id :core_user.id]+ [:= :p.perm_type "perms/manage-table-metadata"]+ [:= :p.perm_value "yes"]]}])+(defn filter-clauses"Honeysql clauses for filtering on users.@@ -571,23 +583,11 @@[:or:core_user.is_data_analyst:core_user.is_superuser- [:in :core_user.id- {:select-distinct [:pgm.user_id]- :from [[:permissions_group_membership :pgm]]- :join [[:data_permissions :p] [:= :p.group_id :pgm.group_id]]- :where [:and- [:= :p.perm_type "perms/manage-table-metadata"]- [:= :p.perm_value "yes"]]}]]+ (table-metadata-perms-exist-clause)][:and[:not :core_user.is_data_analyst][:not :core_user.is_superuser]- [:not-in :core_user.id- {:select-distinct [:pgm.user_id]- :from [[:permissions_group_membership :pgm]]- :join [[:data_permissions :p] [:= :p.group_id :pgm.group_id]]- :where [:and- [:= :p.perm_type "perms/manage-table-metadata"]- [:= :p.perm_value "yes"]]}]]))+ [:not (table-metadata-perms-exist-clause)]]))(some? group-ids) (sql.helpers/right-join:permissions_group_membership[:= :core_user.id :permissions_group_membership.user_id])diff --git a/src/metabase/util/memoize.cljc b/src/metabase/util/memoize.cljcindex 389c430f462..d2c3e29e27b 100644--- a/src/metabase/util/memoize.cljc+++ b/src/metabase/util/memoize.cljc@@ -72,28 +72,36 @@its argument** — it normalizes *within* an equivalence class (pool the contents, reorder amap) without changing equality. The zero-arg arity uses `identity` (intern values as-is).- Concurrent callers converge on a single canonical instance even on a simultaneous miss.+ - `nil` is returned unchanged without calling `canon-fn` or adding an entry to the cache.In CLJ this is backed by a `ConcurrentHashMap`, so it is thread-safe; `canon-fn` may run morethan once for the same value under contention (extra results are discarded), so it must be pure.- Keys and values must be non-nil with a good `Object.hashCode` (Clojure values qualify). Like+ Non-nil values must have a good `Object.hashCode` (Clojure values qualify). Like[[fast-memo]] it doesn't support the memoization API (`memo-swap!`, `memoized?`, etc.).In CLJS (single-threaded) this is a plain atom-backed map with the same contract."([] (fast-interner identity))([canon-fn]+ ;; `if-some` rather than `or` on the lookups so that a cached `false` counts as a hit#?(:clj (let [cache (java.util.concurrent.ConcurrentHashMap.)](fn [x]- (or (.get cache x)+ (when (some? x)+ (if-some [cached (.get cache x)]+ cached(let [canon (canon-fn x)];; key by `canon`, not `x`: `canon` = `x`, so this lookup/insert hits the;; same bin a later equal input would, while retaining the canonical object.- (or (.putIfAbsent cache canon canon) canon)))))+ (if-some [existing (.putIfAbsent cache canon canon)]+ existing+ canon)))))):cljs (let [cache (atom {})](fn [x]- (or (get @cache x)+ (when (some? x)+ (if-some [cached (get @cache x)]+ cached(let [canon (canon-fn x)](swap! cache assoc canon canon)- canon)))))))+ canon))))))))(defn fast-bounded"Variant of [[bounded]] that optimizes a common special case: a function with only a single argument, where thatdiff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.cljindex d7d4cb9879a..2a17638584f 100644--- a/src/metabase/warehouse_schema/models/field.clj+++ b/src/metabase/warehouse_schema/models/field.clj@@ -166,8 +166,9 @@:from [:metabase_field]:where [:and[:= :fk_target_field_id (:id field)]- [:not [:in :id {:select [:field_id]- :from [:metabase_field_user_settings]}]]]}+ [:not [:exists {:select [1]+ :from [:metabase_field_user_settings]+ :where [:= :metabase_field_user_settings.field_id :metabase_field.id]}]]]}sql (sql/format q :dialect (mdb/quoting-style (mdb/db-type)))](t2/insert! :model/FieldUserSettings(map (fn [{:keys [id]}] {:field_id id})diff --git a/src/metabase/warehouse_schema/table.clj b/src/metabase/warehouse_schema/table.cljindex 5dd76c3a58b..22cbec61b05 100644--- a/src/metabase/warehouse_schema/table.clj+++ b/src/metabase/warehouse_schema/table.clj@@ -3,6 +3,7 @@[medley.core :as m][metabase.api.common :as api][metabase.models.interface :as mi]+ [metabase.permissions.core :as perms][metabase.premium-features.core :as premium-features :refer [defenterprise]][metabase.util :as u][metabase.warehouse-schema.models.field-values :as field-values]@@ -69,8 +70,9 @@(batch-fetch-query-metadatas* ids nil))([ids {:keys [include-sensitive-fields?]}](when (seq ids)- (let [tables (->> (t2/select :model/Table :id [:in ids])- (filter can-access-table-for-query-metadata?))+ (let [tables (t2/select :model/Table :id [:in ids])+ _ (perms/prime-db-perms-cache (into #{} (map :db_id) tables))+ tables (filter can-access-table-for-query-metadata? tables)tables (t2/hydrate tables[:fields [:target :has_field_values] :has_field_values :dimensions :name_field]:segments@@ -150,16 +152,33 @@[]"Everything else")+(defn- cards->card-id->metadata-fields+ "Batch-fetch the underlying Fields for the `:result_metadata` columns of `cards`, returning a map of card ID to its+ hydrated Fields. One select (plus hydration) covers every card, instead of the per-card lookup+ [[card-result-metadata->virtual-fields]] falls back to."+ [cards]+ (let [metadata-field-ids (into #{}+ (comp (mapcat :result_metadata)+ (keep :id))+ cards)+ metadata-fields (if (seq metadata-field-ids)+ (-> (t2/select :model/Field :id [:in metadata-field-ids])+ (t2/hydrate [:target :has_field_values] :has_field_values :dimensions :name_field)+ (->> (m/index-by :id)))+ {})]+ (into {}+ (map (fn [card]+ [(:id card) (into []+ (keep (comp metadata-fields :id))+ (:result_metadata card))]))+ cards)))+(defn card->virtual-table"Return metadata for a 'virtual' table for a `card` in the Saved Questions 'virtual' database. Optionally include'virtual' fields as well."[{:keys [database_id] :as card} & {:keys [include-database? include-fields? databases card-id->metadata-fields]}]- ;; if collection isn't already hydrated then do so(let [card-type (:type card)- dataset-query (:dataset_query card)- database (when (int? database_id)- (or (get databases database_id)- (t2/select-one :model/Database :id database_id)))]+ dataset-query (:dataset_query card)](cond-> {:id (str "card__" (u/the-id card)):db_id (:database_id card):display_name (:name card)@@ -174,7 +193,10 @@(assoc :dataset_query dataset-query)include-database?- (assoc :db (when (and database (mi/can-read? database)) database))+ (assoc :db (when-let [database (when (int? database_id)+ (or (get databases database_id)+ (t2/select-one :model/Database :id database_id)))]+ (when (mi/can-read? database) database)))include-fields?(assoc :fields (card-result-metadata->virtual-fields (u/the-id card)@@ -182,6 +204,19 @@(when card-id->metadata-fields(card-id->metadata-fields (u/the-id card))))))))+(defn cards->virtual-tables+ "Return [[card->virtual-table]] metadata for `cards`, batching the Field fetch that `:include-fields?` needs so it+ costs one select for the whole list instead of one per card."+ [cards & {:keys [include-database? include-fields? databases]}]+ (let [card-id->metadata-fields (when include-fields?+ (cards->card-id->metadata-fields cards))]+ (for [card cards]+ (card->virtual-table card+ :include-database? include-database?+ :include-fields? include-fields?+ :databases databases+ :card-id->metadata-fields card-id->metadata-fields))))+(defn- remove-nested-pk-fk-semantic-types"This method clears the semantic_type attribute for PK/FK fields of nested queries. Those fields having a semantictype confuses the frontend and it can really used in the same way"@@ -218,21 +253,7 @@dbs (if (seq cards)(t2/select-pk->fn identity :model/Database :id [:in (into #{} (map :database_id) cards)]){})- metadata-field-ids (into #{}- (comp (mapcat :result_metadata)- (keep :id))- cards)- metadata-fields (if (seq metadata-field-ids)- (-> (t2/select :model/Field :id [:in metadata-field-ids])- (t2/hydrate [:target :has_field_values] :has_field_values :dimensions :name_field)- (->> (m/index-by :id)))- {})- card-id->metadata-fields (into {}- (map (fn [card]- [(:id card) (into []- (keep (comp metadata-fields :id))- (:result_metadata card))]))- cards)+ card-id->metadata-fields (cards->card-id->metadata-fields cards)readable-cards (t2/hydrate (filter mi/can-read? cards) :metrics)](for [card readable-cards];; Models can have user configured FK columns which, for MBQL models, we cannot distinguish fromdiff --git a/src/metabase/warehouse_schema_rest/api/table.clj b/src/metabase/warehouse_schema_rest/api/table.cljindex 214f06bb6bd..0c5f3920d09 100644--- a/src/metabase/warehouse_schema_rest/api/table.clj+++ b/src/metabase/warehouse_schema_rest/api/table.clj@@ -123,6 +123,8 @@(premium-features/any-transforms-enabled?) (conj :transform))](as-> (t2/select :model/Table query) tables(apply t2/hydrate tables hydrations)+ (do (perms/prime-db-perms-cache (into #{} (map :db_id) tables))+ tables)(into [] (comp (filter mi/can-read?)(if can-query (filter mi/can-query?) identity)(if can-write (filter mi/can-write?) identity)diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.cljindex dba916bafd0..84bbe6d2f61 100644--- a/src/metabase/warehouses/models/database.clj+++ b/src/metabase/warehouses/models/database.clj@@ -424,26 +424,32 @@{:pre [(pos-int? database-id)]};; Field has `define-before-delete` deleting children, but we'll delete them all at once because they refer same;; database - iteratively, deleting those that no one depends on first- (loop []- (let [deleted (t2/query-one- {:delete-from (t2/table-name :model/Field)- :where- [:and- [:in :table_id {:from [(t2/table-name :model/Table)]- :select [:id]- :where [:= :db_id database-id]}]- ;; Double-wrapped subquery to work around MySQL limitation- [:not-in :id {:select [:parent_id]- :from [[{:select [:parent_id]- :from [(t2/table-name :model/Field)]- :where [:and- [:not= :parent_id nil]- [:in :table_id {:from [(t2/table-name :model/Table)]- :select [:id]- :where [:= :db_id database-id]}]]}- :parent_fields]]}]]})]- (when (pos? deleted)- (recur)))))+ (let [no-children-clause+ (if (= (mdb/db-type) :mysql)+ ;; double-wrapped subquery to work around the MySQL restriction on selecting from the DELETE target+ [:not-in :id {:select [:parent_id]+ :from [[{:select [:parent_id]+ :from [(t2/table-name :model/Field)]+ :where [:and+ [:not= :parent_id nil]+ [:in :table_id {:from [(t2/table-name :model/Table)]+ :select [:id]+ :where [:= :db_id database-id]}]]}+ :parent_fields]]}]+ [:not [:exists {:select [1]+ :from [[(t2/table-name :model/Field) :child_field]]+ :where [:= :child_field.parent_id :metabase_field.id]}]])]+ (loop []+ (let [deleted (t2/query-one+ {:delete-from (t2/table-name :model/Field)+ :where+ [:and+ [:in :table_id {:from [(t2/table-name :model/Table)]+ :select [:id]+ :where [:= :db_id database-id]}]+ no-children-clause]})]+ (when (pos? deleted)+ (recur))))))(t2/define-before-delete :model/Database[{id :id, driver :engine, :as database}]diff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.cljindex 4abf66dc186..2960f6be5e0 100644--- a/src/metabase/warehouses_rest/api.clj+++ b/src/metabase/warehouses_rest/api.clj@@ -77,6 +77,7 @@:visibility_type nil{:order-by [[:%lower.schema :asc][:%lower.display_name :asc]]})+ _ (perms/prime-db-perms-cache (map :id dbs))filtered-tables (cond->> (filter mi/can-read? all-tables)can-query? (filter mi/can-query?)can-write-metadata? (filter mi/can-write?))@@ -128,7 +129,6 @@native queries, but not to create new ones. With the advent of what is currently being called 'Space-AgePermissions', all Cards' permissions are based on their parent Collection, removing the need for native read perms."[dbs :- [:maybe [:sequential :map]]]- (perms/prime-db-cache (map :id dbs))(for [db dbs](assoc db:native_permissions@@ -200,13 +200,14 @@(card-has-ambiguous-columns? card)))))(defn- ids-of-dbs-that-support-source-queries []- (set (filter (fn [db-id]- (try- (when-let [db (t2/select-one :model/Database :id db-id)]- (driver.u/supports? (driver.u/database->driver db) :nested-queries db))- (catch Throwable e- (log/errorf "Error determining whether Database supports nested queries: %s" (ex-message e)))))- (t2/select-pks-set :model/Database))))+ ;; the nested-queries check only reads the engine — don't realize full rows (decrypted :details etc.) for it+ (set (keep (fn [db]+ (try+ (when (driver.u/supports? (driver.u/database->driver db) :nested-queries db)+ (:id db))+ (catch Throwable e+ (log/errorf "Error determining whether Database supports nested queries: %s" (ex-message e)))))+ (t2/select [:model/Database :id :engine]))))(mu/defn- source-query-cards"Fetch the Cards that can be used as source queries (e.g. presented as virtual tables)."@@ -252,8 +253,8 @@Builder.)"[card-type :- ::queries.schema/card-type& {:keys [include-fields?]}]- (for [card (source-query-cards card-type)]- (schema.table/card->virtual-table card :include-fields? include-fields?)))+ (schema.table/cards->virtual-tables (source-query-cards card-type)+ :include-fields? include-fields?))(mu/defn- saved-cards-virtual-db-metadata[card-type :- ::queries.schema/card-type@@ -347,7 +348,8 @@(:clause (mi/visible-filter-clause :model/Database :id user-info {:perms/manage-table-metadata :yes}))]]base-where)dbs (t2/select :model/Database {:order-by [:%lower.name :%lower.engine]- :where where-clause})]+ :where where-clause})+ _ (perms/prime-db-perms-cache (map :id dbs))](cond-> (-> dbs add-native-perms-info add-transforms-perms-info)include-tables? (add-tables :can-query? can-query? :can-write-metadata? can-write-metadata?)include-schemas? add-schemasdiff --git a/test/metabase/native_query_snippets/api_test.clj b/test/metabase/native_query_snippets/api_test.cljindex f4e968d76a0..902ab6dfe2c 100644--- a/test/metabase/native_query_snippets/api_test.clj+++ b/test/metabase/native_query_snippets/api_test.clj@@ -38,6 +38,28 @@(is (contains? snippets-from-api (select-keys snippet-1 test-snippet-fields)))(is (contains? snippets-from-api (select-keys snippet-2 test-snippet-fields)))))))))))+(deftest list-snippets-call-count-test+ (testing "GET /api/native-query-snippet app-DB call count should not scale with the number of snippets"+ (mt/with-model-cleanup [:model/NativeQuerySnippet]+ (letfn [(insert-snippets! [n]+ (dotimes [_ n]+ (t2/insert! :model/NativeQuerySnippet+ {:name (mt/random-name)+ :content "1 = 1"+ :creator_id (mt/user->id :crowberto)})))+ (warm-call-count! []+ ;; first request pays one-time priming; measure the second. Use a non-admin so the+ ;; per-snippet permission check does not short-circuit on superuser status.+ (mt/user-http-request :rasta :get 200 (snippet-url))+ (t2/with-call-count [call-count]+ (mt/user-http-request :rasta :get 200 (snippet-url))+ (call-count)))]+ (insert-snippets! 2)+ (let [calls-with-2 (warm-call-count!)+ _ (insert-snippets! 8)+ calls-with-10 (warm-call-count!)]+ (is (<= calls-with-10 (+ calls-with-2 3))))))))+(deftest read-snippet-api-test(mt/with-full-data-perms-for-all-users!(testing "GET /api/native-query-snippet/:id"diff --git a/test/metabase/permissions/models/data_permissions_test.clj b/test/metabase/permissions/models/data_permissions_test.cljindex 5380c1680ca..9592f939fc1 100644--- a/test/metabase/permissions/models/data_permissions_test.clj+++ b/test/metabase/permissions/models/data_permissions_test.clj@@ -8,6 +8,7 @@[metabase.permissions.models.data-permissions :as data-perms][metabase.permissions.models.permissions-group :as perms-group][metabase.test :as mt]+ [metabase.test.util.dynamic-redefs :as dynamic-redefs][toucan2.core :as t2])(:import(clojure.lang ExceptionInfo)))@@ -21,25 +22,67 @@:blocked [:perms/view-data #{:blocked}]nil [:perms/view-data #{}])))-(deftest ^:parallel rows->cache-entry-test- (testing "returns nil when there are no rows"- (is (nil? (#'data-perms/rows->cache-entry identity []))))- (testing "duplicate rows keep every distinct value visible to coalescing, and each table-level row keeps its own- schema_name (data_permissions has no unique constraint on (group_id, perm_type, db_id, table_id), and- schema_name is denormalized so rows for one table can disagree)"- (let [entry (#'data-perms/rows->cache-entry- identity- [{:group_id 1 :table_id nil :schema_name nil :perm_value :query-builder}- {:group_id 1 :table_id nil :schema_name nil :perm_value :no}- {:group_id 1 :table_id 10 :schema_name "public" :perm_value :query-builder}- {:group_id 1 :table_id 10 :schema_name "public" :perm_value :no}- {:group_id 2 :table_id 10 :schema_name "legacy" :perm_value :query-builder}])]- (is (= {1 #{:query-builder :no}}- (:groups entry)))- (is (= {10 {1 #{{:schema "public" :value :query-builder}- {:schema "public" :value :no}}- 2 #{{:schema "legacy" :value :query-builder}}}}- (:tables entry))))))+(deftest ^:parallel ranks->most-permissive-value-test+ (testing "reconstructs coalesce from a bucket's [min max] value ranks"+ ;; :perms/create-queries values: [:query-builder-and-native :query-builder :no]+ (are [expected pair] (= expected (#'data-perms/ranks->most-permissive-value :perms/create-queries pair))+ :query-builder-and-native [0 2]+ :query-builder [1 1]+ :no [2 2]+ nil nil))+ (testing ":perms/view-data's special rule: a :blocked row overrides :legacy-no-self-service but not :unrestricted"+ ;; :perms/view-data values: [:unrestricted :legacy-no-self-service :blocked]+ (are [expected pair] (= expected (#'data-perms/ranks->most-permissive-value :perms/view-data pair))+ :unrestricted [0 2]+ :blocked [1 2]+ :legacy-no-self-service [1 1]+ :blocked [2 2])))++(deftest ^:parallel combine-rank-pairs-test+ (is (= [0 2] (#'data-perms/combine-rank-pairs [1 2] [0 1])))+ (is (= [1 1] (#'data-perms/combine-rank-pairs nil [1 1])))+ (is (nil? (#'data-perms/combine-rank-pairs nil nil))))++(deftest permission-caches-load-on-demand-test+ (let [user-id 1+ loads (atom [])+ cache (atom {:db-ids #{} :perms {}})]+ (binding [api/*current-user-id* user-id+ data-perms/*database-permission-cache* cache]+ (mt/with-dynamic-fn-redefs [data-perms/load-database-permission-perms+ (fn [_user-id db-ids]+ (swap! loads conj db-ids)+ {:perms/view-data (zipmap db-ids (repeat :unrestricted))})]+ (testing "a per-function cache loads only databases not already fetched"+ (letfn [(check! [db-ids]+ (#'data-perms/cached-db-perms cache+ @#'data-perms/load-database-permission-perms+ user-id db-ids))]+ (check! [10])+ (check! [10 11])+ (check! [10 11])+ (is (= [[10] [11]] @loads))+ (is (= #{10 11} (:db-ids @cache)))+ (is (= :unrestricted+ (get-in @cache [:perms user-id :perms/view-data 10])))))))))++(deftest permission-caches-memoize-most-permissive-values-test+ (let [user-id 1+ loads (atom 0)+ cache (atom {})]+ (binding [api/*current-user-id* user-id+ data-perms/*most-permissive-permission-cache* cache]+ (mt/with-dynamic-fn-redefs [data-perms/load-most-permissive-db-perms+ (fn [_user-id perm-type]+ (swap! loads inc)+ {10 (data-perms/most-permissive-value perm-type)})]+ (testing "cross-database questions share one memoized load per permission type"+ (is (true? (data-perms/user-has-any-perms-of-type? user-id :perms/create-queries)))+ (is (= :query-builder-and-native+ (data-perms/most-permissive-database-permission-for-user user-id :perms/create-queries 10)))+ (is (= :no+ (data-perms/most-permissive-database-permission-for-user user-id :perms/create-queries 11)))+ (is (= 1 @loads)))))))(deftest ^:parallel at-least-as-permissive?-test(testing "at-least-as-permissive? correctly compares permission values"@@ -267,10 +310,15 @@(t2/with-call-count [call-count](is (= :yes (data-perms/database-permission-for-user user-id :perms/manage-database database-id-1)))(is (zero? (call-count)))))- ;; Fetching perms for a different DB is a cache miss- (t2/with-call-count [call-count]- (is (= :no (data-perms/database-permission-for-user user-id :perms/manage-database database-id-2)))- (is (= 1 (call-count)))))))))+ ;; Fetching perms for a different DB is a cache miss (spy on the load fn rather than counting queries so the+ ;; assertion doesn't depend on how many statements a load issues)+ (let [loads (atom 0)+ orig (dynamic-redefs/original-fn #'data-perms/load-database-permission-perms)]+ (mt/with-dynamic-fn-redefs [data-perms/load-database-permission-perms (fn [user-id db-ids]+ (swap! loads inc)+ (orig user-id db-ids))]+ (is (= :no (data-perms/database-permission-for-user user-id :perms/manage-database database-id-2)))+ (is (= 1 @loads)))))))))(deftest table-permission-for-user-test(mt/with-temp [:model/PermissionsGroup {group-id-1 :id} {}@@ -337,6 +385,37 @@(is (= :unrestricted(data-perms/table-permission-for-user user-id :perms/view-data database-id table-id-1))))))))+(deftest inactive-table-grants-ignored-test+ (testing "a grant that exists only on an inactive table does not count for value-aggregating checks"+ (mt/with-temp [:model/PermissionsGroup {group-id :id} {}+ :model/User {user-id :id} {}+ :model/PermissionsGroupMembership {} {:user_id user-id+ :group_id group-id}+ :model/Database {database-id :id} {}+ :model/Table {table-id :id} {:db_id database-id}]+ (mt/with-no-data-perms-for-all-users!+ (data-perms/set-table-permission! group-id table-id :perms/manage-table-metadata :yes)+ (letfn [(check! [has-any? most-permissive]+ (testing "\nreading straight from the DB"+ (is (= has-any?+ (data-perms/user-has-any-perms-of-type? user-id :perms/manage-table-metadata)))+ (is (= most-permissive+ (data-perms/most-permissive-database-permission-for-user user-id :perms/manage-table-metadata database-id))))+ (testing "\nreading from the request cache"+ (mt/with-current-user user-id+ (is (= has-any?+ (data-perms/user-has-any-perms-of-type? user-id :perms/manage-table-metadata)))+ (is (= most-permissive+ (data-perms/most-permissive-database-permission-for-user user-id :perms/manage-table-metadata database-id))))))]+ (testing "\na grant on an active table counts"+ (check! true :yes))+ (testing "\nthe same grant does not count once the table is deactivated"+ (t2/update! :model/Table table-id {:active false})+ (check! false :no))+ (testing "\nreactivating the table restores it"+ (t2/update! :model/Table table-id {:active true})+ (check! true :yes)))))))+(deftest permissions-for-user-test(mt/with-temp [:model/PermissionsGroup {group-id-1 :id} {}:model/PermissionsGroup {group-id-2 :id} {}@@ -496,21 +575,6 @@:db-id database-id-1:perm-type :perms/view-data)))))))-(deftest most-restrictive-per-group-works- (is (= #{:query-builder-and-native}- (#'data-perms/most-restrictive-per-group :perms/create-queries [{:group-id 1 :value :query-builder-and-native}])))- (is (= #{:no}- (#'data-perms/most-restrictive-per-group :perms/create-queries [{:group-id 1 :value :query-builder}- {:group-id 1 :value :no}])))- (is (= #{:no :query-builder-and-native}- (#'data-perms/most-restrictive-per-group :perms/create-queries [{:group-id 1 :value :query-builder-and-native}- {:group-id 1 :value :no}- {:group-id 2 :value :query-builder-and-native}])))- (is (= #{:no}- (#'data-perms/most-restrictive-per-group :perms/create-queries [{:group-id 1 :value :no}- {:group-id 1 :value :query-builder}- {:group-id 1 :value :query-builder-and-native}]))))-(deftest full-schema-permission-for-user-test(mt/with-temp [:model/PermissionsGroup {group-id-1 :id} {}:model/User {user-id-1 :id} {}diff --git a/test/metabase/util/memoize_test.cljc b/test/metabase/util/memoize_test.cljcindex 7dbf6044e1b..cdd06ecd76e 100644--- a/test/metabase/util/memoize_test.cljc+++ b/test/metabase/util/memoize_test.cljc@@ -70,7 +70,22 @@(testing "the zero-arg arity interns values as-is"(let [intern (memo/fast-interner)](is (= :kw (intern :kw)))- (is (identical? (intern :kw) (intern :kw))))))+ (is (identical? (intern :kw) (intern :kw)))))+ (testing "nil is returned unchanged without calling the canonicalizer"+ (let [calls (atom 0)+ intern (memo/fast-interner (fn [x]+ (swap! calls inc)+ x))]+ (is (nil? (intern nil)))+ (is (zero? @calls))))+ (testing "a cached false counts as a hit, not a miss"+ (let [calls (atom 0)+ intern (memo/fast-interner (fn [x]+ (swap! calls inc)+ x))]+ (is (false? (intern false)))+ (is (false? (intern false)))+ (is (= 1 @calls)))))(deftest ^:parallel bounded-evicts-when-keyspace-overflows-test(let [keyspace (gen-keyspace 100)diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.cljindex 780d3d2d58d..b767f95fcdd 100644--- a/test/metabase/warehouses_rest/api_test.clj+++ b/test/metabase/warehouses_rest/api_test.clj@@ -1242,6 +1242,38 @@(is (not-any? :is_saved_questions(:data (mt/user-http-request :lucky :get 200 "database?saved=true")))))))))+(deftest databases-list-saved-questions-call-count-test+ (testing "GET /api/database?saved=true&include=tables app-DB call count should not scale with the number of Cards"+ (mt/with-model-cleanup [:model/Card]+ (letfn [(insert-cards! [n]+ (dotimes [_ n]+ (t2/insert! :model/Card+ (assoc (card-with-native-query (mt/random-name))+ :creator_id (mt/user->id :crowberto)+ :display :table+ :visualization_settings {}+ ;; a column with a real Field :id exercises the per-card Field+ ;; fetch + hydration path+ :result_metadata [{:id (mt/id :venues :name)+ :name "NAME"+ :display_name "Name"+ :base_type :type/Text}]))))+ (warm-call-count! []+ ;; first request pays one-time priming; measure the second+ (mt/user-http-request :crowberto :get 200 "database?saved=true&include=tables")+ (t2/with-call-count [call-count]+ (mt/user-http-request :crowberto :get 200 "database?saved=true&include=tables")+ (call-count)))]+ (insert-cards! 2)+ (let [calls-with-2 (warm-call-count!)+ _ (insert-cards! 16)+ calls-with-18 (warm-call-count!)]+ ;; [[mi/do-after-select]] re-runs each Card row through toucan2's identity-query, which+ ;; `with-call-count` counts even though it never hits the DB — so growth of 1 per Card is+ ;; expected and allowed. The slack of 6 stays well under the pre-batching behavior of one+ ;; real `metabase_database` select per additional Card on top of that (#78919).+ (is (<= calls-with-18 (+ calls-with-2 16 6))))))))+(deftest fetch-databases-with-invalid-driver-test(testing "GET /api/database"(testing "\nEndpoint should still work even if there is a Database saved with a invalid driver"diff --git a/test/metabase/xrays/automagic_dashboards/core_test.clj b/test/metabase/xrays/automagic_dashboards/core_test.cljindex 2d2d2d24efd..fdee0f28ea8 100644--- a/test/metabase/xrays/automagic_dashboards/core_test.clj+++ b/test/metabase/xrays/automagic_dashboards/core_test.clj@@ -1103,7 +1103,7 @@(let [database (t2/select-one :model/Database :id db-id)](t2/with-call-count [call-count](magic/candidate-tables database)- (is (= 2 (call-count)))))))))+ (is (<= (call-count) 2))))))))(deftest ^:parallel empty-table-test(testing "candidate-tables should work with an empty Table (no Fields)"