Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix N+1 query patterns with permissions (#78919)

Alexander Polyankin70c3f15684d4
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index cdbf541fd4b..d5f5782f73b 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -146,6 +146,7 @@
config
events
models
+ permissions
util}
:model-imports #{:model/Card
:model/Collection
@@ -2519,6 +2520,7 @@
metabot
metrics
models
+ permissions
system
types
util}
diff --git a/enterprise/backend/src/metabase_enterprise/advanced_permissions/common.clj b/enterprise/backend/src/metabase_enterprise/advanced_permissions/common.clj
index a90b5e78155..3bfab6bec2a 100644
--- a/enterprise/backend/src/metabase_enterprise/advanced_permissions/common.clj
+++ b/enterprise/backend/src/metabase_enterprise/advanced_permissions/common.clj
@@ -106,15 +106,17 @@
(empty tables)
:else
- (filter
- (fn [{table-id :id db-id :db_id}]
- (perms/user-has-permission-for-table?
- api/*current-user-id*
- :perms/manage-table-metadata
- :yes
- db-id
- table-id))
- tables)))
+ (do
+ (perms/prime-db-perms-cache (into #{} (map :db_id) tables))
+ (filter
+ (fn [{table-id :id db-id :db_id}]
+ (perms/user-has-permission-for-table?
+ api/*current-user-id*
+ :perms/manage-table-metadata
+ :yes
+ db-id
+ table-id))
+ tables))))
(defn filter-schema-by-data-model-perms
"Given a list of schema, remove the ones for which `*current-user*` does not have data model editing permissions."
diff --git a/enterprise/backend/src/metabase_enterprise/data_studio/api/table.clj b/enterprise/backend/src/metabase_enterprise/data_studio/api/table.clj
index 5ee7159af4f..654139440d1 100644
--- a/enterprise/backend/src/metabase_enterprise/data_studio/api/table.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_studio/api/table.clj
@@ -52,7 +52,13 @@
(if (empty? tables)
#{}
(let [input-table-id (keyword (name input-field) "table_id")
- output-table-id (keyword (name output-field) "table_id")]
+ output-table-id (keyword (name output-field) "table_id")
+ not-in-tables (if (map? tables)
+ [:not [:exists (-> tables
+ (assoc :select [1])
+ (update :where (fn [where]
+ [:and where [:= :id output-table-id]])))]]
+ [:not [:in output-table-id tables]])]
(into #{} (map :table_id)
(t2/reducible-query {:select [[output-table-id :table_id]]
:from [[(t2/table-name :model/Dimension) :dim]]
@@ -63,7 +69,7 @@
:where [:and
[:= :dim.type "external"]
[:in input-table-id tables]
- [:not [:in output-table-id tables]]]})))))
+ not-in-tables]})))))
(defn- upstream-table-ids
"Given a table selector (set of IDs or subquery), find all tables that these tables depend on
diff --git a/enterprise/backend/src/metabase_enterprise/embedding_hub/api.clj b/enterprise/backend/src/metabase_enterprise/embedding_hub/api.clj
index 247ee0f9990..4148799e308 100644
--- a/enterprise/backend/src/metabase_enterprise/embedding_hub/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/embedding_hub/api.clj
@@ -48,9 +48,11 @@
[:or
[:and
[:!= :collection_id (:id (audit/default-audit-collection))]
- [:not-in :collection_id {:select :id
- :from [(t2/table-name :model/Collection)]
- :where [:= :is_sample true]}]]
+ [:not [:exists {:select [1]
+ :from [[(t2/table-name :model/Collection) :sample_coll]]
+ :where [:and
+ [:= :sample_coll.is_sample true]
+ [:= :sample_coll.id :report_card.collection_id]]}]]]
[:is :collection_id nil]]]}))
(defn- has-user-created-tenants? []
diff --git a/src/metabase/activity_feed/models/recent_views.clj b/src/metabase/activity_feed/models/recent_views.clj
index 998ff87e7c3..a5d95d2fb78 100644
--- a/src/metabase/activity_feed/models/recent_views.clj
+++ b/src/metabase/activity_feed/models/recent_views.clj
@@ -38,6 +38,7 @@
[metabase.collections.models.collection.root :as root]
[metabase.config.core :as config]
[metabase.models.interface :as mi]
+ [metabase.permissions.core :as perms]
[metabase.util :as u]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.log :as log]
@@ -438,28 +439,29 @@
(defn- table-recents
"Query to select recent table data"
[table-ids]
- (t2/select :model/Table
- {:select [:t.id :t.name :t.description
- :t.display_name :t.active :t.visibility_type :t.schema
- [:db.name :database-name]
- [:db.id :db_id]
- [:db.initial_sync_status :initial-sync-status]]
- :from [[:metabase_table :t]]
- :where (let [base-condition [:or
- [:= :visibility_type nil]
- [:!= :visibility_type "hidden"]]]
- (if (seq table-ids)
- [:and base-condition [:in :t.id table-ids]]
- base-condition))
- :left-join [[:metabase_database :db]
- [:= :db.id :t.db_id]]}))
+ (if-not (seq table-ids)
+ []
+ (t2/select :model/Table
+ {:select [:t.id :t.name :t.description
+ :t.display_name :t.active :t.visibility_type :t.schema
+ [:db.name :database-name]
+ [:db.id :db_id]
+ [:db.initial_sync_status :initial-sync-status]]
+ :from [[:metabase_table :t]]
+ :where [:and
+ [:or
+ [:= :visibility_type nil]
+ [:!= :visibility_type "hidden"]]
+ [:in :t.id table-ids]]
+ :left-join [[:metabase_database :db]
+ [:= :db.id :t.db_id]]})))
(defmethod fill-recent-view-info :table [{:keys [_model model_id timestamp model_object]}]
(let [table model_object]
(when (and (not= "hidden" (:visibility_type table))
(:database-name table)
(:active table)
- (mi/can-read? :model/Table model_id))
+ (mi/can-read? table))
{:id model_id
:name (:name table)
:description (:description table)
@@ -576,11 +578,13 @@
table-ids :table
document-ids :document} (as-> views views
(group-by (comp keyword :model) views)
- (update-vals views #(mapv :model_id %)))]
+ (update-vals views #(mapv :model_id %)))
+ tables (table-recents table-ids)]
+ (perms/prime-db-perms-cache (into #{} (keep :db_id) tables))
{:card (m/index-by :id (card-recents card-ids))
:dashboard (m/index-by :id (dashboard-recents dashboard-ids))
:collection (m/index-by :id (collection-recents collection-ids))
- :table (m/index-by :id (table-recents table-ids))
+ :table (m/index-by :id tables)
:document (m/index-by :id (document-recents document-ids))}))
(def ^:private ItemValidator (mr/validator Item))
diff --git a/src/metabase/data_studio/api/table.clj b/src/metabase/data_studio/api/table.clj
index dd9dd251b47..8f6269a233a 100644
--- a/src/metabase/data_studio/api/table.clj
+++ b/src/metabase/data_studio/api/table.clj
@@ -55,7 +55,13 @@
(if (empty? tables)
#{}
(let [input-table-id (keyword (name input-field) "table_id")
- output-table-id (keyword (name output-field) "table_id")]
+ output-table-id (keyword (name output-field) "table_id")
+ not-in-tables (if (map? tables)
+ [:not [:exists (-> tables
+ (assoc :select [1])
+ (update :where (fn [where]
+ [:and where [:= :id output-table-id]])))]]
+ [:not [:in output-table-id tables]])]
(into #{} (map :table_id)
(t2/reducible-query {:select [[output-table-id :table_id]]
:from [[(t2/table-name :model/Dimension) :dim]]
@@ -66,7 +72,7 @@
:where [:and
[:= :dim.type "external"]
[:in input-table-id tables]
- [:not [:in output-table-id tables]]]})))))
+ not-in-tables]})))))
(defn- upstream-table-ids
"Given a table selector (set of IDs or subquery), find all tables that these tables depend on
diff --git a/src/metabase/measures/api.clj b/src/metabase/measures/api.clj
index ab77edea3f8..6ebae0ddb43 100644
--- a/src/metabase/measures/api.clj
+++ b/src/metabase/measures/api.clj
@@ -8,6 +8,7 @@
[metabase.lib.core :as lib]
[metabase.metrics.core :as metrics]
[metabase.models.interface :as mi]
+ [metabase.permissions.core :as perms]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
[metabase.util.malli :as mu]
@@ -103,9 +104,12 @@
(api.macros/defendpoint :get "/" :- [:sequential ::measure]
"Fetch *all* `Measures`."
[]
- (as-> (t2/select :model/Measure, :archived false, {:order-by [[:%lower.name :asc]]}) measures
- (filter mi/can-read? measures)
- (t2/hydrate measures :creator :definition_description)))
+ (let [measures (t2/select :model/Measure, :archived false, {:order-by [[:%lower.name :asc]]})
+ table-ids (into #{} (keep :table_id) measures)]
+ (when (seq table-ids)
+ (perms/prime-db-perms-cache (t2/select-fn-set :db_id :model/Table :id [:in table-ids])))
+ (-> (filterv mi/can-read? measures)
+ (t2/hydrate :creator :definition_description))))
(defn- write-check-and-update-measure!
"Check whether current user has write permissions, then update Measure with values in `body`. Publishes appropriate
diff --git a/src/metabase/permissions/core.clj b/src/metabase/permissions/core.clj
index 32b47d03fbb..a2f3f1bf784 100644
--- a/src/metabase/permissions/core.clj
+++ b/src/metabase/permissions/core.clj
@@ -51,7 +51,7 @@
most-permissive-database-permission-for-user
native-download-permission-for-user
permissions-for-user
- prime-db-cache
+ prime-db-perms-cache
sandboxes-for-user
schema-permission-for-user
set-database-permission!
diff --git a/src/metabase/permissions/models/data_permissions.clj b/src/metabase/permissions/models/data_permissions.clj
index a3cd3d89f5d..69f0b08184b 100644
--- a/src/metabase/permissions/models/data_permissions.clj
+++ b/src/metabase/permissions/models/data_permissions.clj
@@ -16,10 +16,8 @@
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
- [metabase.util.memoize :as u.memo]
[methodical.core :as methodical]
- [toucan2.core :as t2]
- [toucan2.realize :as t2.realize])
+ [toucan2.core :as t2])
(:import
(clojure.lang PersistentVector)))
@@ -144,210 +142,6 @@
[perm-type perm-value]
(validate-perm-value! perm-type perm-value))
-;;; ---------------------------------------- Caching ------------------------------------------------------------------
-
-(defn- relevant-permissions-for-user-and-dbs
- "Returns a reducible of all relevant permission rows for the user, excluding permissions for deactivated tables, for
- the given sequence of database ids. Reducing it folds rows one at a time instead of realizing the full result set
- (see [[prime-db-cache]]). Rows are raw realized maps, not model instances, so `:perm_type` and `:perm_value` are
- strings rather than keywords. The per-row [[t2.realize/realize]] is a performance requirement, not a convenience:
- key access on unrealized result-set rows goes through toucan2's deferred-row machinery on every lookup, which
- benchmarked ~15x slower than realizing each row once and reading plain map keys."
- [user-id db-ids]
- (eduction
- (map t2.realize/realize)
- (t2/reducible-query
- {:select [:p.group_id :p.perm_type :p.db_id :p.table_id :p.schema_name :p.perm_value]
- :from [[(t2/table-name :model/PermissionsGroupMembership) :pgm]]
- :join [[(t2/table-name :model/PermissionsGroup) :pg] [:= :pg.id :pgm.group_id]
- [(t2/table-name :model/DataPermissions) :p] [:= :p.group_id :pg.id]]
- :left-join [[(t2/table-name :model/Table) :mt] [:= :mt.id :p.table_id]]
- :where [:and
- [:= :pgm.user_id user-id]
- [:in :p.db_id db-ids]
- [:or
- [:= :p.table_id nil]
- [:= :mt.active true]]]})))
-
-(defn- relevant-permissions-for-user-perm-and-db
- "Returns all relevant rows for a given user, permission type, and db_id, excluding permissions for deactivated
- tables. Rows are raw realized maps rather than model instances -- much cheaper per row, see
- [[relevant-permissions-for-user-and-dbs]] -- so `:perm_value` is a string; [[rows->cache-entry]] normalizes it."
- [user-id perm-type db-id]
- (t2/query {:select [:p.group_id :p.perm_type :p.db_id :p.table_id :p.schema_name :p.perm_value]
- :from [[(t2/table-name :model/PermissionsGroupMembership) :pgm]]
- :join [[(t2/table-name :model/PermissionsGroup) :pg] [:= :pg.id :pgm.group_id]
- [(t2/table-name :model/DataPermissions) :p] [:= :p.group_id :pg.id]]
- :left-join [[(t2/table-name :model/Table) :mt] [:= :mt.id :p.table_id]]
- :where [:and
- [:= :pgm.user_id user-id]
- [:= :p.perm_type (u/qualified-name perm-type)]
- [:= :p.db_id db-id]
- [:or
- [:= :p.table_id nil]
- [:= :mt.active true]]]}))
-
-(def ^:dynamic *permissions-for-user*
- "A dynamically-bound atom containing a cache of data permissions that have been fetched so far for the current user.
- Keys are:
- - :db-ids -> A set of the IDs of databases which have already been fetched.
- - :intern -> The interner used to dedupe entry values; created on first prime so sharing spans every prime
- in the request.
- - :perms -> A map of permissions with the structure `{user-id {perm-type {db-id entry}}}` so that we NEVER
- accidentally use the cache of the wrong user. Each `entry` is a compact map
-
- {:groups {group-id #{perm-value}} ; db-level rows
- :tables {table-id {group-id #{{:schema schema-name ; table-level rows
- :value perm-value}}}}}
-
- The sets almost always hold exactly one value; see [[rows->cache-entry]] for why every distinct
- value of duplicate rows is preserved rather than one row winning.
-
- On instances with many databases these maps are typically identical across most of them, so
- they are interned at load time: equal values share one instance. This keeps the cache size
- proportional to the number of *distinct* permission profiles rather than `groups * databases`
- (see #76077 for how large the raw row count can get). Concretely, on an instance where the
- user's groups have the same db-level grants almost everywhere plus one database with
- table-level rows:
-
- {1
- {:perms/view-data
- {10 {:groups {1 #{:unrestricted}, 2 #{:blocked}} :tables {}} ; ─┐ interning makes
- 11 {:groups {1 #{:unrestricted}, 2 #{:blocked}} :tables {}} ; ├ these :groups maps
- 12 {:groups {1 #{:unrestricted}, 2 #{:blocked}} :tables {}} ; ─┘ identical
- 13 {:groups {1 #{:unrestricted}}
- :tables
- {100 {2 #{{:schema \"public\" :value :blocked}}} ; ─┬ identical, and their
- 101 {2 #{{:schema \"public\" :value :blocked}}} ; ─┘ inner :schema/:value
- 102 {2 #{{:schema \"reporting\" :value :blocked}}}}}}}} ; maps also identical
-
- Each annotated group prints as N equal maps but is held as ONE shared instance plus N
- pointers — 1,000 databases with the same profile cost one :groups map, not 1,000. The sharing
- is established at construction by [[rows->cache-entry]] and is invisible to readers.
-
- When checking permissions, if a DB has not been fetched, it will be added to the cache before the check returns."
- (atom {:db-ids #{} :perms {}}))
-
-(defn- cache-interner
- "A nil-tolerant [[u.memo/fast-interner]]."
- []
- (let [intern! (u.memo/fast-interner)]
- (fn [v]
- (when (some? v)
- (intern! v)))))
-
-(defn- add-row-to-cache-entry
- "Fold a single data_permissions row into an in-progress cache entry (see [[rows->cache-entry]] for the semantics).
- `perm_value` may be a raw result-set string or an already-transformed keyword depending on the fetch path;
- `keyword` normalizes both."
- [interner entry {:keys [table_id schema_name group_id perm_value]}]
- (let [perm-value (keyword perm_value)]
- (if table_id
- (update-in entry [:tables table_id group_id] (fnil conj #{})
- (interner {:schema schema_name :value perm-value}))
- (update-in entry [:groups group_id] (fnil conj #{}) perm-value))))
-
-(defn- finalize-cache-entry
- "Intern the shareable pieces of a folded cache entry: the whole :groups map, and each table's group-id->values map."
- [interner {:keys [groups tables] :or {groups {} tables {}}}]
- {:groups (interner groups)
- :tables (update-vals tables interner)})
-
-(defn- rows->cache-entry
- "Build the compact cache entry (see [[*permissions-for-user*]]) for one (perm-type, db-id) bucket of
- data_permissions rows. Returns nil when there are no rows.
-
- The `interner` calls are a semantic no-op: they never change what any lookup returns, only object identity,
- collapsing =-equal values to one shared instance so that the same profile repeated across many databases (or the
- same :schema/:value map across many tables) is stored once — see the example on [[*permissions-for-user*]].
-
- Values accumulate into sets rather than a single value per key: data_permissions has no unique constraint on
- (group_id, perm_type, db_id, table_id), and its delete-then-insert write paths take different cluster locks, so
- racing writers can — and in production data do — leave multiple rows for the same logical key. The raw rows fed
- every duplicate's value into the consumers' coalescing logic, which resolved conflicts deterministically; a
- single-value map would instead keep whichever row the unordered SELECT returned last, making permission results
- flip with JDBC row order (e.g. duplicate download-results rows {:no, :one-million-rows} granting downloads that
- coalescing would deny)."
- [interner rows]
- (when (seq rows)
- (finalize-cache-entry interner (reduce (partial add-row-to-cache-entry interner) {} rows))))
-
-(defn prime-db-cache
- "Prime the permissions cache for a given user and database IDs.
- This can be called directly prior to checking the permissions for a large number of databases to improve performance"
- [db-ids]
- (let [{cached-db-ids :db-ids interner :intern perms :perms} @*permissions-for-user*
- filtered-ids (remove cached-db-ids db-ids)]
- (when (seq filtered-ids)
- (let [user-id api/*current-user-id*
- interner (or interner (cache-interner))
- folded (reduce (fn [m {:keys [perm_type db_id] :as row}]
- (update-in m [(keyword perm_type) db_id]
- (fnil #(add-row-to-cache-entry interner % row) {})))
- {}
- (relevant-permissions-for-user-and-dbs user-id filtered-ids))
- new-by-type (update-vals folded
- (fn [db-id->entry]
- (update-vals db-id->entry #(finalize-cache-entry interner %))))]
- (reset! *permissions-for-user*
- {:db-ids (into cached-db-ids db-ids)
- :intern interner
- :perms (update perms user-id #(merge-with merge % new-by-type))})))))
-
-(defenterprise enforced-sandboxes-for-user
- "Given a user-id, returns the set of sandboxes that should be enforced for the provided user ID. This result is
- cached for the duration of a request. Empty on OSS instances."
- metabase-enterprise.sandbox.api.util
- [_user-id]
- #{})
-
-(def ^:dynamic *sandboxes-for-user*
- "Filled by `enforced-sandboxes-for-user`. Empty on OSS instances, or EE instances without the `sandboxes` feature."
- (delay nil))
-
-(defn sandboxes-for-user
- "Derefs the *sandboxes-for-user* dynamic variable so it can be contained to this namespace"
- []
- @*sandboxes-for-user*)
-
-(defmacro with-relevant-permissions-for-user
- "Populates the `*permissions-for-user*` and `*sandboxes-for-user*` dynamic vars for use by the cache-aware functions
- in this namespace."
- [user-id & body]
- `(binding [*permissions-for-user* (atom {:db-ids #{} :perms {}})
- *sandboxes-for-user* (delay (enforced-sandboxes-for-user ~user-id))]
- ~@body))
-
-(def ^:dynamic *use-perms-cache?*
- "Bind to `false` to intentionally bypass the permissions cache and fetch data straight from the DB."
- true)
-
-(defmacro disable-perms-cache
- "Intentionally bypass the permissions cache and fetch data straight from the DB."
- {:style/indent 0}
- [& body]
- `(binding [*use-perms-cache?* false]
- ~@body))
-
-(defn- use-cache?
- [user-id]
- (and *use-perms-cache?*
- (= user-id api/*current-user-id*)))
-
-(defn- get-permissions
- "Returns the compact cache entry (see [[*permissions-for-user*]]) for the given user, perm type and database, or
- nil when the user has no permissions rows for it."
- [user-id perm-type db-id]
- (if (use-cache? user-id)
- (do ; Use the cache if we can
- (prime-db-cache [db-id])
- (get-in (:perms @*permissions-for-user*) [user-id perm-type db-id]))
- ;; If we're checking permissions for a *different* user than ourselves, fetch it straight from the DB. The
- ;; entry is read once and discarded, so there is nothing to gain from interning.
- (rows->cache-entry identity (relevant-permissions-for-user-perm-and-db user-id perm-type db-id))))
-
-;;; ---------------------------------------- Fetching a user's permissions --------------------------------------------
-
(defmulti coalesce
"Coalesce a set of permission values into a single value. This is used to determine the permission to enforce for a
user in multiple groups with conflicting permissions. By default, this returns the *most* permissive value that the
@@ -382,6 +176,110 @@
(let [ordered-values (-> permissions.schema/data-permissions perm-type :values reverse)]
(first (filter (set perm-values) ordered-values))))
+;;; ---------------------------------------- Caching ------------------------------------------------------------------
+
+;;; Every permission-checking function below has its own request-scoped cache — one atom, one hand-written aggregate
+;;; query, one cache structure that fits exactly the question the function answers. All the atoms are bound by
+;;; [[with-relevant-permissions-for-user]] and key `:perms` by user ID so we NEVER accidentally use the cache of the
+;;; wrong user.
+;;;
+;;; All coalescing happens in SQL: each permission value is mapped to its rank (see [[value-rank-case]]) and rows
+;;; are aggregated with GROUP BY, so result sets scale with tables/schemas/databases rather than with raw
+;;; data_permissions row counts (dbs x groups x tables x duplicate values). A bucket's (min, max) rank pair is
+;;; enough to reconstruct [[coalesce]] exactly (see [[ranks->most-permissive-value]]), and the
+;;; most-restrictive-within-a-group-then-most-permissive-across-groups questions nest two levels of GROUP BY, so
+;;; even group identity collapses.
+;;;
+;;; All caches are point-in-time snapshots for the request: nothing invalidates them when data_permissions is
+;;; written, so code that mutates permissions and re-checks them within the same request scope reads pre-write
+;;; answers, and the caches may observe a concurrent write at different first-load moments.
+
+(defn- perm-rows-query-base
+ "The FROM/JOIN/WHERE shared by every cache load: one user's groups' rows for every permission type, excluding rows
+ for deactivated tables. All caches must select from this same row set so they can never answer the same permission
+ question differently — change the row set here, not in one of the queries. `db-ids` of nil means every database."
+ [user-id db-ids]
+ {:from [[(t2/table-name :model/PermissionsGroupMembership) :pgm]]
+ :join [[(t2/table-name :model/PermissionsGroup) :pg] [:= :pg.id :pgm.group_id]
+ [(t2/table-name :model/DataPermissions) :p] [:= :p.group_id :pg.id]]
+ :left-join [[(t2/table-name :model/Table) :mt] [:= :mt.id :p.table_id]]
+ :where [:and
+ [:= :pgm.user_id user-id]
+ (when (seq db-ids)
+ [:in :p.db_id db-ids])
+ [:or
+ [:= :p.table_id nil]
+ [:= :mt.active true]]]})
+
+(def ^:private value-rank-case
+ "A HoneySQL CASE expression mapping a data_permissions row's (perm_type, perm_value) to the value's rank in its
+ permission type's ordering — 0 = most permissive. MIN/MAX aggregates of this rank are what let SQL collapse
+ groups, tables and duplicate values."
+ (into [:case]
+ cat
+ (for [[perm-type {:keys [values]}] permissions.schema/data-permissions
+ [i v] (map-indexed vector values)]
+ [[:and
+ [:= :p.perm_type (u/qualified-name perm-type)]
+ [:= :p.perm_value (u/qualified-name v)]]
+ [:inline i]])))
+
+(defn- ranks->most-permissive-value
+ "Reconstructs [[coalesce]] over a bucket of values from the bucket's `[min-rank max-rank]` pair. Verified
+ exhaustively against [[coalesce]] for every permission type and value subset: the most permissive value is the one
+ at the minimum rank, except for :perms/view-data's special rule where a :blocked row overrides
+ :legacy-no-self-service (but not :unrestricted). Returns nil for a nil pair (no rows), matching what coalescing an
+ empty value set returns for callers that fall back to the least permissive value."
+ [perm-type pair]
+ (when-let [[mn mx] pair]
+ (let [^PersistentVector values (-> permissions.schema/data-permissions perm-type :values)]
+ (if (and (= perm-type :perms/view-data)
+ (= (nth values mx) :blocked)
+ (not= (nth values mn) :unrestricted))
+ :blocked
+ (nth values mn)))))
+
+(defn- combine-rank-pairs
+ "Merges `[min max]` rank pairs — equivalent to coalescing the union of the buckets they summarize."
+ [& pairs]
+ (when-let [pairs (seq (remove nil? pairs))]
+ [(apply min (map first pairs))
+ (apply max (map second pairs))]))
+
+(def ^:dynamic *use-perms-cache?*
+ "Bind to `false` to intentionally bypass the permissions caches and fetch data straight from the DB."
+ true)
+
+(defmacro disable-perms-cache
+ "Intentionally bypass the permissions caches and fetch data straight from the DB."
+ {:style/indent 0}
+ [& body]
+ `(binding [*use-perms-cache?* false]
+ ~@body))
+
+(defn- use-cache?
+ [user-id]
+ (and *use-perms-cache?*
+ (= user-id api/*current-user-id*)))
+
+(defn- cached-db-perms
+ "The shared read-through pattern for the per-database caches below: returns the per-user `{perm-type {db-id v}}`
+ map from `cache-atom` (shaped `{:db-ids #{} :perms {user-id ...}}`), loading missing `db-ids` with
+ `(load-fn user-id db-ids)` first. Loads straight through without caching when the request cache doesn't apply."
+ [cache-atom load-fn user-id db-ids]
+ (if (use-cache? user-id)
+ (do
+ (let [missing-db-ids (into [] (remove (:db-ids @cache-atom)) db-ids)]
+ (when (seq missing-db-ids)
+ (let [loaded (load-fn user-id missing-db-ids)]
+ (swap! cache-atom
+ (fn [cache]
+ (-> cache
+ (update :db-ids into missing-db-ids)
+ (update-in [:perms user-id] #(merge-with merge % loaded))))))))
+ (get-in @cache-atom [:perms user-id]))
+ (load-fn user-id db-ids)))
+
(defn is-superuser?
"Returns true if the given user ID is a superuser. Avoids a DB query when checking the current user."
[user-id]
@@ -396,26 +294,28 @@
api/*is-data-analyst?*
(t2/select-one-fn :is_data_analyst :model/User :id user-id)))
-(mu/defn database-permission-for-user :- ::permissions.schema/data-permission-value
- "Returns the effective permission value for a given user, permission type, and database ID. If the user has
- multiple permissions for the given type in different groups, they are coalesced into a single value."
- [user-id perm-type database-id]
- (when (not= :model/Database (model-by-perm-type perm-type))
- (throw (ex-info (tru "Permission type {0} is a table-level permission." perm-type)
- {perm-type (permissions.schema/data-permissions perm-type)})))
- (if (is-superuser? user-id)
- (most-permissive-value perm-type)
- (let [perm-values (into #{} cat (vals (:groups (get-permissions user-id perm-type database-id))))]
- (or (coalesce perm-type perm-values)
- (least-permissive-value perm-type)))))
+;;; ---------------------------------- table-permission-for-user ------------------------------------------------------
-(mu/defn user-has-permission-for-database? :- :boolean
- "Returns a Boolean indicating whether the user has the specified permission value for the given database ID and table ID,
- or a more permissive value."
- [user-id perm-type perm-value database-id]
- (at-least-as-permissive? perm-type
- (database-permission-for-user user-id perm-type database-id)
- perm-value))
+(def ^:dynamic *table-permission-cache*
+ "Request cache for [[table-permission-for-user]]: `{:db-ids #{} :perms {user-id {perm-type {db-id entry}}}}` where
+ each entry is `{:db-value v, :tables {table-id v}}` — one coalesced value per table with table-granular rows, plus
+ the coalesced db-level value. Loaded per database, all permission types at once, one row per (type, db, table)."
+ (atom {:db-ids #{} :perms {}}))
+
+(defn- load-table-permission-perms
+ [user-id db-ids]
+ (reduce (fn [m {:keys [perm_type db_id table_id mn mx]}]
+ (let [perm-type (keyword perm_type)
+ value (ranks->most-permissive-value perm-type [mn mx])]
+ (if table_id
+ (assoc-in m [perm-type db_id :tables table_id] value)
+ (assoc-in m [perm-type db_id :db-value] value))))
+ {}
+ (t2/query (assoc (perm-rows-query-base user-id db-ids)
+ :select [:p.perm_type :p.db_id :p.table_id
+ [[:min value-rank-case] :mn]
+ [[:max value-rank-case] :mx]]
+ :group-by [:p.perm_type :p.db_id :p.table_id]))))
(def ^:dynamic *additional-table-permissions*
"See the `with-additional-table-permission` macro below."
@@ -438,36 +338,6 @@
(defn- get-additional-table-permission! [{:keys [db-id table-id]} perm-type]
(get-in *additional-table-permissions* [db-id table-id perm-type]))
-(mu/defn table-permission-for-groups :- ::permissions.schema/data-permission-value
- "Returns the effective permission value provided by a set of *group-ids*, for a provided permission type, database
- ID, and table ID."
- [group-ids perm-type database-id table-id]
- (when (not= :model/Table (model-by-perm-type perm-type))
- (throw (ex-info (tru "Permission type {0} is not a table-level permission." perm-type)
- {perm-type (permissions.schema/data-permissions perm-type)})))
- (let [perm-values (t2/select-fn-set :value
- :model/DataPermissions
- {:select [[:p.perm_value :value]]
- :from [[:data_permissions :p]]
- :where [:and
- [:in :p.group_id group-ids]
- [:= :p.perm_type (u/qualified-name perm-type)]
- [:= :p.db_id database-id]
- [:or
- [:= :table_id table-id]
- [:= :table_id nil]]]})]
- (or (coalesce perm-type (conj perm-values (get-additional-table-permission! {:db-id database-id :table-id table-id}
- perm-type)))
- (least-permissive-value perm-type))))
-
-(mu/defn groups-have-permission-for-table? :- :boolean
- "Returns a Boolean indicating whether the provided groups grant the specified permission level or higher for the given
- table ID, or a more permissive value. (i.e. if a user is in all of these groups, would they have this permission?)"
- [group-ids perm-type perm-value database-id table-id]
- (at-least-as-permissive? perm-type
- (table-permission-for-groups group-ids perm-type database-id table-id)
- perm-value))
-
(mu/defn table-permission-for-user :- ::permissions.schema/data-permission-value
"Returns the effective permission value for a given user, permission type, and database ID, and table ID. If the user
has multiple permissions for the given type in different groups, they are coalesced into a single value."
@@ -484,11 +354,16 @@
(most-permissive-value perm-type)
:else
- (let [{:keys [groups tables]} (get-permissions user-id perm-type database-id)
- perm-values (-> (into #{} cat (vals groups))
- (into (comp cat (map :value)) (vals (get tables table-id))))
- table-perm (coalesce perm-type (conj perm-values (get-additional-table-permission! {:db-id database-id :table-id table-id}
- perm-type)))]
+ (let [entry (get-in (cached-db-perms *table-permission-cache* load-table-permission-perms
+ user-id [database-id])
+ [perm-type database-id])
+ table-perm (coalesce perm-type
+ (into #{}
+ (remove nil?)
+ [(:db-value entry)
+ (get (:tables entry) table-id)
+ (get-additional-table-permission! {:db-id database-id :table-id table-id}
+ perm-type)]))]
(or (when-not (= table-perm (least-permissive-value perm-type))
table-perm)
(when (pos-int? table-id)
@@ -503,36 +378,92 @@
(table-permission-for-user user-id perm-type database-id table-id)
perm-value))
-(defn- most-restrictive-per-group
- "Given a perm-type and a collection of maps that look like `{:group-id 1 :value :permission-value}`, returns a set
- containing the most restrictive permission value in each group."
- [perm-type perms]
- (->> perms
- (group-by :group-id)
- (m/map-vals (fn [ps]
- (->> ps (map :value) set (coalesce-most-restrictive perm-type))))
- vals
- set))
-
-(defn- entry-group-value-pairs
- "Returns `{:group-id g :value v}` maps for every db-level row in a cache entry, plus every table-level row whose
- schema-name satisfies `schema-match?`."
- [{:keys [groups tables]} schema-match?]
- (into (reduce-kv (fn [acc group-id values]
- (reduce #(conj %1 {:group-id group-id :value %2}) acc values))
- []
- groups)
- (for [[_table-id group-id->rows] tables
- [group-id rows] group-id->rows
- {:keys [schema value]} rows
- :when (schema-match? schema)]
- {:group-id group-id :value value})))
+;;; -------------------------------- database-permission-for-user -----------------------------------------------------
-(mu/defn full-schema-permission-for-user :- ::permissions.schema/data-permission-value
+(def ^:dynamic *database-permission-cache*
+ "Request cache for [[database-permission-for-user]]: `{:db-ids #{} :perms {user-id {perm-type {db-id value}}}}` —
+ the coalesced db-level value. Loaded per database, all permission types at once, one row per (type, db)."
+ (atom {:db-ids #{} :perms {}}))
+
+(defn- load-database-permission-perms
+ [user-id db-ids]
+ (reduce (fn [m {:keys [perm_type db_id mn mx]}]
+ (let [perm-type (keyword perm_type)]
+ (assoc-in m [perm-type db_id] (ranks->most-permissive-value perm-type [mn mx]))))
+ {}
+ (t2/query (-> (perm-rows-query-base user-id db-ids)
+ (assoc :select [:p.perm_type :p.db_id
+ [[:min value-rank-case] :mn]
+ [[:max value-rank-case] :mx]]
+ :group-by [:p.perm_type :p.db_id])
+ (update :where conj [:= :p.table_id nil])))))
+
+(mu/defn database-permission-for-user :- ::permissions.schema/data-permission-value
+ "Returns the effective permission value for a given user, permission type, and database ID. If the user has
+ multiple permissions for the given type in different groups, they are coalesced into a single value."
+ [user-id perm-type database-id]
+ (when (not= :model/Database (model-by-perm-type perm-type))
+ (throw (ex-info (tru "Permission type {0} is a table-level permission." perm-type)
+ {perm-type (permissions.schema/data-permissions perm-type)})))
+ (if (is-superuser? user-id)
+ (most-permissive-value perm-type)
+ (or (get-in (cached-db-perms *database-permission-cache* load-database-permission-perms
+ user-id [database-id])
+ [perm-type database-id])
+ (least-permissive-value perm-type))))
+
+(mu/defn user-has-permission-for-database? :- :boolean
+ "Returns a Boolean indicating whether the user has the specified permission value for the given database ID and table ID,
+ or a more permissive value."
+ [user-id perm-type perm-value database-id]
+ (at-least-as-permissive? perm-type
+ (database-permission-for-user user-id perm-type database-id)
+ perm-value))
+
+;;; --------------------------------- schema-permission-for-user ------------------------------------------------------
+
+(def ^:dynamic *schema-permission-cache*
+ "Request cache for [[schema-permission-for-user]]: `{:db-ids #{} :perms {user-id {perm-type {db-id entry}}}}`
+ where each entry is `{:default v, :schemas {schema v}}` — per schema, the coalesced value of the schema's table
+ rows combined with the db-level rows; `:default` (the db-level value alone) answers schemas with no rows of their
+ own. Schema names are normalized (nil = \"\"). Loaded per database, all permission types at once."
+ (atom {:db-ids #{} :perms {}}))
+
+(defn- load-schema-permission-perms
+ [user-id db-ids]
+ (let [table-level-case [:case [:= :p.table_id nil] [:inline 0] :else [:inline 1]]
+ folded (reduce (fn [m {:keys [perm_type db_id schema_name table_level mn mx]}]
+ (if (pos? table_level)
+ (update-in m [(keyword perm_type) db_id :schemas (or schema_name "")]
+ combine-rank-pairs [mn mx])
+ (assoc-in m [(keyword perm_type) db_id :db-level] [mn mx])))
+ {}
+ (t2/query (assoc (perm-rows-query-base user-id db-ids)
+ :select [:p.perm_type :p.db_id :p.schema_name
+ [table-level-case :table_level]
+ [[:min value-rank-case] :mn]
+ [[:max value-rank-case] :mx]]
+ :group-by [:p.perm_type :p.db_id :p.schema_name table-level-case])))]
+ (into {}
+ (map (fn [[perm-type db-id->folded]]
+ [perm-type
+ (update-vals db-id->folded
+ (fn [{:keys [db-level schemas]}]
+ {:default (ranks->most-permissive-value perm-type db-level)
+ :schemas (update-vals schemas
+ #(ranks->most-permissive-value
+ perm-type
+ (combine-rank-pairs db-level %)))}))]))
+ folded)))
+
+(mu/defn schema-permission-for-user :- ::permissions.schema/data-permission-value
"Returns the effective *schema-level* permission value for a given user, permission type, and database ID, and
schema name. If the user has multiple permissions for the given type in different groups, they are coalesced into a
- single value. The schema-level permission is the *most* restrictive table-level permission within that schema."
- [user-id perm-type database-id schema-name]
+ single value. The schema-level permission is the *least* restrictive table-level permission within that schema.
+
+ Schema names are compared with nil and the empty string treated as equivalent, matching how `database-schemas`
+ presents these databases to the API."
+ [user-id perm-type database-id schema-name :- [:maybe :string]]
(when (not= :model/Table (model-by-perm-type perm-type))
(throw (ex-info (tru "Permission type {0} is not a table-level permission." perm-type)
{perm-type (permissions.schema/data-permissions perm-type)})))
@@ -545,16 +476,44 @@
(most-permissive-value perm-type)
:else
- ;; The schema-level permission is the most-restrictive table-level permission within a schema. So for each group,
- ;; select the most-restrictive table-level permission. Then use normal coalesce logic to select the *least*
- ;; restrictive group permission.
- (let [perm-values (most-restrictive-per-group
- perm-type
- (entry-group-value-pairs (get-permissions user-id perm-type database-id)
- #(= % schema-name)))]
- (or (coalesce perm-type perm-values)
+ (let [{:keys [default schemas]} (get-in (cached-db-perms *schema-permission-cache* load-schema-permission-perms
+ user-id [database-id])
+ [perm-type database-id])]
+ (or (get schemas (or schema-name ""))
+ default
(least-permissive-value perm-type)))))
+(mu/defn user-has-permission-for-schema? :- :boolean
+ "Returns a Boolean indicating whether the user has the specified permission value for the given database ID and schema,
+ or a more permissive value."
+ [user-id perm-type perm-value database-id schema]
+ (at-least-as-permissive? perm-type
+ (schema-permission-for-user user-id perm-type database-id schema)
+ perm-value))
+
+;;; --------------------------------- full-db-permission-for-user -----------------------------------------------------
+
+(def ^:dynamic *full-db-permission-cache*
+ "Request cache for [[full-db-permission-for-user]]: `{:db-ids #{} :perms {user-id {perm-type {db-id value}}}}` —
+ per database, each group's most restrictive value across the whole database, coalesced most-permissively across
+ groups. Group identity collapses inside the nested query. Loaded per database, all permission types at once."
+ (atom {:db-ids #{} :perms {}}))
+
+(defn- load-full-db-permission-perms
+ [user-id db-ids]
+ (reduce (fn [m {:keys [perm_type db_id mn mx]}]
+ (let [perm-type (keyword perm_type)]
+ (assoc-in m [perm-type db_id] (ranks->most-permissive-value perm-type [mn mx]))))
+ {}
+ (t2/query {:select [:i.perm_type :i.db_id
+ [[:min :i.gmax] :mn] [[:max :i.gmax] :mx]]
+ :from [[(assoc (perm-rows-query-base user-id db-ids)
+ :select [:p.perm_type :p.db_id :p.group_id
+ [[:max value-rank-case] :gmax]]
+ :group-by [:p.perm_type :p.db_id :p.group_id])
+ :i]]
+ :group-by [:i.perm_type :i.db_id]})))
+
(mu/defn full-db-permission-for-user :- ::permissions.schema/data-permission-value
"Returns the effective *db-level* permission value for a given user, permission type, and database ID. If the user
has multiple permissions for the given type in different groups, they are coalesced into a single value. The
@@ -572,24 +531,78 @@
(most-permissive-value perm-type)
:else
- ;; The DB-level permission is the most-restrictive table-level permission within a DB. So for each group, select the
- ;; most-restrictive table-level permission. Then use normal coalesce logic to select the *least* restrictive group
- ;; permission.
- (let [perm-values (most-restrictive-per-group
- perm-type
- (entry-group-value-pairs (get-permissions user-id perm-type database-id)
- (constantly true)))]
- (or (coalesce perm-type perm-values)
- (least-permissive-value perm-type)))))
+ (or (get-in (cached-db-perms *full-db-permission-cache* load-full-db-permission-perms
+ user-id [database-id])
+ [perm-type database-id])
+ (least-permissive-value perm-type))))
-(mu/defn schema-permission-for-user :- ::permissions.schema/data-permission-value
+(mu/defn native-download-permission-for-user :- ::permissions.schema/data-permission-value
+ "Returns the effective download permission value for a given user and database ID, for native queries on the database.
+ For each group, the native download permission for a database is equal to the lowest permission level of any table in
+ the database — exactly [[full-db-permission-for-user]] for :perms/download-results, whose cache it shares."
+ [user-id :- ::lib.schema.id/user
+ database-id :- ::lib.schema.id/database]
+ (full-db-permission-for-user user-id :perms/download-results database-id))
+
+;;; -------------------------------- full-schema-permission-for-user --------------------------------------------------
+
+(def ^:dynamic *full-schema-permission-cache*
+ "Request cache for [[full-schema-permission-for-user]]: `{:db-ids #{} :perms {user-id {perm-type {db-id entry}}}}`
+ where each entry is `{:default v, :schemas {schema v}}` — per schema, each group's most restrictive value across
+ the schema's table rows unioned with the group's db-level rows, coalesced most-permissively across groups (the
+ nested query expands db-level rows into every schema to keep the per-group pairing). `:default` — the same
+ aggregation over db-level rows alone — answers schemas with no rows of their own. Schema names are exact. Loaded
+ per database, all permission types at once."
+ (atom {:db-ids #{} :perms {}}))
+
+(defn- load-full-schema-permission-perms
+ [user-id db-ids]
+ (let [base (perm-rows-query-base user-id db-ids)
+ with-where (fn [q extra-clause]
+ (cond-> q extra-clause (update :where conj extra-clause)))
+ branch (fn [bucket schema-expr extra-clause]
+ (-> base
+ (assoc :select [:p.perm_type :p.db_id [schema-expr :schema_name]
+ [[:inline bucket] :bucket] :p.group_id
+ [value-rank-case :r]])
+ (with-where extra-clause)))
+ schemas-sub {:select-distinct [[:p.perm_type :perm_type] [:p.db_id :db_id] [:p.schema_name :schema_name]]
+ :from (:from base)
+ :join (:join base)
+ :left-join (:left-join base)
+ :where (conj (:where base) [:not= :p.table_id nil])}
+ expanded (-> base
+ (update :join conj [schemas-sub :s] [:and
+ [:= :s.perm_type :p.perm_type]
+ [:= :s.db_id :p.db_id]])
+ (assoc :select [:p.perm_type :p.db_id [:s.schema_name :schema_name]
+ [[:inline "schema"] :bucket] :p.group_id
+ [value-rank-case :r]])
+ (update :where conj [:= :p.table_id nil]))]
+ (reduce (fn [m {:keys [perm_type db_id schema_name bucket mn mx]}]
+ (let [perm-type (keyword perm_type)
+ value (ranks->most-permissive-value perm-type [mn mx])]
+ (case bucket
+ "schema" (assoc-in m [perm-type db_id :schemas schema_name] value)
+ "db" (assoc-in m [perm-type db_id :default] value))))
+ {}
+ (t2/query {:select [:i.perm_type :i.db_id :i.schema_name :i.bucket
+ [[:min :i.gmax] :mn] [[:max :i.gmax] :mx]]
+ :from [[{:select [:u.perm_type :u.db_id :u.schema_name :u.bucket :u.group_id
+ [[:max :u.r] :gmax]]
+ :from [[{:union-all [(branch "schema" :p.schema_name [:not= :p.table_id nil])
+ expanded
+ (branch "db" nil [:= :p.table_id nil])]}
+ :u]]
+ :group-by [:u.perm_type :u.db_id :u.schema_name :u.bucket :u.group_id]}
+ :i]]
+ :group-by [:i.perm_type :i.db_id :i.schema_name :i.bucket]}))))
+
+(mu/defn full-schema-permission-for-user :- ::permissions.schema/data-permission-value
"Returns the effective *schema-level* permission value for a given user, permission type, and database ID, and
schema name. If the user has multiple permissions for the given type in different groups, they are coalesced into a
- single value. The schema-level permission is the *least* restrictive table-level permission within that schema.
-
- Schema names are compared with nil and the empty string treated as equivalent, matching how `database-schemas`
- presents these databases to the API."
- [user-id perm-type database-id schema-name :- [:maybe :string]]
+ single value. The schema-level permission is the *most* restrictive table-level permission within that schema."
+ [user-id perm-type database-id schema-name]
(when (not= :model/Table (model-by-perm-type perm-type))
(throw (ex-info (tru "Permission type {0} is not a table-level permission." perm-type)
{perm-type (permissions.schema/data-permissions perm-type)})))
@@ -602,28 +615,53 @@
(most-permissive-value perm-type)
:else
- ;; The schema-level permission is the most-restrictive table-level permission within a schema. So for each group,
- ;; select the most-restrictive table-level permission. Then use normal coalesce logic to select the *least*
- ;; restrictive group permission.
- (let [schema-name (or schema-name "")
- perm-values (into #{}
- (map :value)
- (entry-group-value-pairs (get-permissions user-id perm-type database-id)
- #(= (or % "") schema-name)))]
- (or (coalesce perm-type perm-values)
+ (let [{:keys [default schemas]} (get-in (cached-db-perms *full-schema-permission-cache*
+ load-full-schema-permission-perms
+ user-id [database-id])
+ [perm-type database-id])]
+ (or (get schemas schema-name)
+ default
(least-permissive-value perm-type)))))
-(mu/defn user-has-permission-for-schema? :- :boolean
- "Returns a Boolean indicating whether the user has the specified permission value for the given database ID and schema,
- or a more permissive value."
- [user-id perm-type perm-value database-id schema]
- (at-least-as-permissive? perm-type
- (schema-permission-for-user user-id perm-type database-id schema)
- perm-value))
+;;; ------------------- most-permissive-database-permission-for-user / user-has-any-perms-of-type? --------------------
+
+(def ^:dynamic *most-permissive-permission-cache*
+ "Request cache for [[most-permissive-database-permission-for-user]] and [[user-has-any-perms-of-type?]] — the
+ cross-database questions: `{user-id {perm-type {db-id value}}}`, one coalesced value per database, loaded for
+ *every* database in one aggregated query per permission type. Loading per-row permission data for every database
+ instead stops scaling once instances have thousands of databases and groups."
+ (atom {}))
+
+(defn- load-most-permissive-db-perms
+ [user-id perm-type]
+ (reduce (fn [m {:keys [db_id mn mx]}]
+ (assoc m db_id (ranks->most-permissive-value perm-type [mn mx])))
+ {}
+ (t2/query (-> (perm-rows-query-base user-id nil)
+ (assoc :select [:p.db_id
+ [[:min value-rank-case] :mn]
+ [[:max value-rank-case] :mx]]
+ :group-by [:p.db_id])
+ (update :where conj [:= :p.perm_type (u/qualified-name perm-type)])))))
+
+(defn- most-permissive-db-values
+ "Returns `{db-id most-permissive-value}` for `perm-type` across every database, memoized per request in
+ [[*most-permissive-permission-cache*]]. Databases where the user has no permission rows are absent from the map
+ (they coalesce to the least permissive value)."
+ [user-id perm-type]
+ (if (use-cache? user-id)
+ (or (get-in @*most-permissive-permission-cache* [user-id perm-type])
+ (let [db-id->value (load-most-permissive-db-perms user-id perm-type)]
+ (swap! *most-permissive-permission-cache* assoc-in [user-id perm-type] db-id->value)
+ db-id->value))
+ (load-most-permissive-db-perms user-id perm-type)))
(mu/defn most-permissive-database-permission-for-user :- ::permissions.schema/data-permission-value
"Similar to checking _partial_ permissions with permissions paths - what is the *most permissive* permission the
- user has on any of the tables within this database?"
+ user has on any of the tables within this database?
+
+ Backed by its own light per-request cache — one value per database for the permission type — so checks looping
+ over every database cost a single query however many permission rows exist."
[user-id perm-type database-id]
(when (not= :model/Table (model-by-perm-type perm-type))
(throw (ex-info (tru "Permission type {0} is not a table-level permission." perm-type)
@@ -637,46 +675,107 @@
(most-permissive-value perm-type)
:else
- (let [perm-values (into #{}
- (map :value)
- (entry-group-value-pairs (get-permissions user-id perm-type database-id)
- (constantly true)))]
- (or (coalesce perm-type perm-values)
- (least-permissive-value perm-type)))))
-
-(mu/defn native-download-permission-for-user :- ::permissions.schema/data-permission-value
- "Returns the effective download permission value for a given user and database ID, for native queries on the database.
- For each group, the native download permission for a database is equal to the lowest permission level of any table in
- the database."
- [user-id :- ::lib.schema.id/user
- database-id :- ::lib.schema.id/database]
- (if (is-superuser? user-id)
- (most-permissive-value :perms/download-results)
- (let [pairs (entry-group-value-pairs (get-permissions user-id :perms/download-results database-id)
- (constantly true))]
- (or (coalesce :perms/download-results
- (most-restrictive-per-group :perms/download-results pairs))
- (least-permissive-value :perms/download-results)))))
+ (or (get (most-permissive-db-values user-id perm-type) database-id)
+ (least-permissive-value perm-type))))
(mu/defn user-has-any-perms-of-type? :- :boolean
"Returns a Boolean indicating whether the user has the highest level of access for the given permission type in any
- group, for at least one database or table. Optionally takes `:exclude-db-ids` to exclude specific databases from the check."
+ group, for at least one database or table. Optionally takes `:exclude-db-ids` to exclude specific databases from the
+ check.
+
+ Answered from [[*most-permissive-permission-cache*]] — one aggregated query, memoized per request — so callers
+ that invoke this repeatedly within one request (e.g. once per snippet in a list) cost at most one query, however
+ many databases the instance has. Note that permission rows for inactive tables do not count."
[user-id perm-type & {:keys [exclude-db-ids]}]
(or (is-superuser? user-id)
(and (= perm-type :perms/manage-table-metadata)
(is-data-analyst? user-id))
- (let [value (most-permissive-value perm-type)]
- (t2/exists? :model/DataPermissions
- {:select [[:p.perm_value :value]]
- :from [[:permissions_group_membership :pgm]]
- :join [[:permissions_group :pg] [:= :pg.id :pgm.group_id]
- [:data_permissions :p] [:= :p.group_id :pg.id]]
- :where (into [:and
- [:= :pgm.user_id user-id]
- [:= :p.perm_type (u/qualified-name perm-type)]
- [:= :p.perm_value (u/qualified-name value)]]
- (when (seq exclude-db-ids)
- [[:not-in :p.db_id exclude-db-ids]]))}))))
+ (let [value (most-permissive-value perm-type)
+ exclude? (set exclude-db-ids)]
+ (boolean (some (fn [[db-id db-value]]
+ (and (= db-value value)
+ (not (exclude? db-id))))
+ (most-permissive-db-values user-id perm-type))))))
+
+;;; --------------------------------------- cache assembly ------------------------------------------------------------
+
+(defenterprise enforced-sandboxes-for-user
+ "Given a user-id, returns the set of sandboxes that should be enforced for the provided user ID. This result is
+ cached for the duration of a request. Empty on OSS instances."
+ metabase-enterprise.sandbox.api.util
+ [_user-id]
+ #{})
+
+(def ^:dynamic *sandboxes-for-user*
+ "Filled by `enforced-sandboxes-for-user`. Empty on OSS instances, or EE instances without the `sandboxes` feature."
+ (delay nil))
+
+(defn sandboxes-for-user
+ "Derefs the *sandboxes-for-user* dynamic variable so it can be contained to this namespace"
+ []
+ @*sandboxes-for-user*)
+
+(defmacro with-relevant-permissions-for-user
+ "Populates the per-function permission caches above (and `*sandboxes-for-user*`) for use by the cache-aware
+ functions in this namespace."
+ [user-id & body]
+ `(binding [*table-permission-cache* (atom {:db-ids #{} :perms {}})
+ *database-permission-cache* (atom {:db-ids #{} :perms {}})
+ *schema-permission-cache* (atom {:db-ids #{} :perms {}})
+ *full-db-permission-cache* (atom {:db-ids #{} :perms {}})
+ *full-schema-permission-cache* (atom {:db-ids #{} :perms {}})
+ *most-permissive-permission-cache* (atom {})
+ *sandboxes-for-user* (delay (enforced-sandboxes-for-user ~user-id))]
+ ~@body))
+
+(defn prime-db-perms-cache
+ "Eagerly load every per-database permission cache for the given database IDs for the current user. The checks fill
+ their caches one database at a time, so call this before checking permissions across many databases to pay a
+ handful of small aggregate queries instead of some per database. A no-op for superusers (their checks never read
+ the caches) and when the request cache is unavailable."
+ [db-ids]
+ (when (and (use-cache? api/*current-user-id*)
+ (not api/*is-superuser?*)
+ (seq db-ids))
+ (doseq [[cache-atom load-fn] [[*table-permission-cache* load-table-permission-perms]
+ [*database-permission-cache* load-database-permission-perms]
+ [*schema-permission-cache* load-schema-permission-perms]
+ [*full-db-permission-cache* load-full-db-permission-perms]
+ [*full-schema-permission-cache* load-full-schema-permission-perms]]]
+ (cached-db-perms cache-atom load-fn api/*current-user-id* db-ids))
+ nil))
+
+;;; ---------------------------------------- Fetching a user's permissions --------------------------------------------
+
+(mu/defn table-permission-for-groups :- ::permissions.schema/data-permission-value
+ "Returns the effective permission value provided by a set of *group-ids*, for a provided permission type, database
+ ID, and table ID."
+ [group-ids perm-type database-id table-id]
+ (when (not= :model/Table (model-by-perm-type perm-type))
+ (throw (ex-info (tru "Permission type {0} is not a table-level permission." perm-type)
+ {perm-type (permissions.schema/data-permissions perm-type)})))
+ (let [perm-values (t2/select-fn-set :value
+ :model/DataPermissions
+ {:select [[:p.perm_value :value]]
+ :from [[:data_permissions :p]]
+ :where [:and
+ [:in :p.group_id group-ids]
+ [:= :p.perm_type (u/qualified-name perm-type)]
+ [:= :p.db_id database-id]
+ [:or
+ [:= :table_id table-id]
+ [:= :table_id nil]]]})]
+ (or (coalesce perm-type (conj perm-values (get-additional-table-permission! {:db-id database-id :table-id table-id}
+ perm-type)))
+ (least-permissive-value perm-type))))
+
+(mu/defn groups-have-permission-for-table? :- :boolean
+ "Returns a Boolean indicating whether the provided groups grant the specified permission level or higher for the given
+ table ID, or a more permissive value. (i.e. if a user is in all of these groups, would they have this permission?)"
+ [group-ids perm-type perm-value database-id table-id]
+ (at-least-as-permissive? perm-type
+ (table-permission-for-groups group-ids perm-type database-id table-id)
+ perm-value))
(defn- admin-permission-graph
"Returns the graph representing admin permissions for all groups"
@@ -760,6 +859,7 @@
(assert-value-matches-perm-type perm_type perm_value))
;; Memoized per application DB. Destination status is immutable after creation, so the cache can't go stale.
+
(def ^:private destination-db-id?
"Whether `db-id` is a destination database — one with `router_database_id` set."
(mdb/memoize-for-application-db
@@ -1184,9 +1284,11 @@
:where [:and
[:in :group_id group-ids]
[:in :perm_type ["perms/create-queries" "perms/download-results"]]
- [:not-in :db_id {:select [:id]
- :from [(t2/table-name :model/Database)]
- :where [:= :is_audit true]}]]}))
+ [:not [:exists {:select [1]
+ :from [[(t2/table-name :model/Database) :audit_db]]
+ :where [:and
+ [:= :audit_db.is_audit true]
+ [:= :audit_db.id :data_permissions.db_id]]}]]]}))
;; Group by (group_id, perm_type) → set of values
perms-by-grp (when all-perms
(reduce (fn [acc {:keys [group_id perm_type perm_value]}]
diff --git a/src/metabase/permissions/user.clj b/src/metabase/permissions/user.clj
index a170bd8cf3c..c3f873c3e67 100644
--- a/src/metabase/permissions/user.clj
+++ b/src/metabase/permissions/user.clj
@@ -49,7 +49,6 @@
based on their create-queries permissions across all databases."
[user-id]
(let [db-ids (t2/select-pks-set :model/Database)
- _ (data-perms/prime-db-cache db-ids)
create-query-perms (into #{}
(map #(data-perms/most-permissive-database-permission-for-user
user-id :perms/create-queries %))
diff --git a/src/metabase/permissions_rest/data_permissions/graph.clj b/src/metabase/permissions_rest/data_permissions/graph.clj
index 49cd95f5626..0f6dd347d5b 100644
--- a/src/metabase/permissions_rest/data_permissions/graph.clj
+++ b/src/metabase/permissions_rest/data_permissions/graph.clj
@@ -211,9 +211,8 @@
Rows are fetched with a raw query rather than a model select, and realized one at a time: key access on unrealized
result-set rows goes through toucan2's deferred-row machinery on every lookup, which benchmarked ~15x slower than
- realizing each row once and reading plain map keys (see
- [[metabase.permissions.models.data-permissions/relevant-permissions-for-user-and-dbs]]). The raw query skips the
- model transforms, so `:type` and `:value` arrive as strings and are keywordized here."
+ realizing each row once and reading plain map keys. The raw query skips the model transforms, so `:type` and
+ `:value` arrive as strings and are keywordized here."
[{:keys [group-id group-ids db-id perm-type audit?]}]
(eduction
(map (fn [row]
@@ -234,9 +233,11 @@
(when group-id [:= :group_id group-id])
(when group-ids [:in :group_id group-ids])
(when-not audit? [:not= :db_id audit/audit-db-id])
- [:not-in :db_id {:select [:id]
- :from [(t2/table-name :model/Database)]
- :where [:not= :router_database_id nil]}]]
+ [:not [:exists {:select [1]
+ :from [[(t2/table-name :model/Database) :router_db]]
+ :where [:and
+ [:not= :router_db.router_database_id nil]
+ [:= :router_db.id :db_id]]}]]]
:order-by [:group_id :db_id]})))
(defn- add-perm
diff --git a/src/metabase/pulse/task/send_pulses.clj b/src/metabase/pulse/task/send_pulses.clj
index bbc65e886b8..5c8d9346c1f 100644
--- a/src/metabase/pulse/task/send_pulses.clj
+++ b/src/metabase/pulse/task/send_pulses.clj
@@ -110,11 +110,12 @@
(tracing/with-span :tasks "task.pulse.clear-orphan-channels" {:pulse/id pulse-id}
(when-let [ids-to-delete (seq
(for [channel (t2/select [:model/PulseChannel :id :details :channel_id :channel_type]
- :pulse_id pulse-id
- :id [:not-in {:select [[:pulse_channel_id :id]]
- :from :pulse_channel_recipient
- :group-by [:pulse_channel_id]
- :having [:>= :%count.* [:raw 1]]}])
+ {:where [:and
+ [:= :pulse_id pulse-id]
+ [:not [:exists {:select [1]
+ :from [:pulse_channel_recipient]
+ :where [:= :pulse_channel_recipient.pulse_channel_id
+ :pulse_channel.id]}]]]})
:when (case (:channel_type channel)
:email
(empty? (get-in channel [:details :emails]))
diff --git a/src/metabase/queries/metadata.clj b/src/metabase/queries/metadata.clj
index e7c1e4ac120..daeed8ef482 100644
--- a/src/metabase/queries/metadata.clj
+++ b/src/metabase/queries/metadata.clj
@@ -40,7 +40,7 @@
(defn- get-databases
[ids]
(when (seq ids)
- (perms/prime-db-cache ids)
+ (perms/prime-db-perms-cache ids)
(into [] (comp (filter mi/can-read?)
(map #(assoc % :native_permissions (get-native-perms-info %))))
(t2/select :model/Database :id [:in ids]))))
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index 31ea460a604..317d015f9fa 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -260,7 +260,7 @@
(query-perms/with-card-instances (when (seq source-card-ids)
(t2/select-fn->fn :id identity [:model/Card :id :collection_id :card_schema]
:id [:in source-card-ids]))
- (perms/prime-db-cache (into #{} (map :database_id cards)))
+ (perms/prime-db-perms-cache (into #{} (keep :database_id) cards-with-non-empty-queries))
(mi/instances-with-hydrated-data
cards :can_run_adhoc_query
(fn []
diff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.clj
index 2819327bb80..d7e27b0dc01 100644
--- a/src/metabase/search/appdb/index.clj
+++ b/src/metabase/search/appdb/index.clj
@@ -117,21 +117,22 @@
(defn- orphan-indexes []
(map (comp keyword u/lower-case-en :table_name)
- (t2/query {:select [:table_name]
- :from :information_schema.tables
+ (t2/query {:select [:ist.table_name]
+ :from [[:information_schema.tables :ist]]
:where [:and
- [:= :table_schema :%current_schema]
+ [:= :ist.table_schema :%current_schema]
[:or
- [:like [:lower :table_name] [:inline "search\\_index\\_\\_%"]]
+ [:like [:lower :ist.table_name] [:inline "search\\_index\\_\\_%"]]
;; legacy table names
- [:in [:lower :table_name]
+ [:in [:lower :ist.table_name]
(mapv #(vector :inline %) ["search_index" "search_index_next" "search_index_retired"])]]
;; Exclude temp tables — they are managed by with-temp-index-table
- [:not-like [:lower :table_name] [:inline "%\\_temp"]]
- [:not-in [:lower :table_name]
- {:select [:%lower.index_name]
- :from [(t2/table-name :model/SearchIndexMetadata)]
- :where [:= :engine [:inline "appdb"]]}]]})))
+ [:not-like [:lower :ist.table_name] [:inline "%\\_temp"]]
+ [:not [:exists {:select [1]
+ :from [[(t2/table-name :model/SearchIndexMetadata) :sim]]
+ :where [:and
+ [:= :sim.engine [:inline "appdb"]]
+ [:= [:lower :sim.index_name] [:lower :ist.table_name]]]}]]]})))
(defn- delete-obsolete-tables! []
;; Delete metadata around indexes that are no longer needed.
diff --git a/src/metabase/segments/api.clj b/src/metabase/segments/api.clj
index 192efc85024..9fb31776e4d 100644
--- a/src/metabase/segments/api.clj
+++ b/src/metabase/segments/api.clj
@@ -7,6 +7,7 @@
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.models.interface :as mi]
+ [metabase.permissions.core :as perms]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]
@@ -75,11 +76,14 @@
(api.macros/defendpoint :get "/"
"Fetch *all* `Segments`."
[]
- (as-> (t2/select :model/Segment
- :archived false
- {:order-by [[:%lower.name :asc]]}) segments
- (filter mi/can-read? segments)
- (t2/hydrate segments :creator :definition_description)))
+ (let [segments (t2/select :model/Segment
+ :archived false
+ {:order-by [[:%lower.name :asc]]})
+ table-ids (into #{} (keep :table_id) segments)]
+ (when (seq table-ids)
+ (perms/prime-db-perms-cache (t2/select-fn-set :db_id :model/Table :id [:in table-ids])))
+ (-> (filterv mi/can-read? segments)
+ (t2/hydrate :creator :definition_description))))
(defn- write-check-and-update-segment!
"Check whether current user has write permissions, then update Segment with values in `body`. Publishes appropriate
diff --git a/src/metabase/transforms/models/transform_run_cancelation.clj b/src/metabase/transforms/models/transform_run_cancelation.clj
index e801158534b..731e12f99c3 100644
--- a/src/metabase/transforms/models/transform_run_cancelation.clj
+++ b/src/metabase/transforms/models/transform_run_cancelation.clj
@@ -38,19 +38,24 @@
[]
(t2/reducible-select :model/TransformRunCancelation))
+(defn- no-active-run-clause
+ []
+ [:not [:exists {:select [1]
+ :from [[:transform_run :wr]]
+ :where [:and
+ [:= :wr.id :transform_run_cancelation.run_id]
+ :wr.is_active]}]])
+
(defn delete-cancelation!
"Delete a cancelation once it has been handled."
[run-id]
(t2/delete! :model/TransformRunCancelation
- :run_id run-id
- :run_id [:not-in {:select :wr.id
- :from [[:transform_run :wr]]
- :where :wr.is_active}]))
+ {:where [:and
+ [:= :run_id run-id]
+ (no-active-run-clause)]}))
(defn delete-old-canceling-runs!
"Delete cancelations for runs that are no longer running."
[]
(t2/delete! :model/TransformRunCancelation
- :run_id [:not-in {:select :wr.id
- :from [[:transform_run :wr]]
- :where :wr.is_active}]))
+ {:where (no-active-run-clause)}))
diff --git a/src/metabase/typed_schemas/api/schema/table.clj b/src/metabase/typed_schemas/api/schema/table.clj
index e22846875cd..4ac3cd42a3a 100644
--- a/src/metabase/typed_schemas/api/schema/table.clj
+++ b/src/metabase/typed_schemas/api/schema/table.clj
@@ -5,6 +5,7 @@
[metabase.lib-be.core :as lib-be]
[metabase.metabot.core :as metabot]
[metabase.models.interface :as mi]
+ [metabase.permissions.core :as perms]
[metabase.typed-schemas.api.common :as common]
[metabase.typed-schemas.api.schema.common :as schema.common]
[metabase.typed-schemas.api.scope :as scope]
@@ -37,6 +38,12 @@
:tableId (when (integer? table-id) table-id)
:defaultTemporalBucket (:unit field)))))
+(defn- filter-readable-tables
+ "Filters tables down to the ones the current user can read."
+ [tables]
+ (perms/prime-db-perms-cache (into #{} (map :db_id) tables))
+ (filter mi/can-read? tables))
+
(defn select-tables
"Returns readable tables, with optional database and table-id scopes.
@@ -50,7 +57,7 @@
database-ids (conj (scope/database-id-filter-clause database-ids :db_id))
table-ids (conj (scope/id-filter-clause table-ids :id)))
:order-by [[:name :asc] [:id :asc]]})
- (filter mi/can-read?))))
+ (filter-readable-tables))))
(defn select-library-tables
"Returns published tables from the library based on the given scope."
@@ -61,7 +68,7 @@
[:= :is_published true]
(scope/id-filter-clause data-collection-ids :collection_id)]
:order-by [[:name :asc] [:id :asc]]})
- (filter mi/can-read?)))
+ (filter-readable-tables)))
(defn segment-schema
"Returns the schema for a segment."
diff --git a/src/metabase/users/models/user.clj b/src/metabase/users/models/user.clj
index 3ffdadfc4bd..9e1cc948a8d 100644
--- a/src/metabase/users/models/user.clj
+++ b/src/metabase/users/models/user.clj
@@ -545,6 +545,18 @@
[:like :%lower.last_name (wildcard-query query)]
[:like :%lower.email (wildcard-query query)]])
+(defn- table-metadata-perms-exist-clause
+ "EXISTS clause, correlated to :core_user.id, testing whether the user is in a group that grants
+ manage-table-metadata."
+ []
+ [:exists {:select [1]
+ :from [[:permissions_group_membership :pgm]]
+ :join [[:data_permissions :p] [:= :p.group_id :pgm.group_id]]
+ :where [:and
+ [:= :pgm.user_id :core_user.id]
+ [:= :p.perm_type "perms/manage-table-metadata"]
+ [:= :p.perm_value "yes"]]}])
+
(defn filter-clauses
"Honeysql clauses for filtering on users.
@@ -571,23 +583,11 @@
[:or
:core_user.is_data_analyst
:core_user.is_superuser
- [:in :core_user.id
- {:select-distinct [:pgm.user_id]
- :from [[:permissions_group_membership :pgm]]
- :join [[:data_permissions :p] [:= :p.group_id :pgm.group_id]]
- :where [:and
- [:= :p.perm_type "perms/manage-table-metadata"]
- [:= :p.perm_value "yes"]]}]]
+ (table-metadata-perms-exist-clause)]
[:and
[:not :core_user.is_data_analyst]
[:not :core_user.is_superuser]
- [:not-in :core_user.id
- {:select-distinct [:pgm.user_id]
- :from [[:permissions_group_membership :pgm]]
- :join [[:data_permissions :p] [:= :p.group_id :pgm.group_id]]
- :where [:and
- [:= :p.perm_type "perms/manage-table-metadata"]
- [:= :p.perm_value "yes"]]}]]))
+ [:not (table-metadata-perms-exist-clause)]]))
(some? group-ids) (sql.helpers/right-join
:permissions_group_membership
[:= :core_user.id :permissions_group_membership.user_id])
diff --git a/src/metabase/util/memoize.cljc b/src/metabase/util/memoize.cljc
index 389c430f462..d2c3e29e27b 100644
--- a/src/metabase/util/memoize.cljc
+++ b/src/metabase/util/memoize.cljc
@@ -72,28 +72,36 @@
its argument** — it normalizes *within* an equivalence class (pool the contents, reorder a
map) without changing equality. The zero-arg arity uses `identity` (intern values as-is).
- Concurrent callers converge on a single canonical instance even on a simultaneous miss.
+ - `nil` is returned unchanged without calling `canon-fn` or adding an entry to the cache.
In CLJ this is backed by a `ConcurrentHashMap`, so it is thread-safe; `canon-fn` may run more
than once for the same value under contention (extra results are discarded), so it must be pure.
- Keys and values must be non-nil with a good `Object.hashCode` (Clojure values qualify). Like
+ Non-nil values must have a good `Object.hashCode` (Clojure values qualify). Like
[[fast-memo]] it doesn't support the memoization API (`memo-swap!`, `memoized?`, etc.).
In CLJS (single-threaded) this is a plain atom-backed map with the same contract."
([] (fast-interner identity))
([canon-fn]
+ ;; `if-some` rather than `or` on the lookups so that a cached `false` counts as a hit
#?(:clj (let [cache (java.util.concurrent.ConcurrentHashMap.)]
(fn [x]
- (or (.get cache x)
+ (when (some? x)
+ (if-some [cached (.get cache x)]
+ cached
(let [canon (canon-fn x)]
;; key by `canon`, not `x`: `canon` = `x`, so this lookup/insert hits the
;; same bin a later equal input would, while retaining the canonical object.
- (or (.putIfAbsent cache canon canon) canon)))))
+ (if-some [existing (.putIfAbsent cache canon canon)]
+ existing
+ canon))))))
:cljs (let [cache (atom {})]
(fn [x]
- (or (get @cache x)
+ (when (some? x)
+ (if-some [cached (get @cache x)]
+ cached
(let [canon (canon-fn x)]
(swap! cache assoc canon canon)
- canon)))))))
+ canon))))))))
(defn fast-bounded
"Variant of [[bounded]] that optimizes a common special case: a function with only a single argument, where that
diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.clj
index d7d4cb9879a..2a17638584f 100644
--- a/src/metabase/warehouse_schema/models/field.clj
+++ b/src/metabase/warehouse_schema/models/field.clj
@@ -166,8 +166,9 @@
:from [:metabase_field]
:where [:and
[:= :fk_target_field_id (:id field)]
- [:not [:in :id {:select [:field_id]
- :from [:metabase_field_user_settings]}]]]}
+ [:not [:exists {:select [1]
+ :from [:metabase_field_user_settings]
+ :where [:= :metabase_field_user_settings.field_id :metabase_field.id]}]]]}
sql (sql/format q :dialect (mdb/quoting-style (mdb/db-type)))]
(t2/insert! :model/FieldUserSettings
(map (fn [{:keys [id]}] {:field_id id})
diff --git a/src/metabase/warehouse_schema/table.clj b/src/metabase/warehouse_schema/table.clj
index 5dd76c3a58b..22cbec61b05 100644
--- a/src/metabase/warehouse_schema/table.clj
+++ b/src/metabase/warehouse_schema/table.clj
@@ -3,6 +3,7 @@
[medley.core :as m]
[metabase.api.common :as api]
[metabase.models.interface :as mi]
+ [metabase.permissions.core :as perms]
[metabase.premium-features.core :as premium-features :refer [defenterprise]]
[metabase.util :as u]
[metabase.warehouse-schema.models.field-values :as field-values]
@@ -69,8 +70,9 @@
(batch-fetch-query-metadatas* ids nil))
([ids {:keys [include-sensitive-fields?]}]
(when (seq ids)
- (let [tables (->> (t2/select :model/Table :id [:in ids])
- (filter can-access-table-for-query-metadata?))
+ (let [tables (t2/select :model/Table :id [:in ids])
+ _ (perms/prime-db-perms-cache (into #{} (map :db_id) tables))
+ tables (filter can-access-table-for-query-metadata? tables)
tables (t2/hydrate tables
[:fields [:target :has_field_values] :has_field_values :dimensions :name_field]
:segments
@@ -150,16 +152,33 @@
[]
"Everything else")
+(defn- cards->card-id->metadata-fields
+ "Batch-fetch the underlying Fields for the `:result_metadata` columns of `cards`, returning a map of card ID to its
+ hydrated Fields. One select (plus hydration) covers every card, instead of the per-card lookup
+ [[card-result-metadata->virtual-fields]] falls back to."
+ [cards]
+ (let [metadata-field-ids (into #{}
+ (comp (mapcat :result_metadata)
+ (keep :id))
+ cards)
+ metadata-fields (if (seq metadata-field-ids)
+ (-> (t2/select :model/Field :id [:in metadata-field-ids])
+ (t2/hydrate [:target :has_field_values] :has_field_values :dimensions :name_field)
+ (->> (m/index-by :id)))
+ {})]
+ (into {}
+ (map (fn [card]
+ [(:id card) (into []
+ (keep (comp metadata-fields :id))
+ (:result_metadata card))]))
+ cards)))
+
(defn card->virtual-table
"Return metadata for a 'virtual' table for a `card` in the Saved Questions 'virtual' database. Optionally include
'virtual' fields as well."
[{:keys [database_id] :as card} & {:keys [include-database? include-fields? databases card-id->metadata-fields]}]
- ;; if collection isn't already hydrated then do so
(let [card-type (:type card)
- dataset-query (:dataset_query card)
- database (when (int? database_id)
- (or (get databases database_id)
- (t2/select-one :model/Database :id database_id)))]
+ dataset-query (:dataset_query card)]
(cond-> {:id (str "card__" (u/the-id card))
:db_id (:database_id card)
:display_name (:name card)
@@ -174,7 +193,10 @@
(assoc :dataset_query dataset-query)
include-database?
- (assoc :db (when (and database (mi/can-read? database)) database))
+ (assoc :db (when-let [database (when (int? database_id)
+ (or (get databases database_id)
+ (t2/select-one :model/Database :id database_id)))]
+ (when (mi/can-read? database) database)))
include-fields?
(assoc :fields (card-result-metadata->virtual-fields (u/the-id card)
@@ -182,6 +204,19 @@
(when card-id->metadata-fields
(card-id->metadata-fields (u/the-id card))))))))
+(defn cards->virtual-tables
+ "Return [[card->virtual-table]] metadata for `cards`, batching the Field fetch that `:include-fields?` needs so it
+ costs one select for the whole list instead of one per card."
+ [cards & {:keys [include-database? include-fields? databases]}]
+ (let [card-id->metadata-fields (when include-fields?
+ (cards->card-id->metadata-fields cards))]
+ (for [card cards]
+ (card->virtual-table card
+ :include-database? include-database?
+ :include-fields? include-fields?
+ :databases databases
+ :card-id->metadata-fields card-id->metadata-fields))))
+
(defn- remove-nested-pk-fk-semantic-types
"This method clears the semantic_type attribute for PK/FK fields of nested queries. Those fields having a semantic
type confuses the frontend and it can really used in the same way"
@@ -218,21 +253,7 @@
dbs (if (seq cards)
(t2/select-pk->fn identity :model/Database :id [:in (into #{} (map :database_id) cards)])
{})
- metadata-field-ids (into #{}
- (comp (mapcat :result_metadata)
- (keep :id))
- cards)
- metadata-fields (if (seq metadata-field-ids)
- (-> (t2/select :model/Field :id [:in metadata-field-ids])
- (t2/hydrate [:target :has_field_values] :has_field_values :dimensions :name_field)
- (->> (m/index-by :id)))
- {})
- card-id->metadata-fields (into {}
- (map (fn [card]
- [(:id card) (into []
- (keep (comp metadata-fields :id))
- (:result_metadata card))]))
- cards)
+ card-id->metadata-fields (cards->card-id->metadata-fields cards)
readable-cards (t2/hydrate (filter mi/can-read? cards) :metrics)]
(for [card readable-cards]
;; Models can have user configured FK columns which, for MBQL models, we cannot distinguish from
diff --git a/src/metabase/warehouse_schema_rest/api/table.clj b/src/metabase/warehouse_schema_rest/api/table.clj
index 214f06bb6bd..0c5f3920d09 100644
--- a/src/metabase/warehouse_schema_rest/api/table.clj
+++ b/src/metabase/warehouse_schema_rest/api/table.clj
@@ -123,6 +123,8 @@
(premium-features/any-transforms-enabled?) (conj :transform))]
(as-> (t2/select :model/Table query) tables
(apply t2/hydrate tables hydrations)
+ (do (perms/prime-db-perms-cache (into #{} (map :db_id) tables))
+ tables)
(into [] (comp (filter mi/can-read?)
(if can-query (filter mi/can-query?) identity)
(if can-write (filter mi/can-write?) identity)
diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.clj
index dba916bafd0..84bbe6d2f61 100644
--- a/src/metabase/warehouses/models/database.clj
+++ b/src/metabase/warehouses/models/database.clj
@@ -424,26 +424,32 @@
{:pre [(pos-int? database-id)]}
;; Field has `define-before-delete` deleting children, but we'll delete them all at once because they refer same
;; database - iteratively, deleting those that no one depends on first
- (loop []
- (let [deleted (t2/query-one
- {:delete-from (t2/table-name :model/Field)
- :where
- [:and
- [:in :table_id {:from [(t2/table-name :model/Table)]
- :select [:id]
- :where [:= :db_id database-id]}]
- ;; Double-wrapped subquery to work around MySQL limitation
- [:not-in :id {:select [:parent_id]
- :from [[{:select [:parent_id]
- :from [(t2/table-name :model/Field)]
- :where [:and
- [:not= :parent_id nil]
- [:in :table_id {:from [(t2/table-name :model/Table)]
- :select [:id]
- :where [:= :db_id database-id]}]]}
- :parent_fields]]}]]})]
- (when (pos? deleted)
- (recur)))))
+ (let [no-children-clause
+ (if (= (mdb/db-type) :mysql)
+ ;; double-wrapped subquery to work around the MySQL restriction on selecting from the DELETE target
+ [:not-in :id {:select [:parent_id]
+ :from [[{:select [:parent_id]
+ :from [(t2/table-name :model/Field)]
+ :where [:and
+ [:not= :parent_id nil]
+ [:in :table_id {:from [(t2/table-name :model/Table)]
+ :select [:id]
+ :where [:= :db_id database-id]}]]}
+ :parent_fields]]}]
+ [:not [:exists {:select [1]
+ :from [[(t2/table-name :model/Field) :child_field]]
+ :where [:= :child_field.parent_id :metabase_field.id]}]])]
+ (loop []
+ (let [deleted (t2/query-one
+ {:delete-from (t2/table-name :model/Field)
+ :where
+ [:and
+ [:in :table_id {:from [(t2/table-name :model/Table)]
+ :select [:id]
+ :where [:= :db_id database-id]}]
+ no-children-clause]})]
+ (when (pos? deleted)
+ (recur))))))
(t2/define-before-delete :model/Database
[{id :id, driver :engine, :as database}]
diff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.clj
index 4abf66dc186..2960f6be5e0 100644
--- a/src/metabase/warehouses_rest/api.clj
+++ b/src/metabase/warehouses_rest/api.clj
@@ -77,6 +77,7 @@
:visibility_type nil
{:order-by [[:%lower.schema :asc]
[:%lower.display_name :asc]]})
+ _ (perms/prime-db-perms-cache (map :id dbs))
filtered-tables (cond->> (filter mi/can-read? all-tables)
can-query? (filter mi/can-query?)
can-write-metadata? (filter mi/can-write?))
@@ -128,7 +129,6 @@
native queries, but not to create new ones. With the advent of what is currently being called 'Space-Age
Permissions', all Cards' permissions are based on their parent Collection, removing the need for native read perms."
[dbs :- [:maybe [:sequential :map]]]
- (perms/prime-db-cache (map :id dbs))
(for [db dbs]
(assoc db
:native_permissions
@@ -200,13 +200,14 @@
(card-has-ambiguous-columns? card)))))
(defn- ids-of-dbs-that-support-source-queries []
- (set (filter (fn [db-id]
- (try
- (when-let [db (t2/select-one :model/Database :id db-id)]
- (driver.u/supports? (driver.u/database->driver db) :nested-queries db))
- (catch Throwable e
- (log/errorf "Error determining whether Database supports nested queries: %s" (ex-message e)))))
- (t2/select-pks-set :model/Database))))
+ ;; the nested-queries check only reads the engine — don't realize full rows (decrypted :details etc.) for it
+ (set (keep (fn [db]
+ (try
+ (when (driver.u/supports? (driver.u/database->driver db) :nested-queries db)
+ (:id db))
+ (catch Throwable e
+ (log/errorf "Error determining whether Database supports nested queries: %s" (ex-message e)))))
+ (t2/select [:model/Database :id :engine]))))
(mu/defn- source-query-cards
"Fetch the Cards that can be used as source queries (e.g. presented as virtual tables)."
@@ -252,8 +253,8 @@
Builder.)"
[card-type :- ::queries.schema/card-type
& {:keys [include-fields?]}]
- (for [card (source-query-cards card-type)]
- (schema.table/card->virtual-table card :include-fields? include-fields?)))
+ (schema.table/cards->virtual-tables (source-query-cards card-type)
+ :include-fields? include-fields?))
(mu/defn- saved-cards-virtual-db-metadata
[card-type :- ::queries.schema/card-type
@@ -347,7 +348,8 @@
(:clause (mi/visible-filter-clause :model/Database :id user-info {:perms/manage-table-metadata :yes}))]]
base-where)
dbs (t2/select :model/Database {:order-by [:%lower.name :%lower.engine]
- :where where-clause})]
+ :where where-clause})
+ _ (perms/prime-db-perms-cache (map :id dbs))]
(cond-> (-> dbs add-native-perms-info add-transforms-perms-info)
include-tables? (add-tables :can-query? can-query? :can-write-metadata? can-write-metadata?)
include-schemas? add-schemas
diff --git a/test/metabase/native_query_snippets/api_test.clj b/test/metabase/native_query_snippets/api_test.clj
index f4e968d76a0..902ab6dfe2c 100644
--- a/test/metabase/native_query_snippets/api_test.clj
+++ b/test/metabase/native_query_snippets/api_test.clj
@@ -38,6 +38,28 @@
(is (contains? snippets-from-api (select-keys snippet-1 test-snippet-fields)))
(is (contains? snippets-from-api (select-keys snippet-2 test-snippet-fields)))))))))))
+(deftest list-snippets-call-count-test
+ (testing "GET /api/native-query-snippet app-DB call count should not scale with the number of snippets"
+ (mt/with-model-cleanup [:model/NativeQuerySnippet]
+ (letfn [(insert-snippets! [n]
+ (dotimes [_ n]
+ (t2/insert! :model/NativeQuerySnippet
+ {:name (mt/random-name)
+ :content "1 = 1"
+ :creator_id (mt/user->id :crowberto)})))
+ (warm-call-count! []
+ ;; first request pays one-time priming; measure the second. Use a non-admin so the
+ ;; per-snippet permission check does not short-circuit on superuser status.
+ (mt/user-http-request :rasta :get 200 (snippet-url))
+ (t2/with-call-count [call-count]
+ (mt/user-http-request :rasta :get 200 (snippet-url))
+ (call-count)))]
+ (insert-snippets! 2)
+ (let [calls-with-2 (warm-call-count!)
+ _ (insert-snippets! 8)
+ calls-with-10 (warm-call-count!)]
+ (is (<= calls-with-10 (+ calls-with-2 3))))))))
+
(deftest read-snippet-api-test
(mt/with-full-data-perms-for-all-users!
(testing "GET /api/native-query-snippet/:id"
diff --git a/test/metabase/permissions/models/data_permissions_test.clj b/test/metabase/permissions/models/data_permissions_test.clj
index 5380c1680ca..9592f939fc1 100644
--- a/test/metabase/permissions/models/data_permissions_test.clj
+++ b/test/metabase/permissions/models/data_permissions_test.clj
@@ -8,6 +8,7 @@
[metabase.permissions.models.data-permissions :as data-perms]
[metabase.permissions.models.permissions-group :as perms-group]
[metabase.test :as mt]
+ [metabase.test.util.dynamic-redefs :as dynamic-redefs]
[toucan2.core :as t2])
(:import
(clojure.lang ExceptionInfo)))
@@ -21,25 +22,67 @@
:blocked [:perms/view-data #{:blocked}]
nil [:perms/view-data #{}])))
-(deftest ^:parallel rows->cache-entry-test
- (testing "returns nil when there are no rows"
- (is (nil? (#'data-perms/rows->cache-entry identity []))))
- (testing "duplicate rows keep every distinct value visible to coalescing, and each table-level row keeps its own
- schema_name (data_permissions has no unique constraint on (group_id, perm_type, db_id, table_id), and
- schema_name is denormalized so rows for one table can disagree)"
- (let [entry (#'data-perms/rows->cache-entry
- identity
- [{:group_id 1 :table_id nil :schema_name nil :perm_value :query-builder}
- {:group_id 1 :table_id nil :schema_name nil :perm_value :no}
- {:group_id 1 :table_id 10 :schema_name "public" :perm_value :query-builder}
- {:group_id 1 :table_id 10 :schema_name "public" :perm_value :no}
- {:group_id 2 :table_id 10 :schema_name "legacy" :perm_value :query-builder}])]
- (is (= {1 #{:query-builder :no}}
- (:groups entry)))
- (is (= {10 {1 #{{:schema "public" :value :query-builder}
- {:schema "public" :value :no}}
- 2 #{{:schema "legacy" :value :query-builder}}}}
- (:tables entry))))))
+(deftest ^:parallel ranks->most-permissive-value-test
+ (testing "reconstructs coalesce from a bucket's [min max] value ranks"
+ ;; :perms/create-queries values: [:query-builder-and-native :query-builder :no]
+ (are [expected pair] (= expected (#'data-perms/ranks->most-permissive-value :perms/create-queries pair))
+ :query-builder-and-native [0 2]
+ :query-builder [1 1]
+ :no [2 2]
+ nil nil))
+ (testing ":perms/view-data's special rule: a :blocked row overrides :legacy-no-self-service but not :unrestricted"
+ ;; :perms/view-data values: [:unrestricted :legacy-no-self-service :blocked]
+ (are [expected pair] (= expected (#'data-perms/ranks->most-permissive-value :perms/view-data pair))
+ :unrestricted [0 2]
+ :blocked [1 2]
+ :legacy-no-self-service [1 1]
+ :blocked [2 2])))
+
+(deftest ^:parallel combine-rank-pairs-test
+ (is (= [0 2] (#'data-perms/combine-rank-pairs [1 2] [0 1])))
+ (is (= [1 1] (#'data-perms/combine-rank-pairs nil [1 1])))
+ (is (nil? (#'data-perms/combine-rank-pairs nil nil))))
+
+(deftest permission-caches-load-on-demand-test
+ (let [user-id 1
+ loads (atom [])
+ cache (atom {:db-ids #{} :perms {}})]
+ (binding [api/*current-user-id* user-id
+ data-perms/*database-permission-cache* cache]
+ (mt/with-dynamic-fn-redefs [data-perms/load-database-permission-perms
+ (fn [_user-id db-ids]
+ (swap! loads conj db-ids)
+ {:perms/view-data (zipmap db-ids (repeat :unrestricted))})]
+ (testing "a per-function cache loads only databases not already fetched"
+ (letfn [(check! [db-ids]
+ (#'data-perms/cached-db-perms cache
+ @#'data-perms/load-database-permission-perms
+ user-id db-ids))]
+ (check! [10])
+ (check! [10 11])
+ (check! [10 11])
+ (is (= [[10] [11]] @loads))
+ (is (= #{10 11} (:db-ids @cache)))
+ (is (= :unrestricted
+ (get-in @cache [:perms user-id :perms/view-data 10])))))))))
+
+(deftest permission-caches-memoize-most-permissive-values-test
+ (let [user-id 1
+ loads (atom 0)
+ cache (atom {})]
+ (binding [api/*current-user-id* user-id
+ data-perms/*most-permissive-permission-cache* cache]
+ (mt/with-dynamic-fn-redefs [data-perms/load-most-permissive-db-perms
+ (fn [_user-id perm-type]
+ (swap! loads inc)
+ {10 (data-perms/most-permissive-value perm-type)})]
+ (testing "cross-database questions share one memoized load per permission type"
+ (is (true? (data-perms/user-has-any-perms-of-type? user-id :perms/create-queries)))
+ (is (= :query-builder-and-native
+ (data-perms/most-permissive-database-permission-for-user user-id :perms/create-queries 10)))
+ (is (= :no
+ (data-perms/most-permissive-database-permission-for-user user-id :perms/create-queries 11)))
+ (is (= 1 @loads)))))))
(deftest ^:parallel at-least-as-permissive?-test
(testing "at-least-as-permissive? correctly compares permission values"
@@ -267,10 +310,15 @@
(t2/with-call-count [call-count]
(is (= :yes (data-perms/database-permission-for-user user-id :perms/manage-database database-id-1)))
(is (zero? (call-count)))))
- ;; Fetching perms for a different DB is a cache miss
- (t2/with-call-count [call-count]
- (is (= :no (data-perms/database-permission-for-user user-id :perms/manage-database database-id-2)))
- (is (= 1 (call-count)))))))))
+ ;; Fetching perms for a different DB is a cache miss (spy on the load fn rather than counting queries so the
+ ;; assertion doesn't depend on how many statements a load issues)
+ (let [loads (atom 0)
+ orig (dynamic-redefs/original-fn #'data-perms/load-database-permission-perms)]
+ (mt/with-dynamic-fn-redefs [data-perms/load-database-permission-perms (fn [user-id db-ids]
+ (swap! loads inc)
+ (orig user-id db-ids))]
+ (is (= :no (data-perms/database-permission-for-user user-id :perms/manage-database database-id-2)))
+ (is (= 1 @loads)))))))))
(deftest table-permission-for-user-test
(mt/with-temp [:model/PermissionsGroup {group-id-1 :id} {}
@@ -337,6 +385,37 @@
(is (= :unrestricted
(data-perms/table-permission-for-user user-id :perms/view-data database-id table-id-1))))))))
+(deftest inactive-table-grants-ignored-test
+ (testing "a grant that exists only on an inactive table does not count for value-aggregating checks"
+ (mt/with-temp [:model/PermissionsGroup {group-id :id} {}
+ :model/User {user-id :id} {}
+ :model/PermissionsGroupMembership {} {:user_id user-id
+ :group_id group-id}
+ :model/Database {database-id :id} {}
+ :model/Table {table-id :id} {:db_id database-id}]
+ (mt/with-no-data-perms-for-all-users!
+ (data-perms/set-table-permission! group-id table-id :perms/manage-table-metadata :yes)
+ (letfn [(check! [has-any? most-permissive]
+ (testing "\nreading straight from the DB"
+ (is (= has-any?
+ (data-perms/user-has-any-perms-of-type? user-id :perms/manage-table-metadata)))
+ (is (= most-permissive
+ (data-perms/most-permissive-database-permission-for-user user-id :perms/manage-table-metadata database-id))))
+ (testing "\nreading from the request cache"
+ (mt/with-current-user user-id
+ (is (= has-any?
+ (data-perms/user-has-any-perms-of-type? user-id :perms/manage-table-metadata)))
+ (is (= most-permissive
+ (data-perms/most-permissive-database-permission-for-user user-id :perms/manage-table-metadata database-id))))))]
+ (testing "\na grant on an active table counts"
+ (check! true :yes))
+ (testing "\nthe same grant does not count once the table is deactivated"
+ (t2/update! :model/Table table-id {:active false})
+ (check! false :no))
+ (testing "\nreactivating the table restores it"
+ (t2/update! :model/Table table-id {:active true})
+ (check! true :yes)))))))
+
(deftest permissions-for-user-test
(mt/with-temp [:model/PermissionsGroup {group-id-1 :id} {}
:model/PermissionsGroup {group-id-2 :id} {}
@@ -496,21 +575,6 @@
:db-id database-id-1
:perm-type :perms/view-data)))))))
-(deftest most-restrictive-per-group-works
- (is (= #{:query-builder-and-native}
- (#'data-perms/most-restrictive-per-group :perms/create-queries [{:group-id 1 :value :query-builder-and-native}])))
- (is (= #{:no}
- (#'data-perms/most-restrictive-per-group :perms/create-queries [{:group-id 1 :value :query-builder}
- {:group-id 1 :value :no}])))
- (is (= #{:no :query-builder-and-native}
- (#'data-perms/most-restrictive-per-group :perms/create-queries [{:group-id 1 :value :query-builder-and-native}
- {:group-id 1 :value :no}
- {:group-id 2 :value :query-builder-and-native}])))
- (is (= #{:no}
- (#'data-perms/most-restrictive-per-group :perms/create-queries [{:group-id 1 :value :no}
- {:group-id 1 :value :query-builder}
- {:group-id 1 :value :query-builder-and-native}]))))
-
(deftest full-schema-permission-for-user-test
(mt/with-temp [:model/PermissionsGroup {group-id-1 :id} {}
:model/User {user-id-1 :id} {}
diff --git a/test/metabase/util/memoize_test.cljc b/test/metabase/util/memoize_test.cljc
index 7dbf6044e1b..cdd06ecd76e 100644
--- a/test/metabase/util/memoize_test.cljc
+++ b/test/metabase/util/memoize_test.cljc
@@ -70,7 +70,22 @@
(testing "the zero-arg arity interns values as-is"
(let [intern (memo/fast-interner)]
(is (= :kw (intern :kw)))
- (is (identical? (intern :kw) (intern :kw))))))
+ (is (identical? (intern :kw) (intern :kw)))))
+ (testing "nil is returned unchanged without calling the canonicalizer"
+ (let [calls (atom 0)
+ intern (memo/fast-interner (fn [x]
+ (swap! calls inc)
+ x))]
+ (is (nil? (intern nil)))
+ (is (zero? @calls))))
+ (testing "a cached false counts as a hit, not a miss"
+ (let [calls (atom 0)
+ intern (memo/fast-interner (fn [x]
+ (swap! calls inc)
+ x))]
+ (is (false? (intern false)))
+ (is (false? (intern false)))
+ (is (= 1 @calls)))))
(deftest ^:parallel bounded-evicts-when-keyspace-overflows-test
(let [keyspace (gen-keyspace 100)
diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.clj
index 780d3d2d58d..b767f95fcdd 100644
--- a/test/metabase/warehouses_rest/api_test.clj
+++ b/test/metabase/warehouses_rest/api_test.clj
@@ -1242,6 +1242,38 @@
(is (not-any? :is_saved_questions
(:data (mt/user-http-request :lucky :get 200 "database?saved=true")))))))))
+(deftest databases-list-saved-questions-call-count-test
+ (testing "GET /api/database?saved=true&include=tables app-DB call count should not scale with the number of Cards"
+ (mt/with-model-cleanup [:model/Card]
+ (letfn [(insert-cards! [n]
+ (dotimes [_ n]
+ (t2/insert! :model/Card
+ (assoc (card-with-native-query (mt/random-name))
+ :creator_id (mt/user->id :crowberto)
+ :display :table
+ :visualization_settings {}
+ ;; a column with a real Field :id exercises the per-card Field
+ ;; fetch + hydration path
+ :result_metadata [{:id (mt/id :venues :name)
+ :name "NAME"
+ :display_name "Name"
+ :base_type :type/Text}]))))
+ (warm-call-count! []
+ ;; first request pays one-time priming; measure the second
+ (mt/user-http-request :crowberto :get 200 "database?saved=true&include=tables")
+ (t2/with-call-count [call-count]
+ (mt/user-http-request :crowberto :get 200 "database?saved=true&include=tables")
+ (call-count)))]
+ (insert-cards! 2)
+ (let [calls-with-2 (warm-call-count!)
+ _ (insert-cards! 16)
+ calls-with-18 (warm-call-count!)]
+ ;; [[mi/do-after-select]] re-runs each Card row through toucan2's identity-query, which
+ ;; `with-call-count` counts even though it never hits the DB — so growth of 1 per Card is
+ ;; expected and allowed. The slack of 6 stays well under the pre-batching behavior of one
+ ;; real `metabase_database` select per additional Card on top of that (#78919).
+ (is (<= calls-with-18 (+ calls-with-2 16 6))))))))
+
(deftest fetch-databases-with-invalid-driver-test
(testing "GET /api/database"
(testing "\nEndpoint should still work even if there is a Database saved with a invalid driver"
diff --git a/test/metabase/xrays/automagic_dashboards/core_test.clj b/test/metabase/xrays/automagic_dashboards/core_test.clj
index 2d2d2d24efd..fdee0f28ea8 100644
--- a/test/metabase/xrays/automagic_dashboards/core_test.clj
+++ b/test/metabase/xrays/automagic_dashboards/core_test.clj
@@ -1103,7 +1103,7 @@
(let [database (t2/select-one :model/Database :id db-id)]
(t2/with-call-count [call-count]
(magic/candidate-tables database)
- (is (= 2 (call-count)))))))))
+ (is (<= (call-count) 2))))))))
(deftest ^:parallel empty-table-test
(testing "candidate-tables should work with an empty Table (no Fields)"