The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
fix n+1s in db, table, group creation (#72087)
John Swanson6c65769bc518
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex e5799db8778..24f735a8024 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1445,7 +1445,6 @@{:team "UX West":api #{metabase.permissions-rest.api}:uses #{api- app-dbaudit-appeventspermissionsdiff --git a/enterprise/backend/src/metabase_enterprise/advanced_permissions/common.clj b/enterprise/backend/src/metabase_enterprise/advanced_permissions/common.cljindex 8f69e8fcc6b..0174a6d6a4c 100644--- a/enterprise/backend/src/metabase_enterprise/advanced_permissions/common.clj+++ b/enterprise/backend/src/metabase_enterprise/advanced_permissions/common.clj@@ -163,75 +163,75 @@[]dbs)))-(defenterprise new-database-view-data-permission-level- "Returns the default view-data permission level for a new database for a given group. This is `blocked` if the- group has block permissions for any existing database, or if any connection impersonation policies or sandboxes- exist. Otherwise, it is `unrestricted`."+(defenterprise new-group-view-data-permission-levels+ "Returns a map of {db-id → permission-level} for multiple databases.":feature :advanced-permissions- [group-id]- (if (or- (t2/exists? :model/DataPermissions- :perm_type :perms/view-data- :perm_value :blocked- :group_id group-id)- (t2/exists? :model/ConnectionImpersonation- :group_id group-id)- (and- (premium-features/enable-sandboxes?)- (t2/exists? :model/Sandbox- :group_id group-id)))- :blocked- :unrestricted))--(defenterprise new-table-view-data-permission-level- "Returns the view-data permission level to set for a new table in a given group and database. This is `blocked`- if the group has `blocked` for the database or any table in the DB, if any connection impersonation policies or- sandboxes exist for the database and group. otherwise it is `unrestricted`."+ [db-ids]+ (if (empty? db-ids)+ {}+ (let [all-users-group-id (u/the-id (perms/all-users-group))+ blocked-db-ids (t2/select-fn-set :db_id :model/DataPermissions+ :perm_type :perms/view-data+ :perm_value :blocked+ :group_id all-users-group-id+ :db_id [:in db-ids])+ impersonation-db-ids (t2/select-fn-set :db_id :model/ConnectionImpersonation+ :group_id all-users-group-id+ :db_id [:in db-ids])+ sandbox-db-ids (when (premium-features/enable-sandboxes?)+ (into #{}+ (map :db_id)+ (t2/query {:select [[:t.db_id :db_id]]+ :from [[(t2/table-name :model/Sandbox) :s]]+ :join [[(t2/table-name :model/Table) :t] [:= :s.table_id :t.id]]+ :where [:and+ [:= :s.group_id all-users-group-id]+ [:in :t.db_id db-ids]]})))+ blocked-dbs (into (or blocked-db-ids #{})+ (concat impersonation-db-ids sandbox-db-ids))]+ (zipmap db-ids (map #(if (blocked-dbs %) :blocked :unrestricted) db-ids)))))++(defenterprise new-database-view-data-permission-levels+ "Returns a map of {group-id → permission-level} for multiple groups.":feature :advanced-permissions- [db-id group-id]- ;; We don't check for connection impersonations here, because impersonations are set at the DB-level, so a new table- ;; should get `:unrestricted` permissions and then inherit the DB-level impersonation policy.- (if (or- (t2/exists? :model/DataPermissions- :db_id db-id- :perm_type :perms/view-data- :perm_value :blocked- :group_id group-id)- (and- (premium-features/enable-sandboxes?)- (t2/exists?- :model/Sandbox- {:select [:s.id]- :from [[(t2/table-name :model/Sandbox) :s]]- :join [[(t2/table-name :model/Table) :t] [:= :t.id :s.table_id]]- :where [:and- [:= :s.group_id group-id]- [:= :t.db_id db-id]]})))- :blocked- :unrestricted))--(defenterprise new-group-view-data-permission-level- "Returns the default view-data permission level for a new group for a given database. This is `blocked` if All Users- has block permissions for the database, or if any connection impersonation policies or sandboxes exist. Otherwise, it- is `unrestricted`."+ [group-ids]+ (if (empty? group-ids)+ {}+ (let [blocked-group-ids (t2/select-fn-set :group_id :model/DataPermissions+ :perm_type :perms/view-data+ :perm_value :blocked+ :group_id [:in group-ids])+ impersonation-group-ids (t2/select-fn-set :group_id :model/ConnectionImpersonation+ :group_id [:in group-ids])+ sandbox-group-ids (when (premium-features/enable-sandboxes?)+ (t2/select-fn-set :group_id :model/Sandbox+ :group_id [:in group-ids]))+ blocked-groups (into (or blocked-group-ids #{})+ (concat impersonation-group-ids sandbox-group-ids))]+ (zipmap group-ids (map #(if (blocked-groups %) :blocked :unrestricted) group-ids)))))++(defenterprise new-table-view-data-permission-levels+ "Returns a map of {group-id → permission-level} for multiple groups and a single DB.":feature :advanced-permissions- [db-id]- (let [all-users-group-id (u/the-id (perms/all-users-group))]- (if (or- (t2/exists? :model/DataPermissions- :perm_type :perms/view-data- :perm_value :blocked- :db_id db-id- :group_id all-users-group-id)- (t2/exists? :model/ConnectionImpersonation- :group_id all-users-group-id- :db_id db-id)- (and- (premium-features/enable-sandboxes?)- (t2/exists? :model/Sandbox- :group_id all-users-group-id- {:from [[:sandboxes :s]]- :join [[:metabase_table :t] [:= :s.table_id :t.id]]- :where [:= :t.db_id db-id]})))- :blocked- :unrestricted)))+ [db-id group-ids]+ (if (empty? group-ids)+ {}+ ;; We don't check for connection impersonations here, because impersonations are set at the+ ;; DB-level, so a new table should get `:unrestricted` and inherit the DB-level impersonation policy.+ (let [blocked-group-ids (t2/select-fn-set :group_id :model/DataPermissions+ :db_id db-id+ :perm_type :perms/view-data+ :perm_value :blocked+ :group_id [:in group-ids])+ sandbox-group-ids (when (premium-features/enable-sandboxes?)+ (into #{}+ (map :group_id)+ (t2/query {:select [[:s.group_id :group_id]]+ :from [[(t2/table-name :model/Sandbox) :s]]+ :join [[(t2/table-name :model/Table) :t] [:= :t.id :s.table_id]]+ :where [:and+ [:in :s.group_id group-ids]+ [:= :t.db_id db-id]]})))+ blocked-groups (into (or blocked-group-ids #{})+ sandbox-group-ids)]+ (zipmap group-ids (map #(if (blocked-groups %) :blocked :unrestricted) group-ids)))))diff --git a/enterprise/backend/test/metabase_enterprise/advanced_permissions/common_test.clj b/enterprise/backend/test/metabase_enterprise/advanced_permissions/common_test.cljindex cd15aba36ef..fee20a1ce18 100644--- a/enterprise/backend/test/metabase_enterprise/advanced_permissions/common_test.clj+++ b/enterprise/backend/test/metabase_enterprise/advanced_permissions/common_test.clj@@ -77,7 +77,7 @@:can_create_native_queries false}(user-permissions :rasta))))))))))-(deftest new-database-view-data-permission-level-test+(deftest new-database-view-data-permission-levels-test(mt/with-additional-premium-features #{:sandboxes :advanced-permissions}(mt/with-temp [:model/Database {db-id :id} {}:model/PermissionsGroup {group-id :id} {}]@@ -110,7 +110,7 @@:model/Database {db-id-2 :id} {}](is (= :blocked (perm-value db-id-2)))))))))-(deftest new-table-view-data-permission-level-test+(deftest new-table-view-data-permission-levels-test(mt/with-additional-premium-features #{:sandboxes :advanced-permissions}(mt/with-temp [:model/PermissionsGroup {group-id :id} {}:model/Database {db-id :id} {}@@ -139,24 +139,24 @@(is (= :unrestricted (perm-value table-id-1)))(is (= :blocked (perm-value table-id-3)))))))))-(deftest new-group-view-data-permission-level+(deftest new-group-view-data-permission-levels-test(mt/with-additional-premium-features #{:sandboxes :advanced-permissions}(mt/with-temp [:model/Database {db-id :id} {}](let [all-users-group-id (u/the-id (perms-group/all-users))](testing "A new group defaults to `:unrestricted` for a DB if All Users has `:unrestricted`"(data-perms/set-database-permission! all-users-group-id db-id :perms/view-data :unrestricted)- (is (= :unrestricted (advanced-permissions.common/new-group-view-data-permission-level db-id))))+ (is (= {db-id :unrestricted} (advanced-permissions.common/new-group-view-data-permission-levels [db-id]))))(testing "A new group defaults to `:blocked` for a DB if All Users has `:blocked`"(data-perms/set-database-permission! all-users-group-id db-id :perms/view-data :blocked)- (is (= :blocked (advanced-permissions.common/new-group-view-data-permission-level db-id))))+ (is (= {db-id :blocked} (advanced-permissions.common/new-group-view-data-permission-levels [db-id]))))(testing "A new group defaults to `:blocked` if All Users has any connection impersonation"(data-perms/set-database-permission! all-users-group-id db-id :perms/view-data :unrestricted)(advanced-perms.api.tu/with-impersonations! {:impersonations [{:db-id db-id:attribute "impersonation_attr":attributes {"impersonation_attr" "impersonation_role"}}]}- (is (= :blocked (advanced-permissions.common/new-group-view-data-permission-level db-id)))))+ (is (= {db-id :blocked} (advanced-permissions.common/new-group-view-data-permission-levels [db-id])))))(testing "A new database defaults to `:blocked` if All Users group has any sandbox"(data-perms/set-database-permission! all-users-group-id db-id :perms/view-data :unrestricted)@@ -166,7 +166,7 @@:group_id all-users-group-id:card_id card-id:attribute_remappings {"foo" 1}}]- (is (= :blocked (advanced-permissions.common/new-group-view-data-permission-level db-id)))))))))+ (is (= {db-id :blocked} (advanced-permissions.common/new-group-view-data-permission-levels [db-id])))))))));;; +----------------------------------------------------------------------------------------------------------------+;;; | Data model permission enforcement |diff --git a/src/metabase/app_db/cluster_lock.clj b/src/metabase/app_db/cluster_lock.cljindex fd9c8f9fda0..0ae5295bc07 100644--- a/src/metabase/app_db/cluster_lock.clj+++ b/src/metabase/app_db/cluster_lock.clj@@ -1,5 +1,21 @@(ns metabase.app-db.cluster-lock- "Utility for taking a cluster wide lock using the application database"+ "Utility for taking a cluster wide lock using the application database.++ Supports two modes:+ - `:exclusive` (default) — row-level `FOR UPDATE`. Two exclusive holders of the+ same lock-name serialize.+ - `:share` — row-level `FOR SHARE`. Multiple shared holders of the same lock-name+ can proceed in parallel, but any shared holder blocks an exclusive acquirer+ and vice versa.++ This lets callers build intent-lock patterns (shared on a root + exclusive on+ a leaf) — see [[metabase.permissions.models.data-permissions/with-db-scoped-permissions-lock]]+ for an example.++ Lock ordering rule: if you acquire multiple cluster locks, always acquire them+ in a consistent order across call sites to avoid deadlock. The multi-lock arity+ of [[do-with-cluster-lock]] acquires them in the order given; prefer that over+ manually nesting `with-cluster-lock` forms."(:require[clojure.string :as str][metabase.app-db.connection :as mdb.connection]@@ -10,7 +26,10 @@[metabase.util.retry :as retry][toucan2.core :as t2])(:import- (java.sql Connection PreparedStatement SQLIntegrityConstraintViolationException)))+ (java.sql Connection PreparedStatement SQLIntegrityConstraintViolationException)+ (java.util.concurrent ConcurrentHashMap)+ (java.util.concurrent.locks Lock ReentrantReadWriteLock)+ (java.util.function Function)))(set! *warn-on-reflection* true)@@ -32,13 +51,32 @@:delay-ms 1000 ;; Constant delay between retries.:retry-if (fn [_ e] (retryable? e))})+;; MySQL 8.0+ supports `SELECT ... FOR SHARE`, but MariaDB (all versions as of+;; writing) only understands the older `LOCK IN SHARE MODE` syntax.+;; `LOCK IN SHARE MODE` works on all supported MySQL-family versions including+;; MySQL 8, so we use it uniformly for `:mysql`.+(defn- lock-clause+ "Returns the trailing locking clause to append to the base SELECT."+ [mode]+ (case mode+ :exclusive " FOR UPDATE"+ :share (if (= (mdb.connection/db-type) :mysql)+ " LOCK IN SHARE MODE"+ " FOR SHARE")))++(def ^:private base-lock-sql+ (delay+ (first (mdb.query/compile {:select [:lock.lock_name]+ :from [[:metabase_cluster_lock :lock]]+ :where [:= :lock.lock_name [:raw "?"]]}))))++(defn- lock-sql ^String [mode]+ (str @base-lock-sql (lock-clause mode)))+(defn- prepare-statement- "Create a prepared statement to query cache"- ^PreparedStatement [^Connection conn lock-name-str timeout]- (let [stmt (.prepareStatement conn ^String (first (mdb.query/compile {:select [:lock.lock_name]- :from [[:metabase_cluster_lock :lock]]- :where [:= :lock.lock_name [:raw "?"]]- :for :update})))]+ "Create a prepared statement to acquire a lock row in the given mode."+ ^PreparedStatement [^Connection conn lock-name-str timeout mode]+ (let [stmt (.prepareStatement conn (lock-sql mode))](try(doto stmt(.setQueryTimeout timeout)@@ -48,52 +86,167 @@(.close stmt)(throw e)))))-(defn- do-with-cluster-lock*- [lock-name-str timeout-seconds thunk]+(defn- acquire-lock-row!+ [^Connection conn lock-name-str timeout mode]+ (with-open [stmt (prepare-statement conn lock-name-str timeout mode)+ result-set (.executeQuery stmt)]+ (when-not (.next result-set)+ ;; this record will not be visible until the tx commits, so there's no need to lock it+ ;; we instead rely on concurrent threads having constraint violation trying to insert their own record+ (t2/query-one {:insert-into [:metabase_cluster_lock]+ :columns [:lock_name]+ :values [[lock-name-str]]})))+ (log/debugf "Obtained cluster lock: %s (%s)" lock-name-str mode))++(defn- do-with-cluster-locks*+ "Acquire all `locks` (each a `{:lock-name-str, :mode}` map) inside a single+ transaction, then run `thunk`."+ [locks timeout-seconds thunk](t2/with-transaction [conn]- (with-open [stmt (prepare-statement conn lock-name-str timeout-seconds)- result-set (.executeQuery stmt)]- (when-not (.next result-set)- ;; this record will not be visible until the tx commits, so there's no need to lock it- ;; we instead rely on concurrent threads having constraint violation trying to insert their own record- (t2/query-one {:insert-into [:metabase_cluster_lock]- :columns [:lock_name]- :values [[lock-name-str]]})))- (log/debugf "Obtained cluster lock: %s" lock-name-str)+ (doseq [{:keys [lock-name-str mode]} locks]+ (acquire-lock-row! conn lock-name-str timeout-seconds mode))(thunk)))+;; ---------- h2 in-process rw locks ----------+;;+;; h2 does not respect query timeout when taking SELECT ... FOR UPDATE locks, and+;; Metabase does not support multi-instance h2 deployments. So for h2 we take an+;; in-process `ReentrantReadWriteLock` keyed by lock-name. Shared mode → read lock,+;; exclusive mode → write lock. Still supports reentrancy within a single thread.++(defonce ^:private ^ConcurrentHashMap h2-locks (ConcurrentHashMap.))++(defn- h2-rw-lock ^ReentrantReadWriteLock [lock-name-str]+ (.computeIfAbsent h2-locks lock-name-str+ (reify Function (apply [_ _] (ReentrantReadWriteLock.)))))++(defn- do-with-h2-cluster-locks*+ [locks thunk]+ (let [^java.util.List held (java.util.ArrayList.)]+ (try+ (doseq [{:keys [lock-name-str mode]} locks]+ (let [rw (h2-rw-lock lock-name-str)+ ^Lock lock (if (= mode :share) (.readLock rw) (.writeLock rw))]+ (.lock lock)+ (.add held lock)+ (log/debugf "Obtained h2 cluster lock: %s (%s)" lock-name-str mode)))+ (thunk)+ (finally+ (doseq [^Lock lock (reverse held)]+ (.unlock lock))))))++;; ---------- public API ----------++(defn- keyword->lock-name-str [kw]+ (str (namespace kw) "/" (name kw)))++(defn- normalize-lock-spec+ "Turn an element of `:locks` into a `{:lock-name-str, :mode}` map. Each element+ may be a bare keyword (→ exclusive) or a `{:lock, :mode}` map."+ [spec]+ (cond+ (keyword? spec)+ {:lock-name-str (keyword->lock-name-str spec) :mode :exclusive}++ (map? spec)+ (let [{:keys [lock mode] :or {mode :exclusive}} spec]+ (when-not (keyword? lock)+ (throw (ex-info "Cluster-lock spec map must have a :lock keyword" {:spec spec})))+ {:lock-name-str (keyword->lock-name-str lock) :mode mode})++ :else+ (throw (ex-info "Invalid cluster-lock spec" {:spec spec}))))++(defn- parse-opts+ "Turn user-supplied `opts` into `{:locks, :timeout-seconds, :retry-config}`."+ [opts]+ (cond+ (keyword? opts)+ {:locks [(normalize-lock-spec opts)]}++ (map? opts)+ (let [{:keys [lock locks timeout-seconds retry-config]} opts]+ (when (and lock locks)+ (throw (ex-info "Cluster-lock opts must specify exactly one of :lock or :locks"+ {:opts opts})))+ (when-not (or lock locks)+ (throw (ex-info "Cluster-lock opts must specify :lock or :locks" {:opts opts})))+ (cond-> {:locks (if lock+ [(normalize-lock-spec (if (map? lock)+ lock+ {:lock lock :mode (or (:mode opts) :exclusive)}))]+ (mapv normalize-lock-spec locks))}+ timeout-seconds (assoc :timeout-seconds timeout-seconds)+ retry-config (assoc :retry-config retry-config)))++ :else+ (throw (ex-info "Invalid cluster-lock opts" {:opts opts}))))+(mu/defn do-with-cluster-lock"Impl for `with-cluster-lock`.- Call `thunk` after first synchronizing with the metabase cluster by taking a lock in the appdb."- [opts :- [:or :keyword+ Call `thunk` after first synchronizing with the metabase cluster by taking one+ or more locks in the appdb. `opts` can be:++ - a keyword `lock-name` — shorthand for exclusive lock on that name with default+ timeout and retry config.+ - a map with `:lock` (a keyword) or `:locks` (a seq of specs), plus optional+ `:mode`, `:timeout-seconds`, and `:retry-config`:++ {:lock ::foo} ; exclusive on ::foo+ {:lock ::foo :mode :share} ; shared on ::foo+ {:lock ::foo :timeout-seconds 5} ; with timeout override+ {:locks [::foo ::bar]} ; two exclusive locks+ {:locks [{:lock ::root :mode :share} ; intent-lock pattern:+ {:lock ::leaf :mode :exclusive}] ; shared root + exclusive+ :timeout-seconds 5} ; leaf, with timeout++ In the `:locks` form, each element is either a bare keyword (exclusive) or a+ map `{:lock, :mode}`. Per-element timeout/retry config is not supported —+ those live on the top-level opts map. All locks are acquired in order inside+ a single transaction."+ [opts :- [:or+ :keyword[:map- [:lock-name :keyword]+ [:lock {:optional true} :keyword]+ [:locks {:optional true} [:sequential+ [:or :keyword+ [:map+ [:lock :keyword]+ [:mode {:optional true} [:enum :exclusive :share]]]]]]+ [:mode {:optional true} [:enum :exclusive :share]][:timeout-seconds {:optional true} :int][:retry-config {:optional true} [:ref ::retry/retry-overrides]]]]thunk :- ifn?]- (cond- ;; h2 does not respect the query timeout when taking the lock- ;; we do not support multiple instances for h2 however, so an in-process lock is sufficient.- (= (mdb.connection/db-type) :h2) (locking do-with-cluster-lock (thunk))- (keyword? opts) (do-with-cluster-lock {:lock-name opts} thunk)- :else (let [{:keys [timeout-seconds retry-config lock-name] :or {timeout-seconds cluster-lock-timeout-seconds}} opts- lock-name-str (str (namespace lock-name) "/" (name lock-name))- config (merge default-retry-config retry-config)]- (try- (retry/with-retry config- (do-with-cluster-lock* lock-name-str timeout-seconds thunk))- (catch Throwable e- (if (retryable? e)- (throw (ex-info (str "Failed to obtain cluster lock: " lock-name-str)- {:lock-name lock-name- :retries (:max-retries config)}- e))- (throw e)))))))+ (let [{:keys [locks timeout-seconds retry-config]+ :or {timeout-seconds cluster-lock-timeout-seconds}} (parse-opts opts)]+ (cond+ ;; h2 does not respect the query timeout when taking the lock and is not cross-process,+ ;; so we fall back to an in-process ReentrantReadWriteLock per lock name.+ (= (mdb.connection/db-type) :h2)+ (do-with-h2-cluster-locks* locks thunk)++ :else+ (let [config (merge default-retry-config retry-config)]+ (try+ (retry/with-retry config+ (do-with-cluster-locks* locks timeout-seconds thunk))+ (catch Throwable e+ (if (retryable? e)+ (throw (ex-info (str "Failed to obtain cluster lock: "+ (str/join ", " (map :lock-name-str locks)))+ {:lock-names (mapv :lock-name-str locks)+ :retries (:max-retries config)}+ e))+ (throw e))))))))(defmacro with-cluster-lock"Run `body` in a transaction that tries to take a lock from the metabase_cluster_lock table of- the specified name to coordinate concurrency with other metabase instances sharing the appdb."+ the specified name to coordinate concurrency with other metabase instances sharing the appdb.++ `lock-options` may be a lock-name keyword, an options map+ `{:lock-name, :mode, :timeout-seconds, :retry-config}`, or a vector of such specs+ (all acquired in order inside one transaction)."([lock-options & body]`(do-with-cluster-lock ~lock-options (fn [] ~@body))))diff --git a/src/metabase/permissions/core.clj b/src/metabase/permissions/core.cljindex a763ebfef7a..8bddae9bf97 100644--- a/src/metabase/permissions/core.clj+++ b/src/metabase/permissions/core.clj@@ -52,9 +52,6 @@sandboxes-for-userschema-permission-for-userset-database-permission!- set-external-group-permissions!- set-new-database-permissions!- set-new-table-permissions!set-table-permission!set-table-permissions!table-permission-for-user@@ -66,7 +63,12 @@with-additional-table-permissionwith-relevant-permissions-for-userhas-any-transforms-permission?- has-db-transforms-permission?]+ has-db-transforms-permission?+ set-default-group-permissions!+ set-default-database-permissions!+ set-default-table-permissions!+ with-global-permissions-lock+ with-db-scoped-permissions-lock][metabase.permissions.models.data-permissions.sqlUserInfoPermissionMappingdiff --git a/src/metabase/permissions/models/data_permissions.clj b/src/metabase/permissions/models/data_permissions.cljindex 030e47a8c8f..2eea1a4e8f0 100644--- a/src/metabase/permissions/models/data_permissions.clj+++ b/src/metabase/permissions/models/data_permissions.clj@@ -28,15 +28,68 @@(methodical/defmethod t2/table-name :model/DataPermissions [_model] :data_permissions)+(def ^:dynamic ^:private *skip-cluster-locks*+ "When true, skip per-(db-id, perm-type) cluster locks. Should only be bound to true+ when a coarser lock is already held by the calling code."+ false)++;; Permission mutation locks form a two-level intent-lock hierarchy over the+;; `metabase_cluster_lock` table:+;;+;; root = ::batch-permissions-update+;; leaf = ::batch-permissions-update-db-<db-id> (one per DB)+;;+;; - `with-global-permissions-lock` takes the root in :exclusive mode.+;; - `with-db-scoped-permissions-lock` takes the root in :share mode + the leaf+;; for its db-id in :exclusive mode.+;;+;; That gives us:+;; - parallel DB-scoped writers for different DBs → no contention+;; - DB-scoped writers for the same DB → serialize on the leaf+;; - global writer vs any DB-scoped writer → mutually exclusive+;; - two global writers → serialize on the root++(defn db-scoped-leaf-lock-name+ "Returns the cluster-lock keyword for the per-db leaf used by+ [[with-db-scoped-permissions-lock]]. Exposed so that macro expansions in+ other namespaces can reference it."+ [db-id]+ (keyword "metabase.permissions.models.data-permissions"+ (str "batch-permissions-update-db-" db-id)))++(defmacro with-global-permissions-lock+ "Acquires an exclusive cluster-wide lock over all permission mutations. Use for+ operations that touch multiple DBs' permission rows (graph update, new group+ creation). Blocks and is blocked by every `with-db-scoped-permissions-lock`."+ [& body]+ `(cluster-lock/with-cluster-lock ::batch-permissions-update+ (binding [*skip-cluster-locks* true]+ ~@body)))++(defmacro with-db-scoped-permissions-lock+ "Acquires a shared lock on the permissions root + an exclusive lock on the+ per-db leaf for `db-id`. Use for operations that only touch one DB's+ permission rows (new table, new DB). Parallel calls for different DBs run+ concurrently; parallel calls for the same DB serialize on the leaf row."+ [db-id & body]+ `(let [db-id# ~db-id]+ (cluster-lock/with-cluster-lock+ {:locks [{:lock ::batch-permissions-update :mode :share}+ {:lock (db-scoped-leaf-lock-name db-id#) :mode :exclusive}]}+ (binding [*skip-cluster-locks* true]+ ~@body))))+(mu/defn- with-cluster-lock-fn[m :- [:map[:db-id ms/PositiveInt][:perm-type :string]]f :- fn?]- (cluster-lock/with-cluster-lock (keyword "data-permissions-" (str/join "-"- [(:db-id m)- (:perm-type m)]))- (f)))+ (if *skip-cluster-locks*+ (f)+ (cluster-lock/with-cluster-lock (keyword "data-permissions-" (str/join "-"+ [(:db-id m)+ (:perm-type m)]))+ (f))))(defmacro with-cluster-lock"Takes a map with `db-id` and `perm-type`, obtains a cluster lock for that combo, and executes the body"@@ -704,92 +757,17 @@(when (seq to-insert)(batch-insert-permissions! to-insert)))))-(defn- lowest-permission-level-in-any-database- "Given a group and a permission type, returns the lowest permission level for that group in any database, at the DB or table-level.- This is used to determine the default permission level for the group when a new database is added."- [group-id perm-type]- (let [lowest-to-highest-values (-> permissions.schema/data-permissions perm-type :values reverse)]- (first (filter- (fn [value]- (t2/exists? :model/DataPermissions- :perm_type perm-type- :perm_value value- :group_id group-id))- lowest-to-highest-values))))--(defenterprise new-group-view-data-permission-level- "Returns the default view-data permission level for a new group for a given database. On OSS, this is always `unrestricted`."+(defenterprise new-group-view-data-permission-levels+ "Returns a map of {db-id → permission-level} for multiple databases. On OSS, all are `:unrestricted`."metabase-enterprise.advanced-permissions.common- [_group-id]- :unrestricted)--(defn- new-group-permissions- "Returns a map of {perm-type value} to be set for a new group, for the provided database."- [db-or-id all-users-group-id]- (let [db-id (u/the-id db-or-id)- view-data-level (new-group-view-data-permission-level db-id)- create-queries-level (or (->> (t2/select-fn-set :value- [:model/DataPermissions [:perm_value :value]]- :perm_type :perms/create-queries- :db_id db-id- :group_id all-users-group-id)- (coalesce-most-restrictive :perms/create-queries))- :query-builder-and-native)- download-level (or (->> (t2/select-fn-set :value- [:model/DataPermissions [:perm_value :value]]- :perm_type :perms/download-results- :db_id db-id- :group_id all-users-group-id)- (coalesce-most-restrictive :perms/download-results))- :one-million-rows)]- {:perms/view-data view-data-level- :perms/create-queries create-queries-level- :perms/download-results download-level- :perms/manage-table-metadata :no- :perms/manage-database :no}))--(defn set-external-group-permissions!- "Sets the appropriate data permissions for a new external group or database - always the minimum possible data permissions."- [group-or-id db-id]- (doseq [[perm-type perm-value] (m/map-vals (fn [{:keys [values]}] (last values)) permissions.schema/data-permissions)]- (set-database-permission! group-or-id db-id perm-type perm-value)))--(defn set-new-group-permissions!- "Sets permissions for a newly-added group to their appropriate values for a single database. This is generally based- on the permissions of the All Users group."- [group-or-id db-or-id all-users-group-id]- (doseq [[perm-type perm-value] (new-group-permissions db-or-id all-users-group-id)]- (set-database-permission! group-or-id db-or-id perm-type perm-value)))--(defenterprise new-database-view-data-permission-level- "Returns the default view-data permission level for a new database for a given group. On OSS, this is always `unrestricted`."+ [db-ids]+ (zipmap db-ids (repeat :unrestricted)))++(defenterprise new-database-view-data-permission-levels+ "Returns a map of {group-id → permission-level} for multiple groups. On OSS, all are `:unrestricted`."metabase-enterprise.advanced-permissions.common- [_group-id]- :unrestricted)--(defn- new-database-permissions- "Returns a map of {perm-type value} to be set for a new database, for the provided group."- [group-or-id]- (let [group-id (u/the-id group-or-id)- view-data-level (new-database-view-data-permission-level group-id)- create-queries-level (or (lowest-permission-level-in-any-database group-id :perms/create-queries)- :query-builder-and-native)- download-level (if (= view-data-level :blocked)- :no- (or (lowest-permission-level-in-any-database group-id :perms/download-results)- :one-million-rows))]- {:perms/view-data view-data-level- :perms/create-queries create-queries-level- :perms/download-results download-level- :perms/manage-table-metadata :no- :perms/manage-database :no}))--(defn set-new-database-permissions!- "Sets permissions for a newly-added database to their appropriate values for a single group. For certain permission- types, the value computed based on the existing permissions the group has for other databases."- [group-or-id db-or-id]- (doseq [[perm-type perm-value] (new-database-permissions group-or-id)]- (set-database-permission! group-or-id db-or-id perm-type perm-value)))+ [group-ids]+ (zipmap group-ids (repeat :unrestricted)))(defn- build-new-table-perms"Builds new permission entries for the given table permissions."@@ -974,102 +952,250 @@value :- :keyword](set-table-permissions! group-or-id perm-type {table-or-id value}))-(defn- schema-permission-value- "Infers the permission value for a new table based on existing permissions in the schema. Returns a permission value- if every table in the schema has the same value, otherwise returns nil."- [db-id group-id schema-name perm-type]- (let [possible-values (:values (get permissions.schema/data-permissions perm-type))- schema-perms-check (mapv (fn [value]- (t2/exists? :model/DataPermissions- :perm_type (u/qualified-name perm-type)- :db_id db-id- :group_id group-id- :schema_name schema-name- :perm_value value))- possible-values)- single-perm-val? (= (count (filter true? schema-perms-check)) 1)]- (when single-perm-val?- (nth possible-values (.indexOf ^PersistentVector schema-perms-check true)))))--(defenterprise new-table-view-data-permission-level- "Returns the view-data permission level to set for a new table in a given group and database. On OSS, this is always- `unrestricted`."+(defenterprise new-table-view-data-permission-levels+ "Returns a map of {group-id → permission-level} for multiple groups and a single DB.+ On OSS, all are `:unrestricted`."metabase-enterprise.advanced-permissions.common- [_db-id _group-id]- :unrestricted)--(mu/defn set-new-table-permissions!- "Sets permissions for a single table all the provided groups, based on the following rules:- - :view-data is set to :blocked if any other tables in the DB are :blocked or sandboxed- - If all existing tables in the schema have the same permission value, the new table is set to match them.- - If permissions are set at the DB-level, no table permission is inserted.- - Otherwise we use the provided `default-value`."- [groups-or-ids :- [:sequential TheIdable]- table-or-id :- TheIdable- perm-type :- ::permissions.schema/data-permission-type- default-value :- :keyword]- (when (not= :model/Table (model-by-perm-type perm-type))- (throw (ex-info (tru "Permission type {0} cannot be set on tables." perm-type)- {perm-type (permissions.schema/data-permissions perm-type)})))- (when (seq groups-or-ids)- (let [table (if (map? table-or-id)- table-or-id- (t2/select-one [:model/Table :id :db_id :schema] :id table-or-id))- db-id (:db_id table)- group-ids (map u/the-id groups-or-ids)]- (with-cluster-lock {:db-id db-id :perm-type (u/qualified-name perm-type)}- (let [schema-name (:schema table)- db-level-perms (t2/select :model/DataPermissions- {:where- [:and- [:= :db_id db-id]- [:= :table_id nil]- [:= :perm_type (u/qualified-name perm-type)]- [:in :group_id group-ids]]})- db-level-group-ids (set (map :group_id db-level-perms))- new-perms (reduce- (fn [new-perms group-id]- (let [new-value (or- ;; Make sure we set `blocked` data access if we're on EE and *any*- ;; other table in the DB has `blocked` or `sandboxed`- (and (= perm-type :perms/view-data)- (new-table-view-data-permission-level db-id group-id))- ;; Otherwise, if all tables in the schema have the same- ;; value, use that value for the new table- (schema-permission-value db-id group-id schema-name perm-type)- ;; Otherwise, use the default value passed in- default-value)- new-perm {:perm_type perm-type- :group_id group-id- :perm_value new-value- :db_id db-id- :table_id (u/the-id table)- :schema_name schema-name}]- (cond- ;; Perms that are being added at the table-level for a group currently set at the DB- ;; level. This should only happen when adding a table to a DB where some existing- ;; tables are sandboxed, because the DB might have `:unrestricted` DB-level perms which- ;; need to be split out to table-level perms.- (and (db-level-group-ids group-id)- (= new-value :blocked))- (update new-perms :going-granular conj new-perm)-- ;; Otherwise, we only add a new table-level permission row if existing perms- ;; are table-level- (not (db-level-group-ids group-id))- (update new-perms :simple-perms conj new-perm)-- :else- new-perms)))- {:simple-perms [] :going-granular []}- group-ids)- {:keys [going-granular- simple-perms]} new-perms]- ;; These perms might need existing DB-level perms to be broken out to table-level perms- (doseq [{:keys [perm_type perm_value group_id]} going-granular]- (set-table-permissions-internal! group_id perm_type {table perm_value}))- ;; These perms can be inserted raw, and don't require changes to existing perms in the DB- (t2/insert! :model/DataPermissions simple-perms))))))+ [_db-id group-ids]+ (zipmap group-ids (repeat :unrestricted)))++;;; ---------------------------------------- Bulk permission functions ------------------------------------------------+;; These functions set permissions for newly-created entities (groups, databases, tables) using batch SQL operations+;; instead of per-row mutations. They are intended to be called from within a coarse cluster lock.++(defn- least-permissive-defaults+ "Returns a map of {perm-type → least-permissive-value} from the schema definition."+ []+ (m/map-vals (fn [{:keys [values]}] (last values)) permissions.schema/data-permissions))++(defn set-default-group-permissions!+ "Bulk-sets default permissions for a newly-created group across all databases.+ When `use-all-users-perms?` is true (regular groups), values are based on the All Users group's+ current permissions. When false (tenant/external groups), uses the most restrictive values.+ Uses batch SQL operations instead of per-row mutations."+ [group-or-id db-ids use-all-users-perms?]+ (when (seq db-ids)+ (let [group-id (u/the-id group-or-id)]+ (if-not use-all-users-perms?+ ;; External/tenant groups: all least-permissive values (static, no queries needed)+ (batch-insert-permissions!+ (for [db-id db-ids+ [perm-type perm-value] (least-permissive-defaults)]+ {:perm_type perm-type+ :group_id group-id+ :perm_value perm-value+ :db_id db-id}))+ ;; Regular groups: compute based on All Users group+ (let [au-id (t2/select-one-pk :model/PermissionsGroup+ :magic_group_type "all-internal-users")+ au-perms (t2/select :model/DataPermissions :group_id au-id)+ au-by-db (reduce (fn [acc {:keys [db_id perm_type perm_value]}]+ (update-in acc [db_id perm_type] (fnil conj #{}) perm_value))+ {}+ au-perms)+ view-data-levels (new-group-view-data-permission-levels db-ids)]+ (batch-insert-permissions!+ (for [db-id db-ids+ :let [view-data-level (get view-data-levels db-id :unrestricted)+ cq-values (get-in au-by-db [db-id :perms/create-queries])+ cq-level (or (when (seq cq-values) (coalesce-most-restrictive :perms/create-queries cq-values))+ :query-builder-and-native)+ dl-values (get-in au-by-db [db-id :perms/download-results])+ dl-level (or (when (seq dl-values) (coalesce-most-restrictive :perms/download-results dl-values))+ :one-million-rows)+ perm-map (cond-> {:perms/view-data view-data-level+ :perms/create-queries cq-level+ :perms/download-results dl-level+ :perms/manage-table-metadata :no+ :perms/manage-database :no}+ (or (not= view-data-level :unrestricted)+ (not= cq-level :query-builder-and-native))+ (assoc :perms/transforms :no))]+ [perm-type perm-value] perm-map]+ {:perm_type perm-type+ :group_id group-id+ :perm_value perm-value+ :db_id db-id})))))))++(defn set-default-database-permissions!+ "Bulk-sets default permissions for a newly-created database across all groups.+ For tenant groups, uses least-permissive values. For audit DBs, uses hardcoded values.+ For other groups, values are based on the group's lowest existing permission level.+ Uses batch SQL operations instead of per-row mutations."+ [database groups]+ (when (seq groups)+ (let [db-id (u/the-id database)+ is-audit (:is_audit database)+ group-ids (map u/the-id groups)+ defaults (least-permissive-defaults)+ ;; Batch-fetch distinct (group, perm-type, value) triples — we only need the set of unique values per+ ;; group to find the most restrictive level;+ all-perms (when-not is-audit+ (t2/query {:select-distinct [:group_id :perm_type :perm_value]+ :from [[(t2/table-name :model/DataPermissions)]]+ :where [:and+ [:in :group_id group-ids]+ [:in :perm_type ["perms/create-queries" "perms/download-results"]]]}))+ ;; Group by (group_id, perm_type) → set of values+ perms-by-grp (when all-perms+ (reduce (fn [acc {:keys [group_id perm_type perm_value]}]+ (update-in acc [group_id (keyword perm_type)] (fnil conj #{}) (keyword perm_value)))+ {}+ all-perms))+ ;; Batch-fetch view-data levels for all groups at once+ view-data-levels (when-not is-audit+ (new-database-view-data-permission-levels group-ids))+ perm-rows (mapcat+ (fn [group]+ (let [group-id (u/the-id group)+ perm-map+ (cond+ ;; Tenant groups always get least-permissive+ (:is_tenant_group group)+ defaults++ ;; Audit DB gets hardcoded restrictive values+ is-audit+ {:perms/view-data :unrestricted+ :perms/create-queries :no+ :perms/download-results :one-million-rows+ :perms/manage-table-metadata :no+ :perms/manage-database :no+ :perms/transforms :no}++ ;; Normal: compute based on group's lowest existing perm level+ :else+ (let [view-data-level (get view-data-levels group-id :unrestricted)+ grp-vals (get perms-by-grp group-id)+ cq-values (get grp-vals :perms/create-queries)+ cq-level (or (when (seq cq-values)+ (coalesce-most-restrictive :perms/create-queries cq-values))+ :query-builder-and-native)+ download-level (if (= view-data-level :blocked)+ :no+ (let [dl-values (get grp-vals :perms/download-results)]+ (or (when (seq dl-values)+ (coalesce-most-restrictive :perms/download-results dl-values))+ :one-million-rows)))]+ (cond-> {:perms/view-data view-data-level+ :perms/create-queries cq-level+ :perms/download-results download-level+ :perms/manage-table-metadata :no+ :perms/manage-database :no}+ (or (not= view-data-level :unrestricted)+ (not= cq-level :query-builder-and-native))+ (assoc :perms/transforms :no))))]+ (for [[perm-type perm-value] perm-map]+ {:perm_type perm-type+ :group_id group-id+ :perm_value perm-value+ :db_id db-id})))+ groups)]+ (batch-insert-permissions! perm-rows))))++(defn set-default-table-permissions!+ "Bulk-sets default permissions for a newly-created table across all relevant groups.+ Handles three cases per (group, perm-type):+ - Group has DB-level perm matching default → no-op (DB-level covers it)+ - Group has DB-level perm but new table needs :blocked → going-granular (expand to per-table rows)+ - Group has no DB-level perm (already table-granular) → simple insert for the new table++ `group-perm-defaults` is a seq of {:group-id G :perm-type PT :default-value V} triples."+ [table group-perm-defaults]+ (when (seq group-perm-defaults)+ (let [table (if (map? table)+ table+ (t2/select-one [:model/Table :id :db_id :schema] :id table))+ db-id (:db_id table)+ table-id (u/the-id table)+ schema (:schema table)+ group-ids (distinct (map :group-id group-perm-defaults))+ perm-types (distinct (map :perm-type group-perm-defaults))+ ;; Batch SELECT #1: all DB-level permissions for this DB across all relevant groups + perm-types+ db-level-perms (t2/select :model/DataPermissions+ {:where [:and+ [:= :db_id db-id]+ [:= :table_id nil]+ [:in :group_id group-ids]+ [:in :perm_type (mapv u/qualified-name perm-types)]]})+ ;; Index: {[group_id perm_type] → db-level-perm-row}+ db-level-idx (into {} (map (fn [p] [[(:group_id p) (:perm_type p)] p]) db-level-perms))+ ;; Batch SELECT #2: all existing table-level permissions for this DB+ ;; (needed for schema-permission-value logic and going-granular expansion)+ table-perms (t2/select :model/DataPermissions+ {:where [:and+ [:= :db_id db-id]+ [:not= :table_id nil]+ [:in :group_id group-ids]+ [:in :perm_type (mapv u/qualified-name perm-types)]]})+ ;; Index: {[group_id perm_type schema_name] → set of values}+ schema-vals-idx (reduce (fn [acc {:keys [group_id perm_type schema_name perm_value]}]+ (update-in acc [group_id perm_type schema_name] (fnil conj #{}) perm_value))+ {}+ table-perms)+ ;; Batch SELECT #3: all tables in this DB (for going-granular expansion)+ all-db-tables (t2/select [:model/Table :id :db_id :schema] :db_id db-id :active true)+ ;; Batch-fetch view-data levels for all groups at once+ view-data-levels (new-table-view-data-permission-levels db-id group-ids)+ ;; Classify each (group, perm-type) triple+ {:keys [simple-perms going-granular db-rows-to-delete]}+ (reduce+ (fn [acc {:keys [group-id perm-type default-value]}]+ (let [;; Enterprise hook override for view-data+ actual-value (or (when (= perm-type :perms/view-data)+ (get view-data-levels group-id))+ ;; Schema-level consistency: if all tables in schema have same value, use it+ (let [sv (get-in schema-vals-idx [group-id perm-type schema])]+ (when (and (seq sv) (= (count sv) 1))+ (first sv)))+ default-value)+ db-level-perm (get db-level-idx [group-id perm-type])+ new-perm {:perm_type perm-type+ :group_id group-id+ :perm_value actual-value+ :db_id db-id+ :table_id table-id+ :schema_name schema}]+ (cond+ ;; Group has DB-level perm and new table needs :blocked → going-granular+ (and db-level-perm (= actual-value :blocked))+ (-> acc+ (update :going-granular conj {:group-id group-id :perm-type perm-type+ :new-perm new-perm :db-perm db-level-perm})+ (update :db-rows-to-delete conj (:id db-level-perm)))++ ;; Group has no DB-level perm (already table-granular) → simple insert+ (not db-level-perm)+ (update acc :simple-perms conj new-perm)++ ;; Group has DB-level perm matching or compatible → no-op+ :else+ acc)))+ {:simple-perms [] :going-granular [] :db-rows-to-delete []}+ group-perm-defaults)]+ ;; Bulk DELETE: DB-level rows that need going-granular expansion+ (when (seq db-rows-to-delete)+ (batch-delete-permissions! db-rows-to-delete))+ ;; Bulk INSERT: expansion rows for going-granular groups + simple insert rows+ (let [expansion-rows (mapcat+ (fn [{:keys [group-id perm-type new-perm db-perm]}]+ (let [db-perm-value (:perm_value db-perm)+ ;; Build per-table rows for all existing tables (with the old DB-level value)+ existing-table-rows+ (keep (fn [t]+ (when (not= (:id t) table-id)+ {:perm_type perm-type+ :group_id group-id+ :perm_value (case db-perm-value+ :query-builder-and-native :query-builder+ db-perm-value)+ :db_id db-id+ :table_id (:id t)+ :schema_name (:schema t)}))+ all-db-tables)]+ (cons new-perm existing-table-rows)))+ going-granular)]+ (batch-insert-permissions! (concat expansion-rows simple-perms))))))(defenterprise download-perms-level"Return the download permission for the query that the given user has. OSS returns :full"diff --git a/src/metabase/permissions/models/permissions_group.clj b/src/metabase/permissions/models/permissions_group.cljindex 3ac6875f60b..7e6b92c404a 100644--- a/src/metabase/permissions/models/permissions_group.clj+++ b/src/metabase/permissions/models/permissions_group.clj@@ -120,11 +120,9 @@(defn- set-default-permission-values![group]- (t2/with-transaction [_conn]- (doseq [db-id (t2/select-pks-vec :model/Database :router_database_id nil)]- (if (:is_tenant_group group)- (data-perms/set-external-group-permissions! group db-id)- (data-perms/set-new-group-permissions! group db-id (u/the-id (all-users)))))))+ (data-perms/with-global-permissions-lock+ (let [db-ids (t2/select-pks-vec :model/Database :router_database_id nil)]+ (data-perms/set-default-group-permissions! group db-ids (not (:is_tenant_group group))))))(t2/define-after-insert :model/PermissionsGroup[group]diff --git a/src/metabase/permissions_rest/data_permissions/graph.clj b/src/metabase/permissions_rest/data_permissions/graph.cljindex 8f559fa8966..bfd32213565 100644--- a/src/metabase/permissions_rest/data_permissions/graph.clj+++ b/src/metabase/permissions_rest/data_permissions/graph.clj@@ -9,7 +9,6 @@(:require[clojure.string :as str][medley.core :as m]- [metabase.app-db.cluster-lock :as cluster-lock][metabase.audit-app.core :as audit][metabase.permissions-rest.schema :as permissions-rest.schema][metabase.permissions.core :as perms]@@ -485,14 +484,13 @@impersonations and sandboxes are consistent if necessary."([graph-updates :- ::permissions-rest.schema/data-permissions-graph](when (seq graph-updates)- (cluster-lock/with-cluster-lock ::update-data-perms-graph+ (perms/with-global-permissions-lock(let [group-updates (:groups graph-updates)](check-data-analyst-locked-permissions group-updates)(check-audit-db-permissions group-updates)- (t2/with-transaction [_conn]- (update-data-perms-graph!* group-updates)- (delete-impersonations-if-needed-after-permissions-change! group-updates)- (delete-gtaps-if-needed-after-permissions-change! group-updates))))))+ (update-data-perms-graph!* group-updates)+ (delete-impersonations-if-needed-after-permissions-change! group-updates)+ (delete-gtaps-if-needed-after-permissions-change! group-updates)))));; The following arity is provided solely for convenience for tests/REPL usage([ks :- [:vector :any] new-value]diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.cljindex f7f23d3fc74..3a9322a9318 100644--- a/src/metabase/warehouse_schema/models/table.clj+++ b/src/metabase/warehouse_schema/models/table.clj@@ -223,28 +223,36 @@:else (merge table changes)))))+(defn- group-perm-defaults+ "Build the list of {:group-id G :perm-type PT :default-value V} triples for a new table."+ [table all-users-group non-magic-groups non-admin-groups]+ (let [au-id (u/the-id all-users-group)+ is-audit (= (:db_id table) audit/audit-db-id)+ defaults (fn [groups perm-type value]+ (mapv (fn [g] {:group-id (u/the-id g) :perm-type perm-type :default-value value}) groups))]+ (concat+ ;; view-data: all non-admin → :unrestricted+ (defaults non-admin-groups :perms/view-data :unrestricted)+ ;; create-queries+ (if is-audit+ (defaults non-admin-groups :perms/create-queries :no)+ (concat [{:group-id au-id :perm-type :perms/create-queries :default-value :query-builder}]+ (defaults non-magic-groups :perms/create-queries :no)))+ ;; download-results+ [{:group-id au-id :perm-type :perms/download-results :default-value :one-million-rows}]+ (defaults non-magic-groups :perms/download-results :no)+ ;; manage-table-metadata+ (defaults non-admin-groups :perms/manage-table-metadata :no))))+(defn- set-new-table-permissions![table]- (t2/with-transaction [_conn]+ (perms/with-db-scoped-permissions-lock (:db_id table)(let [all-users-group (perms/all-users-group)non-magic-groups (perms/non-magic-groups)non-admin-groups (conj non-magic-groups all-users-group)]- ;; Data access permissions- (if (= (:db_id table) audit/audit-db-id)- (do- ;; Tables in audit DB should start out with no query access in all groups- (perms/set-new-table-permissions! non-admin-groups table :perms/view-data :unrestricted)- (perms/set-new-table-permissions! non-admin-groups table :perms/create-queries :no))- (do- ;; Normal tables start out with unrestricted data access in all groups, but query access only in All Users- (perms/set-new-table-permissions! non-admin-groups table :perms/view-data :unrestricted)- (perms/set-new-table-permissions! [all-users-group] table :perms/create-queries :query-builder)- (perms/set-new-table-permissions! non-magic-groups table :perms/create-queries :no)))- ;; Download permissions- (perms/set-new-table-permissions! [all-users-group] table :perms/download-results :one-million-rows)- (perms/set-new-table-permissions! non-magic-groups table :perms/download-results :no)- ;; Table metadata management- (perms/set-new-table-permissions! non-admin-groups table :perms/manage-table-metadata :no))))+ (perms/set-default-table-permissions!+ table+ (group-perm-defaults table all-users-group non-magic-groups non-admin-groups)))))(t2/define-after-insert :model/Table[table]diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.cljindex aa16dfc3c2f..3584a9eec7a 100644--- a/src/metabase/warehouses/models/database.clj+++ b/src/metabase/warehouses/models/database.clj@@ -321,30 +321,12 @@(defn- set-new-database-permissions- (t2/with-transaction [_conn]- (let [all-users-group (perms/all-users-group)+ (perms/with-db-scoped-permissions-lock (u/the-id database)+ (let [all-users-group (perms/all-users-group)all-external-users-group (perms/all-external-users-group)- non-magic-groups (perms/non-magic-groups)+ non-magic-groups (perms/non-magic-groups)non-admin-groups (conj non-magic-groups all-users-group all-external-users-group)]- (if (:is_audit database)- (doseq [group non-admin-groups]- (if-not (:is_tenant_group group)- (do- (perms/set-database-permission! group database :perms/view-data :unrestricted)- (perms/set-database-permission! group database :perms/create-queries :no)- (perms/set-database-permission! group database :perms/download-results :one-million-rows)- (perms/set-database-permission! group database :perms/manage-table-metadata :no)- (perms/set-database-permission! group database :perms/manage-database :no))- (do- (perms/set-database-permission! group database :perms/view-data :no)- (perms/set-database-permission! group database :perms/create-queries :no)- (perms/set-database-permission! group database :perms/download-results :no)- (perms/set-database-permission! group database :perms/manage-table-metadata :no)- (perms/set-database-permission! group database :perms/manage-database :no))))- (doseq [group non-admin-groups]- (if-not (:is_tenant_group group)- (perms/set-new-database-permissions! group database)- (perms/set-external-group-permissions! group database))))))))+ (perms/set-default-database-permissions! database non-admin-groups)))))(t2/define-after-insert :model/Database[database]diff --git a/test/metabase/app_db/cluster_lock_test.clj b/test/metabase/app_db/cluster_lock_test.cljindex f70f0357187..651bcd5ae99 100644--- a/test/metabase/app_db/cluster_lock_test.clj+++ b/test/metabase/app_db/cluster_lock_test.clj@@ -40,6 +40,104 @@(future (Thread/sleep 500) (a/>!! fin-chan :done))(is (nil? (sut/with-cluster-lock ::test-lock (Thread/sleep 1))))))))+(defn- run-with-lock+ "Run `thunk` with the given lock opts on a future. Returns a map of+ `{:entered, :release, :done}` latches/promise the caller can use to+ coordinate with the running future."+ [lock-opts]+ (let [entered (CountDownLatch. 1)+ release (CountDownLatch. 1)+ done (promise)]+ (future+ (try+ (sut/with-cluster-lock lock-opts+ (.countDown entered)+ (.await release))+ (deliver done :ok)+ (catch Throwable e+ (deliver done [:err (ex-message e)]))))+ {:entered entered :release release :done done}))++(defn- acquirable-within?+ "Returns true if `lock-opts` can be acquired within `timeout-ms` milliseconds."+ [lock-opts timeout-ms]+ (let [entered (promise)+ acquired (future+ (try+ (sut/with-cluster-lock lock-opts+ (deliver entered :yes))+ (catch Throwable _+ (deliver entered :err))))+ result (deref entered timeout-ms :timeout)]+ (future-cancel acquired)+ (= result :yes)))++(deftest share-exclusive-mode-test+ ;; h2 uses in-process ReentrantReadWriteLock; everything else uses real row locks.+ ;; Warm up the row first so we're not racing on the initial INSERT.+ (sut/with-cluster-lock ::rw-test :warm)+ (testing "two :share holders run concurrently"+ (let [a (run-with-lock {:lock ::rw-test :mode :share})+ b (run-with-lock {:lock ::rw-test :mode :share})]+ (is (.await ^CountDownLatch (:entered a) 3 TimeUnit/SECONDS))+ (is (.await ^CountDownLatch (:entered b) 3 TimeUnit/SECONDS))+ (.countDown ^CountDownLatch (:release a))+ (.countDown ^CountDownLatch (:release b))+ (is (= :ok (deref (:done a) 3000 :timeout)))+ (is (= :ok (deref (:done b) 3000 :timeout)))))+ (testing ":share blocks :exclusive while held"+ (let [a (run-with-lock {:lock ::rw-test :mode :share})]+ (is (.await ^CountDownLatch (:entered a) 3 TimeUnit/SECONDS))+ (is (false? (acquirable-within? {:lock ::rw-test :mode :exclusive} 500)))+ (.countDown ^CountDownLatch (:release a))+ (is (= :ok (deref (:done a) 3000 :timeout)))+ ;; After release, exclusive should be able to enter.+ (is (true? (acquirable-within? {:lock ::rw-test :mode :exclusive} 3000)))))+ (testing ":exclusive blocks :share while held"+ (let [a (run-with-lock {:lock ::rw-test :mode :exclusive})]+ (is (.await ^CountDownLatch (:entered a) 3 TimeUnit/SECONDS))+ (is (false? (acquirable-within? {:lock ::rw-test :mode :share} 500)))+ (.countDown ^CountDownLatch (:release a))+ (is (= :ok (deref (:done a) 3000 :timeout)))+ (is (true? (acquirable-within? {:lock ::rw-test :mode :share} 3000))))))++(deftest intent-lock-multi-test+ ;; Warm both leaves so we're not racing on initial INSERTs.+ (sut/with-cluster-lock {:locks [{:lock ::intent-root :mode :share}+ {:lock ::intent-leaf-1 :mode :exclusive}]}+ :warm-1)+ (sut/with-cluster-lock {:locks [{:lock ::intent-root :mode :share}+ {:lock ::intent-leaf-2 :mode :exclusive}]}+ :warm-2)+ (testing "shared-on-root + exclusive-on-leaf: different leaves run in parallel"+ (let [a (run-with-lock {:locks [{:lock ::intent-root :mode :share}+ {:lock ::intent-leaf-1 :mode :exclusive}]})+ b (run-with-lock {:locks [{:lock ::intent-root :mode :share}+ {:lock ::intent-leaf-2 :mode :exclusive}]})]+ (is (.await ^CountDownLatch (:entered a) 3 TimeUnit/SECONDS))+ (is (.await ^CountDownLatch (:entered b) 3 TimeUnit/SECONDS))+ (.countDown ^CountDownLatch (:release a))+ (.countDown ^CountDownLatch (:release b))+ (is (= :ok (deref (:done a) 3000 :timeout)))+ (is (= :ok (deref (:done b) 3000 :timeout)))))+ (testing "shared-on-root + exclusive-on-leaf: same leaf serializes"+ (let [a (run-with-lock {:locks [{:lock ::intent-root :mode :share}+ {:lock ::intent-leaf-1 :mode :exclusive}]})]+ (is (.await ^CountDownLatch (:entered a) 3 TimeUnit/SECONDS))+ (is (false? (acquirable-within? {:locks [{:lock ::intent-root :mode :share}+ {:lock ::intent-leaf-1 :mode :exclusive}]}+ 500)))+ (.countDown ^CountDownLatch (:release a))+ (is (= :ok (deref (:done a) 3000 :timeout)))))+ (testing "exclusive-on-root (global) blocks any shared-on-root db-scoped writer"+ (let [a (run-with-lock {:lock ::intent-root :mode :exclusive})]+ (is (.await ^CountDownLatch (:entered a) 3 TimeUnit/SECONDS))+ (is (false? (acquirable-within? {:locks [{:lock ::intent-root :mode :share}+ {:lock ::intent-leaf-1 :mode :exclusive}]}+ 500)))+ (.countDown ^CountDownLatch (:release a))+ (is (= :ok (deref (:done a) 3000 :timeout))))))+(deftest concurrent-lock-creation-race-test(testing "Two threads racing to create the same lock for the first time";; h2 is not race-proof, but it is also not cross-processdiff --git a/test/metabase/permissions/models/data_permissions_test.clj b/test/metabase/permissions/models/data_permissions_test.cljindex de796a2f235..ad1f0dc4b04 100644--- a/test/metabase/permissions/models/data_permissions_test.clj+++ b/test/metabase/permissions/models/data_permissions_test.clj@@ -735,6 +735,220 @@(mt/with-temp [:model/Table {table-id-4 :id} {:db_id db-id :schema "PUBLIC"}](is (= :no (perm-value table-id-4))))))))+;;; ---------------------------------------- Batch permission function tests ----------------------------------------++(deftest set-default-group-permissions!-tenant-test+ (testing "Tenant groups get least-permissive defaults for all databases"+ (mt/with-temp [:model/Database {db-id-1 :id} {}+ :model/Database {db-id-2 :id} {}+ :model/PermissionsGroup {group-id :id} {:is_tenant_group true}]+ ;; Delete the auto-created permissions so we can test the function directly+ (t2/delete! :model/DataPermissions :group_id group-id)+ (data-perms/set-default-group-permissions! group-id [db-id-1 db-id-2] false)+ (let [perms (t2/select :model/DataPermissions :group_id group-id)+ by-db (group-by :db_id perms)+ perm-map (fn [db-id]+ (into {} (map (fn [{:keys [perm_type perm_value]}] [perm_type perm_value])+ (get by-db db-id))))]+ (doseq [db-id [db-id-1 db-id-2]]+ (let [pm (perm-map db-id)]+ (is (= :blocked (get pm :perms/view-data))+ "view-data must be :blocked for tenant groups")+ (is (= :no (get pm :perms/create-queries))+ "create-queries must be :no for tenant groups")+ (is (= :no (get pm :perms/download-results))+ "download-results must be :no for tenant groups")+ (is (= :no (get pm :perms/manage-table-metadata))+ "manage-table-metadata must be :no for tenant groups")+ (is (= :no (get pm :perms/manage-database))+ "manage-database must be :no for tenant groups")))))))++(deftest set-default-group-permissions!-regular-test+ (testing "Regular groups mirror All Users permissions, coalescing to most-restrictive per DB"+ (mt/with-temp [:model/Database {db-id-1 :id} {}+ :model/Database {db-id-2 :id} {}+ :model/PermissionsGroup {group-id :id} {}]+ (let [au-id (:id (perms-group/all-users))]+ ;; Set up All Users with different levels on each DB+ (data-perms/set-database-permission! au-id db-id-1 :perms/create-queries :query-builder-and-native)+ (data-perms/set-database-permission! au-id db-id-2 :perms/create-queries :no)+ (data-perms/set-database-permission! au-id db-id-1 :perms/download-results :one-million-rows)+ (data-perms/set-database-permission! au-id db-id-2 :perms/download-results :no)+ ;; Delete auto-created permissions for the new group+ (t2/delete! :model/DataPermissions :group_id group-id)+ (data-perms/set-default-group-permissions! group-id [db-id-1 db-id-2] true)+ (let [perm-val (fn [db-id perm-type]+ (t2/select-one-fn :perm_value :model/DataPermissions+ :group_id group-id :db_id db-id :perm_type perm-type))]+ (testing "view-data defaults to :unrestricted on OSS"+ (is (= :unrestricted (perm-val db-id-1 :perms/view-data)))+ (is (= :unrestricted (perm-val db-id-2 :perms/view-data))))+ (testing "create-queries mirrors All Users per-DB"+ (is (= :query-builder-and-native (perm-val db-id-1 :perms/create-queries)))+ (is (= :no (perm-val db-id-2 :perms/create-queries))))+ (testing "download-results mirrors All Users per-DB"+ (is (= :one-million-rows (perm-val db-id-1 :perms/download-results)))+ (is (= :no (perm-val db-id-2 :perms/download-results)))))))))++(deftest set-default-database-permissions!-tenant-group-test+ (testing "Tenant groups get least-permissive defaults for any new database"+ (mt/with-temp [:model/PermissionsGroup {group-id :id} {:is_tenant_group true}+ :model/Database {db-id :id} {}]+ ;; Delete auto-created permissions+ (t2/delete! :model/DataPermissions :group_id group-id :db_id db-id)+ (let [group (t2/select-one :model/PermissionsGroup :id group-id)]+ (data-perms/set-default-database-permissions! {:id db-id} [group])+ (let [perm-val (fn [perm-type]+ (t2/select-one-fn :perm_value :model/DataPermissions+ :group_id group-id :db_id db-id :perm_type perm-type))]+ (is (= :blocked (perm-val :perms/view-data)))+ (is (= :no (perm-val :perms/create-queries)))+ (is (= :no (perm-val :perms/download-results)))+ (is (= :no (perm-val :perms/manage-table-metadata)))+ (is (= :no (perm-val :perms/manage-database))))))))++(deftest set-default-database-permissions!-audit-db-test+ (testing "Audit DBs get hardcoded restrictive permission values"+ (mt/with-temp [:model/PermissionsGroup {group-id :id} {}+ :model/Database {db-id :id} {}]+ ;; Delete auto-created permissions+ (t2/delete! :model/DataPermissions :group_id group-id :db_id db-id)+ (let [group (t2/select-one :model/PermissionsGroup :id group-id)]+ (data-perms/set-default-database-permissions! {:id db-id :is_audit true} [group])+ (let [perm-val (fn [perm-type]+ (t2/select-one-fn :perm_value :model/DataPermissions+ :group_id group-id :db_id db-id :perm_type perm-type))]+ (is (= :unrestricted (perm-val :perms/view-data)))+ (is (= :no (perm-val :perms/create-queries)))+ (is (= :one-million-rows (perm-val :perms/download-results)))+ (is (= :no (perm-val :perms/manage-table-metadata)))+ (is (= :no (perm-val :perms/manage-database))))))))++(deftest set-default-database-permissions!-blocked-downloads-test+ (testing "When view-data is :blocked, download-results must be :no"+ (mt/with-temp [:model/PermissionsGroup {group-id :id} {}+ :model/Database {db-id :id} {}]+ ;; Delete auto-created permissions+ (t2/delete! :model/DataPermissions :group_id group-id :db_id db-id)+ (let [group (t2/select-one :model/PermissionsGroup :id group-id)]+ ;; Simulate EE returning :blocked for this group's view-data+ (with-redefs [data-perms/new-database-view-data-permission-levels+ (fn [group-ids] (zipmap group-ids (repeat :blocked)))]+ (data-perms/set-default-database-permissions! {:id db-id} [group])+ (is (= :no (t2/select-one-fn :perm_value :model/DataPermissions+ :group_id group-id :db_id db-id+ :perm_type :perms/download-results))+ "blocked view-data must force :no downloads"))))))++(deftest set-default-table-permissions!-going-granular-test+ (testing "When a group has DB-level perms but new table needs :blocked, it expands to per-table rows"+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/PermissionsGroup {group-id :id} {}+ :model/Table {table-id-1 :id} {:db_id db-id :schema "PUBLIC" :active true}+ :model/Table {table-id-2 :id} {:db_id db-id :schema "PUBLIC" :active true}]+ ;; Ensure group has DB-level view-data permission (not table-level)+ (data-perms/set-database-permission! group-id db-id :perms/view-data :unrestricted)+ (is (some? (t2/select-one :model/DataPermissions+ :group_id group-id :db_id db-id :table_id nil+ :perm_type :perms/view-data))+ "precondition: DB-level row exists")+ ;; Now add a new table, with the enterprise hook returning :blocked+ (mt/with-temp [:model/Table {table-id-3 :id} {:db_id db-id :schema "PUBLIC" :active true}]+ ;; Delete auto-created view-data permission for table-id-3 so we can test set-default-table-permissions! directly+ (t2/delete! :model/DataPermissions :group_id group-id :table_id table-id-3 :perm_type :perms/view-data)+ (with-redefs [data-perms/new-table-view-data-permission-levels+ (fn [_db-id group-ids] (zipmap group-ids (repeat :blocked)))]+ (data-perms/set-default-table-permissions!+ {:id table-id-3 :db_id db-id :schema "PUBLIC"}+ [{:group-id group-id :perm-type :perms/view-data :default-value :unrestricted}]))+ (testing "DB-level row should be deleted"+ (is (nil? (t2/select-one :model/DataPermissions+ :group_id group-id :db_id db-id :table_id nil+ :perm_type :perms/view-data))))+ (testing "existing tables get per-table :unrestricted rows"+ (is (= :unrestricted (t2/select-one-fn :perm_value :model/DataPermissions+ :group_id group-id :db_id db-id+ :table_id table-id-1 :perm_type :perms/view-data)))+ (is (= :unrestricted (t2/select-one-fn :perm_value :model/DataPermissions+ :group_id group-id :db_id db-id+ :table_id table-id-2 :perm_type :perms/view-data))))+ (testing "new table gets :blocked"+ (is (= :blocked (t2/select-one-fn :perm_value :model/DataPermissions+ :group_id group-id :db_id db-id+ :table_id table-id-3 :perm_type :perms/view-data))))))))++(deftest set-default-table-permissions!-simple-insert-test+ (testing "When group is already table-granular, new table gets a simple insert"+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/PermissionsGroup {group-id :id} {}+ :model/Table {table-id-1 :id} {:db_id db-id :schema "PUBLIC"}+ :model/Table {table-id-2 :id} {:db_id db-id :schema "PUBLIC"}]+ ;; Set table-level permissions (this removes DB-level row)+ (data-perms/set-table-permission! group-id table-id-1 :perms/create-queries :query-builder)+ (data-perms/set-table-permission! group-id table-id-2 :perms/create-queries :no)+ (is (nil? (t2/select-one :model/DataPermissions+ :group_id group-id :db_id db-id :table_id nil+ :perm_type :perms/create-queries))+ "precondition: no DB-level row")+ (mt/with-temp [:model/Table {table-id-3 :id} {:db_id db-id :schema "PUBLIC"}]+ ;; Delete the auto-created permissions for table-id-3 so we can test set-default-table-permissions! directly+ (t2/delete! :model/DataPermissions :group_id group-id :table_id table-id-3 :perm_type :perms/create-queries)+ (data-perms/set-default-table-permissions!+ {:id table-id-3 :db_id db-id :schema "PUBLIC"}+ [{:group-id group-id :perm-type :perms/create-queries :default-value :no}])+ ;; Schema has mixed values (:query-builder and :no), so default :no should be used+ (is (= :no (t2/select-one-fn :perm_value :model/DataPermissions+ :group_id group-id :db_id db-id+ :table_id table-id-3 :perm_type :perms/create-queries)))))))++(deftest set-default-table-permissions!-schema-consistency-test+ (testing "New table inherits uniform permission value from schema"+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/PermissionsGroup {group-id :id} {}+ :model/Table {table-id-1 :id} {:db_id db-id :schema "PUBLIC"}+ :model/Table {table-id-2 :id} {:db_id db-id :schema "PUBLIC"}+ :model/Table {table-id-3 :id} {:db_id db-id :schema "other-schema"}]+ ;; Set tables so PUBLIC is uniform at :query-builder, but other-schema differs (prevents consolidation to DB-level)+ (data-perms/set-table-permission! group-id table-id-1 :perms/create-queries :query-builder)+ (data-perms/set-table-permission! group-id table-id-2 :perms/create-queries :query-builder)+ (data-perms/set-table-permission! group-id table-id-3 :perms/create-queries :no)+ (mt/with-temp [:model/Table {table-id-4 :id} {:db_id db-id :schema "PUBLIC"}]+ ;; Delete all auto-created perms for table-id-4 and re-test+ (t2/delete! :model/DataPermissions :group_id group-id :table_id table-id-4 :perm_type :perms/create-queries)+ (data-perms/set-default-table-permissions!+ {:id table-id-4 :db_id db-id :schema "PUBLIC"}+ [{:group-id group-id :perm-type :perms/create-queries :default-value :no}])+ (is (= :query-builder (t2/select-one-fn :perm_value :model/DataPermissions+ :group_id group-id :db_id db-id+ :table_id table-id-4 :perm_type :perms/create-queries))+ "should inherit :query-builder from PUBLIC schema, not the :no default")))))++(deftest batch-permissions-lock-skips-fine-grained-locks-test+ (testing "Fine-grained cluster locks are skipped inside with-global-permissions-lock"+ (mt/with-temp [:model/Database {db-id :id} {}]+ (data-perms/with-global-permissions-lock+ (testing "skip-cluster-locks dynamic var is bound to true"+ (is (true? (deref #'data-perms/*skip-cluster-locks*))))+ (testing "set-database-permission! succeeds without deadlock"+ ;; This would deadlock if with-cluster-lock tried to acquire a per-(db, perm-type) lock+ ;; while the coarse batch lock is already held+ (let [au-id (:id (perms-group/all-users))]+ (data-perms/set-database-permission! au-id db-id :perms/view-data :unrestricted)+ (is (= :unrestricted+ (t2/select-one-fn :perm_value :model/DataPermissions+ :group_id au-id :db_id db-id+ :perm_type :perms/view-data))))))))+ (testing "Fine-grained cluster locks are skipped inside with-db-scoped-permissions-lock"+ (mt/with-temp [:model/Database {db-id :id} {}]+ (data-perms/with-db-scoped-permissions-lock db-id+ (is (true? (deref #'data-perms/*skip-cluster-locks*)))+ (let [au-id (:id (perms-group/all-users))]+ (data-perms/set-database-permission! au-id db-id :perms/view-data :unrestricted)+ (is (= :unrestricted+ (t2/select-one-fn :perm_value :model/DataPermissions+ :group_id au-id :db_id db-id+ :perm_type :perms/view-data))))))))+(deftest additional-table-permissions-works(mt/with-temp [:model/PermissionsGroup {group-id :id} {}:model/Database {db-id :id} {}