Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[perms] Fix N+1 query in sandboxing when updating perms (#74734)

Braden Shepherdson72eb2f945ce3
diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/models/permissions/delete_sandboxes.clj b/enterprise/backend/src/metabase_enterprise/sandbox/models/permissions/delete_sandboxes.clj
index 7b3c356706e..dde6c0e0bec 100644
--- a/enterprise/backend/src/metabase_enterprise/sandbox/models/permissions/delete_sandboxes.clj
+++ b/enterprise/backend/src/metabase_enterprise/sandbox/models/permissions/delete_sandboxes.clj
@@ -2,77 +2,30 @@
(:require
[metabase.app-db.core :as app-db]
[metabase.premium-features.core :refer [defenterprise]]
- [metabase.util :as u]
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]
[toucan2.core :as t2]))
-(defn- delete-gtaps-with-condition! [group-or-id condition]
- (when (seq condition)
- (let [conditions (into
- [:and
- [:= :sandboxes.group_id (u/the-id group-or-id)]]
- [condition])]
- (log/debugf "Deleting GTAPs for Group %d with conditions %s" (u/the-id group-or-id) (pr-str conditions))
- (try
- (if-let [gtap-ids (not-empty (set (map :id (app-db/query
- {:select [[:sandboxes.id :id]]
- :from [[:sandboxes]]
- :left-join [[:metabase_table :table]
- [:= :sandboxes.table_id :table.id]]
- :where conditions}))))]
- (do
- (log/debugf "Deleting %d matching GTAPs: %s" (count gtap-ids) (pr-str gtap-ids))
- (t2/delete! :model/Sandbox :id [:in gtap-ids]))
- (log/debug "No matching GTAPs need to be deleted."))
- (catch Throwable e
- (throw (ex-info (tru "Error deleting Sandboxes: {0}" (ex-message e))
- {:group (u/the-id group-or-id), :conditions conditions}
- e)))))))
-
-(defn- delete-gtaps-for-group-table! [{:keys [group-id table-id] :as _context} changes]
- (log/debugf "Deleting unneeded GTAPs for Group %d for Table %d. Graph changes: %s"
- group-id table-id (pr-str changes))
- (if (not= changes :sandboxed)
- (do
- (log/debugf "Group %d now has full data perms for Table %d, deleting GTAP for this Table if one exists"
- group-id table-id)
- (delete-gtaps-with-condition! group-id [:= :table.id table-id]))
- (log/debugf "Group %d now has full sandboxed query perms for Table %d. Do not need to delete GTAPs."
- group-id table-id)))
-
-(defn- delete-gtaps-for-group-schema! [{:keys [group-id database-id schema-name], :as context} changes]
- (log/debugf "Deleting unneeded GTAPs for Group %d for Database %d, schema %s. Graph changes: %s"
- group-id database-id (pr-str schema-name) (pr-str changes))
- (if (keyword? changes)
- (do
- (log/debugf "Group %d changes has %s perms for Database %d schema %s, deleting all sandboxes for this schema"
- group-id changes database-id (pr-str schema-name))
- (delete-gtaps-with-condition! group-id [:and [:= :table.db_id database-id] [:= :table.schema schema-name]]))
- (doseq [table-id (set (keys changes))]
- (delete-gtaps-for-group-table! (assoc context :table-id table-id) (get changes table-id)))))
-
-(defn- delete-gtaps-for-group-database! [{:keys [group-id database-id], :as context} changes]
- (log/debugf "Deleting unneeded GTAPs for Group %d for Database %d. Graph changes: %s"
- group-id database-id (pr-str changes))
- (if (keyword? changes)
- ;; If we're setting a single permission type for the entire DB, clear all sandboxes in the DB
- (do
- (log/debugf "Group %d now has %s perms for Database %d, deleting all sandboxes for this schema"
- group-id changes database-id)
- (delete-gtaps-with-condition! group-id [:= :table.db_id database-id]))
- (doseq [schema-name (set (keys changes))]
- (delete-gtaps-for-group-schema!
- (assoc context :schema-name schema-name)
- (get changes schema-name)))))
-
-(defn- delete-gtaps-for-group! [{:keys [group-id]} changes]
- (log/debugf "Deleting unneeded GTAPs for Group %d. Graph changes: %s" group-id (pr-str changes))
- (doseq [database-id (set (keys changes))]
- (when-let [data-perm-changes (get-in changes [database-id :view-data])]
- (delete-gtaps-for-group-database!
- {:group-id group-id, :database-id database-id}
- data-perm-changes))))
+(defn- should-delete-sandbox?
+ "Given the changes map and a candidate sandbox, determine if this sandbox should be deleted.
+ A sandbox should be deleted when the permission change for its group+table means it is no longer sandboxed."
+ [changes {:keys [group_id db_id schema table_id]}]
+ (let [view-data-changes (get-in changes [group_id db_id :view-data])]
+ (cond
+ (nil? view-data-changes) false
+ ;; DB-level change (keyword like :unrestricted) → delete all sandboxes for this group+db
+ (keyword? view-data-changes) true
+ :else
+ (let [schema-changes (get view-data-changes schema)]
+ (cond
+ (nil? schema-changes) false
+ ;; Schema-level change → delete all sandboxes for this group+db+schema
+ (keyword? schema-changes) true
+ :else
+ ;; Table-level change → delete unless the new value is :sandboxed
+ (let [table-change (get schema-changes table_id)]
+ (and (some? table-change)
+ (not= table-change :sandboxed))))))))
(defenterprise delete-gtaps-if-needed-after-permissions-change!
"For use only inside `metabase.permissions.models.permissions`; don't call this elsewhere. Delete GTAPs (sandboxes) that are no
@@ -81,6 +34,37 @@
:feature :sandboxes
[changes]
(log/debug "Permissions updated, deleting unneeded GTAPs...")
- (doseq [group-id (set (keys changes))]
- (delete-gtaps-for-group! {:group-id group-id} (get changes group-id)))
+ (try
+ (let [all-group-ids (into #{} (for [[group-id group-changes] changes
+ [_db-id db-changes] group-changes
+ :when (contains? db-changes :view-data)]
+ group-id))
+ all-db-ids (into #{} (for [[_group-id group-changes] changes
+ [db-id db-changes] group-changes
+ :when (contains? db-changes :view-data)]
+ db-id))]
+ (when (and (seq all-group-ids) (seq all-db-ids))
+ (let [candidate-sandboxes (app-db/query
+ {:select [[:sandboxes.id :id]
+ [:sandboxes.group_id :group_id]
+ [:sandboxes.table_id :table_id]
+ [:table.db_id :db_id]
+ [:table.schema :schema]]
+ :from [[:sandboxes]]
+ :left-join [[:metabase_table :table]
+ [:= :sandboxes.table_id :table.id]]
+ :where [:and
+ [:in :sandboxes.group_id all-group-ids]
+ [:in :table.db_id all-db-ids]]})
+ ids-to-delete (into #{}
+ (comp (filter (partial should-delete-sandbox? changes))
+ (map :id))
+ candidate-sandboxes)]
+ (when (seq ids-to-delete)
+ (log/debugf "Deleting %d unneeded GTAPs: %s" (count ids-to-delete) (pr-str ids-to-delete))
+ (t2/delete! :model/Sandbox :id [:in ids-to-delete])))))
+ (catch Throwable e
+ (throw (ex-info (tru "Error deleting Sandboxes: {0}" (ex-message e))
+ {:changes changes}
+ e))))
(log/debug "Done deleting unneeded GTAPs."))
diff --git a/test/metabase/permissions_rest/api_test.clj b/test/metabase/permissions_rest/api_test.clj
index dd9da475039..7da67ab5dc8 100644
--- a/test/metabase/permissions_rest/api_test.clj
+++ b/test/metabase/permissions_rest/api_test.clj
@@ -15,7 +15,11 @@
[metabase.util :as u]
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
- [toucan2.core :as t2]))
+ [methodical.core :as methodical]
+ [toucan2.core :as t2]
+ [toucan2.pipeline :as t2.pipeline]))
+
+(set! *warn-on-reflection* true)
;; there are some issues where it doesn't look like the hydrate function for `member_count` is being added (?)
(comment api.permissions/keep-me)
@@ -573,3 +577,61 @@
(is (=? {:magic_group_type "all-external-users"
:name "All tenant users"}
(get-magic-group "all-external-users"))))))))
+
+;;; ---------------------------------------- Performance tests ------------------------------------------
+
+(defn- count-db-calls
+ "Execute `f` and return the number of database calls (queries) made during its execution."
+ [f]
+ (let [call-count (atom 0)]
+ (methodical/add-aux-method-with-unique-key!
+ #'t2.pipeline/transduce-execute-with-connection
+ :around :default
+ (fn [next-method rf conn query-type model query]
+ (swap! call-count inc)
+ (next-method rf conn query-type model query))
+ ::query-counter)
+ (try
+ (f)
+ (finally
+ (methodical/remove-aux-method-with-unique-key!
+ #'t2.pipeline/transduce-execute-with-connection
+ :around :default
+ ::query-counter)))
+ @call-count))
+
+(deftest permissions-graph-update-query-count-test
+ (testing "PUT /api/permissions/graph should not make an excessive number of DB calls"
+ (mt/with-premium-features #{:advanced-permissions :sandboxes}
+ (mt/with-temp [:model/Database {db-id :id} {}]
+ (let [num-groups 28
+ num-tables 382
+ now (java.time.OffsetDateTime/now)
+ table-ids (t2/insert-returning-pks! (t2/table-name :model/Table)
+ (for [i (range num-tables)]
+ {:db_id db-id
+ :name (format "table_%d" i)
+ :schema "PUBLIC"
+ :active true
+ :created_at now
+ :updated_at now}))
+ group-ids (t2/insert-returning-pks! :model/PermissionsGroup
+ (for [i (range num-groups)]
+ {:name (str "perf-test-group-" i "-" (random-uuid))}))]
+ (try
+ (let [base-graph (data-perms.graph/api-graph {:group-ids group-ids})
+ view-perms {"PUBLIC" (zipmap table-ids (repeat :unrestricted))}
+ cq-perms {"PUBLIC" (zipmap table-ids (repeat :query-builder))}
+ updated-graph (reduce (fn [g gid]
+ (-> g
+ (assoc-in [:groups gid db-id :view-data] view-perms)
+ (assoc-in [:groups gid db-id :create-queries] cq-perms)))
+ base-graph
+ group-ids)
+ num-calls (count-db-calls
+ #(mt/user-http-request :crowberto :put 200 "permissions/graph"
+ updated-graph))]
+ (is (<= num-calls 100)
+ (format "Expected at most 100 database calls, got %d" num-calls)))
+ (finally
+ (t2/delete! :model/PermissionsGroup :id [:in group-ids]))))))))