Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[perms] Break up large SELECTs to avoid the 65K SQL parameter limit (#75032)

Braden Shepherdson9393fa8c9b52
diff --git a/src/metabase/permissions/models/collection/graph.clj b/src/metabase/permissions/models/collection/graph.clj
index 7a8eb279111..32e4bfa9c3e 100644
--- a/src/metabase/permissions/models/collection/graph.clj
+++ b/src/metabase/permissions/models/collection/graph.clj
@@ -20,6 +20,19 @@
;;; | PERMISSIONS GRAPH |
;;; +----------------------------------------------------------------------------------------------------------------+
+;; PostgreSQL limits prepared statements to 65,535 bind parameters.
+(def ^:private max-in-clause-size 20000)
+
+(defn- chunked-select-pks
+ "Like `t2/select-pks-set` but chunks large ID sets to avoid exceeding the bind parameter limit.
+ `where-fn` takes a set of IDs and returns a HoneySQL where clause."
+ [model ids where-fn]
+ (if (<= (count ids) max-in-clause-size)
+ (t2/select-pks-set model {:where (where-fn ids)})
+ (into #{} (mapcat (fn [chunk]
+ (t2/select-pks-set model {:where (where-fn (set chunk))})))
+ (partition-all max-in-clause-size ids))))
+
;;; ---------------------------------------------------- Schemas -----------------------------------------------------
(def ^:private CollectionPermissions
@@ -59,53 +72,17 @@
graph
(assoc-in graph [:groups admin-group-id audit-collection-id] :read))))
-(mu/defn graph :- PermissionsGraph
- "Fetch a sparse graph representing the current permissions status for groups and collections with permissions.
-
- The returned graph is **sparse**: it only includes entries for groups and collections that have explicit
- permissions (`:read` or `:write`). Groups without any permissions and collections without permissions for a
- given group are omitted from the graph.
-
- **Exclusions:**
- - Personal collections and their descendants are never included
- - Archived collections are excluded
- - Trash collections are excluded
- - Collections from other namespaces are excluded when `collection-namespace` is specified
-
- **Parameters:**
- - `collection-namespace` (optional): Restricts the graph to a specific namespace. Default `nil` returns
- collections in the 'default' namespace (normal Card/Dashboard/Pulse Collections).
- - `collection-ids` (optional): When provided, restricts the graph to only the specified collection IDs
- - `group-ids` (optional): When provided, restricts the graph to only the specified permission group IDs
-
- **Structure:**
- The graph has the structure:
- ```
- {:revision <int>
- :groups {<group-id> {<collection-id> <:read|:write>
- :root <:read|:write>}}}
- ```
-
- Note: All Collections are returned at the same level of the 'graph', regardless of how the Collection hierarchy is
- structured. Collections do not inherit permissions from ancestor Collections in the same way data permissions are
- inherited (e.g. full `:read` perms for a Database implies `:read` perms for all its schemas); a 'child' object (e.g.
- schema) *cannot* have more restrictive permissions than its parent (e.g. Database). Child Collections *can* have
- more restrictive permissions than their parent."
- ([]
- (graph nil))
-
- ([collection-namespace :- [:maybe ms/KeywordOrString]]
- (graph collection-namespace nil nil))
-
- ([collection-namespace :- [:maybe ms/KeywordOrString]
- collection-ids :- [:maybe [:set [:or [:= :root] ms/PositiveInt]]]
- group-ids :- [:maybe [:set ms/PositiveInt]]]
- (let [include-root? (or (nil? collection-ids) (contains? collection-ids :root))
- root-object (str "/collection/"
- (when collection-namespace
- (str "namespace/" (name collection-namespace) "/"))
- "root/")]
- (->> (t2/reducible-query
+(defn- reduce-graph-query
+ "Run the graph query for a specific chunk of collection IDs and reduce results into `accum`."
+ [accum collection-namespace include-root? root-object ids-without-root group-ids]
+ (reduce (fn [acc {group-id :group_id collection-id :collection_id :keys [writable readable]}]
+ (assoc-in acc [group-id (or collection-id :root)]
+ (cond
+ (= writable 1) :write
+ (= readable 1) :read
+ :else :none)))
+ accum
+ (t2/reducible-query
{:with [[:eligible_collections
{:select [:id]
:from [:collection]
@@ -115,9 +92,8 @@
:namespace (u/qualified-name collection-namespace))
[:not :archived]
[:= :personal_owner_id nil]
- (let [ids-without-root (disj collection-ids :root)]
- (when (seq ids-without-root)
- [:in :id ids-without-root]))
+ (when (seq ids-without-root)
+ [:in :id ids-without-root])
[:not [:exists {:select [1]
:from [[:collection :pc]]
:where [:and
@@ -173,14 +149,65 @@
[:c.id :collection_id]
[[:inline 1] :writable]
[[:inline 1] :readable]]
- :from [[:eligible_collections :c]]}]})
- (reduce (fn [accum {group-id :group_id collection-id :collection_id :keys [writable readable]}]
- (assoc-in accum [group-id (or collection-id :root)]
- (cond
- (= writable 1) :write
- (= readable 1) :read
- :else :none)))
- {(u/the-id (perms-group/admin)) {:root :write}})
+ :from [[:eligible_collections :c]]}]})))
+
+(mu/defn graph :- PermissionsGraph
+ "Fetch a sparse graph representing the current permissions status for groups and collections with permissions.
+
+ The returned graph is **sparse**: it only includes entries for groups and collections that have explicit
+ permissions (`:read` or `:write`). Groups without any permissions and collections without permissions for a
+ given group are omitted from the graph.
+
+ **Exclusions:**
+ - Personal collections and their descendants are never included
+ - Archived collections are excluded
+ - Trash collections are excluded
+ - Collections from other namespaces are excluded when `collection-namespace` is specified
+
+ **Parameters:**
+ - `collection-namespace` (optional): Restricts the graph to a specific namespace. Default `nil` returns
+ collections in the 'default' namespace (normal Card/Dashboard/Pulse Collections).
+ - `collection-ids` (optional): When provided, restricts the graph to only the specified collection IDs
+ - `group-ids` (optional): When provided, restricts the graph to only the specified permission group IDs
+
+ **Structure:**
+ The graph has the structure:
+ ```
+ {:revision <int>
+ :groups {<group-id> {<collection-id> <:read|:write>
+ :root <:read|:write>}}}
+ ```
+
+ Note: All Collections are returned at the same level of the 'graph', regardless of how the Collection hierarchy is
+ structured. Collections do not inherit permissions from ancestor Collections in the same way data permissions are
+ inherited (e.g. full `:read` perms for a Database implies `:read` perms for all its schemas); a 'child' object (e.g.
+ schema) *cannot* have more restrictive permissions than its parent (e.g. Database). Child Collections *can* have
+ more restrictive permissions than their parent."
+ ([]
+ (graph nil))
+
+ ([collection-namespace :- [:maybe ms/KeywordOrString]]
+ (graph collection-namespace nil nil))
+
+ ([collection-namespace :- [:maybe ms/KeywordOrString]
+ collection-ids :- [:maybe [:set [:or [:= :root] ms/PositiveInt]]]
+ group-ids :- [:maybe [:set ms/PositiveInt]]]
+ (let [include-root? (or (nil? collection-ids)
+ (contains? collection-ids :root))
+ root-object (str "/collection/"
+ (when collection-namespace
+ (str "namespace/" (name collection-namespace) "/"))
+ "root/")
+ ids-without-root (some-> collection-ids (disj :root))
+ chunks (if (or (empty? ids-without-root)
+ (<= (count ids-without-root) max-in-clause-size))
+ [ids-without-root]
+ (map set (partition-all max-in-clause-size ids-without-root)))
+ init {(u/the-id (perms-group/admin)) {:root :write}}]
+ (->> (reduce (fn [accum id-chunk]
+ (reduce-graph-query accum collection-namespace include-root?
+ root-object id-chunk group-ids))
+ init chunks)
collection-permission-graph
modify-instance-analytics-for-admins))))
@@ -236,17 +263,19 @@
"Return a set of IDs from `collection-ids` that are personal Collections or descendants of personal Collections.
These should never appear in permission graphs or be editable via the graph API."
[collection-ids]
- (when (seq (disj collection-ids :root))
- (t2/select-pks-set :model/Collection
- {:where [:and
- [:in :id (disj collection-ids :root)]
- [:or [:not= :personal_owner_id nil]
- [:exists {:select [1]
- :from [[:collection :pc]]
- :where [:and
- [:not= :pc.personal_owner_id nil]
- [:like :collection.location
- [:concat "/" :pc.id "/%"]]]}]]]})))
+ (when-let [ids (seq (disj collection-ids :root))]
+ (chunked-select-pks
+ :model/Collection ids
+ (fn [id-set]
+ [:and
+ [:in :id id-set]
+ [:or [:not= :personal_owner_id nil]
+ [:exists {:select [1]
+ :from [[:collection :pc]]
+ :where [:and
+ [:not= :pc.personal_owner_id nil]
+ [:like :collection.location
+ [:concat "/" :pc.id "/%"]]]}]]]))))
(defn- remove-personal-collections-from-graph
"Remove any personal collection IDs from the graph. Personal collections cannot be edited via the graph API."
@@ -258,14 +287,15 @@
(defn- remove-collections-from-other-namespaces
"Remove any collection IDs from the graph that belong to another namespace from the graph being updated."
[graph collection-ids namespace]
- (let [other-ns-ids (when (seq (disj collection-ids :root))
- ;; This query selects collection IDs that don't match the target namespace:
- ;; - If target namespace is non-nil: collections with different non-nil namespaces OR nil namespaces
- ;; - If target namespace is nil: collections with any non-nil namespace (except the 'analytics' namespace)
- (t2/select-pks-set :model/Collection {:where [:and [:in :id (disj collection-ids :root)]
- (cond->> [[:not= :namespace (some-> namespace name)]]
- (nil? namespace) (into [:and [:not= :namespace "analytics"]])
- (some? namespace) (into [:or [:= :namespace nil]]))]}))]
+ (let [ids (disj collection-ids :root)
+ other-ns-ids (when (seq ids)
+ (chunked-select-pks
+ :model/Collection ids
+ (fn [id-set]
+ [:and [:in :id id-set]
+ (cond->> [[:not= :namespace (some-> namespace name)]]
+ (nil? namespace) (into [:and [:not= :namespace "analytics"]])
+ (some? namespace) (into [:or [:= :namespace nil]]))])))]
(cond-> graph
(seq other-ns-ids) (update :groups update-vals #(apply dissoc % other-ns-ids)))))
diff --git a/test/metabase/permissions/models/collection/graph_test.clj b/test/metabase/permissions/models/collection/graph_test.clj
index 763ed36ec96..2ec3034fa3f 100644
--- a/test/metabase/permissions/models/collection/graph_test.clj
+++ b/test/metabase/permissions/models/collection/graph_test.clj
@@ -582,3 +582,7 @@
(is (malli= [:map [:id :int] [:user_id :int] [:before :some] [:after :some]]
(t2/select-one :model/CollectionPermissionGraphRevision (inc (:revision before))))
"Values for before and after should be present in the revision."))))))
+
+(deftest issue-70596-large-collection-ids-test
+ (testing "graph/graph doesn't throw when called with more collection IDs than the bind parameter limit (#70596)"
+ (is (map? (graph/graph nil (set (range 1 70001)) nil)))))