Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Remove Honey SQL arguments from db.clj (#82175)

Alexander Polyankin28b2ad9eb638
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index cd2030a6c58..3767259b7dd 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1475,6 +1475,7 @@
metabase.metrics.core}
:uses #{api
app-db
+ collections
events
lib
lib-be
@@ -1482,7 +1483,6 @@
models
parameters
permissions
- queries
query-processor
request
server
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index ca1d117a877..565c75817b8 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -271,7 +271,9 @@
(mapcat (fn [[model-type model-rows]]
(let [spec (spec/spec-for-model-type model-type)
model-key (:model-key spec)
- opts {:where [:in :id (mapv :model_id model-rows)] :skip-archived true}
+ opts {:filter-column :id
+ :filter-ids (mapv :model_id model-rows)
+ :skip-archived true}
;; entity-id models: map local id -> entity_id so we can look up the repo path
id->eid (when (and model-key (= :entity-id (:identity spec)))
(remote-sync.db/entity-ids-by-id model-key (mapv :model_id model-rows)))]
@@ -839,7 +841,9 @@
[{:keys [model_type rows]}]
(let [pk-col (spec/pk-col model_type)
id->row (u/index-by :model_id rows)
- opts {:where [:in pk-col (mapv :model_id rows)] :skip-archived true}]
+ opts {:filter-column pk-col
+ :filter-ids (mapv :model_id rows)
+ :skip-archived true}]
;; extract-one must run inside the extract-query reduction, while its ResultSet is open
(into [] (keep (fn [instance]
(when-let [row (id->row (get instance pk-col))]
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
index 31c59c3c935..ecd6b1d9f3e 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
@@ -1173,7 +1173,8 @@
3. Are in one of the provided collections (or descendants)"
[]
(eduction (map (fn [[model ids]]
- (serdes/extract-all model {:where [:in (pk-col model) ids]
+ (serdes/extract-all model {:filter-column (pk-col model)
+ :filter-ids (vec ids)
:skip-archived true})))
cat
(exportable-entities)))
@@ -1185,7 +1186,8 @@
[rows]
(let [by-model (u/group-by :model_type :model_id conj #{} rows)]
(eduction (map (fn [[model ids]]
- (serdes/extract-all model {:where [:in (pk-col model) ids]
+ (serdes/extract-all model {:filter-column (pk-col model)
+ :filter-ids (vec ids)
:skip-archived true})))
cat
by-model)))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/dependency_validation.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/dependency_validation.clj
index f621223f09e..63a4c6b1bf1 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/dependency_validation.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/dependency_validation.clj
@@ -62,7 +62,8 @@
(mapcat resize-batch)
(mapcat (fn [[model batch]]
(serdes/extract-query model (merge opts {:collection-set coll-set
- :where [:in :id batch]}))))
+ :filter-column :id
+ :filter-ids batch}))))
(map entity-deps))
merge-entity-deps
by-model)))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
index 914e26e8b16..b05bf83dcf9 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
@@ -170,7 +170,8 @@
pk-col (fn [model] (if (= model "FieldUserSettings") :field_id :id))
extract-by-ids (fn [[model ids]]
(serdes/extract-all model (merge opts {:collection-set coll-set
- :where [:in (pk-col model) ids]})))
+ :filter-column (pk-col model)
+ :filter-ids (vec ids)})))
extract-all (fn [model]
(serdes/extract-all model (assoc opts :collection-set coll-set)))]
(eduction cat
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/test_util.clj b/enterprise/backend/test/metabase_enterprise/serialization/test_util.clj
index b6df445f635..eae143644b1 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/test_util.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/test_util.clj
@@ -531,10 +531,17 @@
venues-pk-field-id]}]
~@body)))
-(defn extract-one [model-name where]
- (let [where (cond
- (nil? where) true
- (number? where) [:= :id where]
- (string? where) [:= :entity_id where]
- :else where)]
- (u/rfirst (serdes/extract-all model-name {:where where}))))
+(defn extract-one
+ "Extract the first serialized `model-name` entity matching `where`: its primary key, its entity id, a Honey SQL
+ clause, or nil for the first entity of the model. `extract-all` filters by primary key, so a clause is resolved to
+ primary keys here first."
+ [model-name where]
+ (let [model (keyword "model" model-name)
+ pk (first (t2/primary-keys model))
+ ids (cond
+ (nil? where) nil
+ (number? where) [where]
+ (string? where) (t2/select-fn-vec pk [model pk] :entity_id where)
+ :else (t2/select-fn-vec pk [model pk] {:where where}))]
+ (u/rfirst (serdes/extract-all model-name (cond-> {}
+ ids (assoc :filter-column pk :filter-ids ids))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
index 3a98d822ba0..11ea564d05e 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
@@ -1815,14 +1815,14 @@
(is (= {(:id dc1) [s]}
(#'serdes/transform->nested (-> spec :transform :series) {} [dc1])))
(is (=? (assoc dc1 :series [s])
- (u/rfirst (serdes/extract-query "DashboardCard" {:where [:= :id (:id dc1)]})))))
+ (u/rfirst (serdes/extract-query "DashboardCard" {:filter-column :id :filter-ids [(:id dc1)]})))))
(let [spec (serdes/make-spec "Dashboard" nil)]
(is (= {(:id d) [(assoc dc1 :series [s])]}
(#'serdes/transform->nested (-> spec :transform :dashcards) {} [d])))
(is (=? (assoc d
:dashcards [(assoc dc1 :series [s])]
:tabs nil)
- (u/rfirst (serdes/extract-query "Dashboard" {:where [:= :id (:id d)]}))))))))
+ (u/rfirst (serdes/extract-query "Dashboard" {:filter-column :id :filter-ids [(:id d)]}))))))))
(deftest extract-nested-efficient-test
(testing "extract-nested is efficient"
@@ -1846,7 +1846,7 @@
:tabs nil)}
(into #{} (map (fn [dashboard]
(update dashboard :dashcards #(sort-by :id %))))
- (serdes/extract-query "Dashboard" {:where [:in :id [(:id d1) (:id d2)]]}))))
+ (serdes/extract-query "Dashboard" {:filter-column :id :filter-ids [(:id d1) (:id d2)]}))))
;; 1 per dashboard/dashcard/series/tabs
(is (= 4 (qc)))))))
@@ -1860,8 +1860,9 @@
:card_id (:id c1)})))]
(t2/with-call-count [qc]
(is (=? [(assoc d :dashcards dcs)]
- (into [] (serdes/extract-query "Dashboard" {:batch-limit 5
- :where [:= :id (:id d)]}))))
+ (into [] (serdes/extract-query "Dashboard" {:batch-limit 5
+ :filter-column :id
+ :filter-ids [(:id d)]}))))
;; query count breakdown:
;; - 1 for dashboard
;; - 1 for tabs, there are none
diff --git a/src/metabase/collections/db.clj b/src/metabase/collections/db.clj
index c12d568a1c6..6e8ff29c22f 100644
--- a/src/metabase/collections/db.clj
+++ b/src/metabase/collections/db.clj
@@ -127,23 +127,25 @@
`children-location-prefix` (see `metabase.collections.models.collection/children-location`), excluding other
users' Personal Collections (Personal Collections owned by `current-user-id` are still included). When
`archived?` is given (true or false, not nil), further restricted to that archived status.
- `additional-where-clauses` are ANDed in as-is (nil, from a caller with none to add, is treated as empty); used
- by callers that need a permission-filter builder like `visible-collection-filter-clause`, which lives in
- `metabase.collections.models.collection` and so can't be called from here without a require cycle (that
- namespace already requires this one)."
+
+ `visibility-clause` is the one Honey SQL argument left in this namespace, and is always a
+ `metabase.collections.models.collection/visible-collection-filter-clause`. That builder needs this namespace (it
+ looks up the Trash, the user's personal Collection subtree, and their root-Collection permission), so this
+ namespace cannot call it back without a require cycle — building the clause here means first moving the whole
+ Collection-visibility machinery below this namespace."
[children-location-prefix :- :string
current-user-id :- [:maybe ::lib.schema.id/user]
archived? :- [:maybe :boolean]
- additional-where-clauses :- [:maybe [:sequential vector?]]]
+ visibility-clause :- [:maybe vector?]]
(t2/select [:model/Collection :name :id :location :description]
- {:where (into [:and
- [:like :location (str children-location-prefix "%")]
- [:or
- [:= :personal_owner_id nil]
- [:= :personal_owner_id current-user-id]]
- (when (some? archived?)
- [:= :archived archived?])]
- additional-where-clauses)}))
+ {:where [:and
+ [:like :location (str children-location-prefix "%")]
+ [:or
+ [:= :personal_owner_id nil]
+ [:= :personal_owner_id current-user-id]]
+ (when (some? archived?)
+ [:= :archived archived?])
+ visibility-clause]}))
(mu/defn descendant-summaries-with-type
"The name, ID, location, description, and type of the Collections directly under any of `location-prefixes`
@@ -155,15 +157,40 @@
(into [:or] (map (fn [prefix] [:like :location prefix])) location-prefixes)
[:or [:= :personal_owner_id nil] [:= :personal_owner_id current-user-id]]]}))
-(mu/defn effective-children-where
- "The ID, name, description, and type of the Collections matching the Honey SQL `where`."
- [where :- [:maybe vector?]]
- (t2/select [:model/Collection :id :name :description :type] {:where where}))
+(mu/defn effective-children
+ "The ID, name, description, and type of the Collections matching `effective-children-clause`, a
+ `metabase.collections.models.collection/effective-children-where-clause`.
+
+ Like [[descendant-summaries]]'s `visibility-clause`, that builder needs this namespace and so cannot be called
+ from here."
+ [effective-children-clause :- [:maybe vector?]]
+ (t2/select [:model/Collection :id :name :description :type] {:where effective-children-clause}))
(mu/defn collections-for-serdes-reducible
- "Reducible Collections matching the Honey SQL `where` in stable storage order."
- [where :- [:maybe vector?]]
- (t2/reducible-select :model/Collection {:where where, :order-by serdes/stable-storage-order}))
+ "A reducible of the Collections to export via serdes, in stable storage order (which keeps filename de-dup suffixes
+ stable across exports, see GHY-3754). The Trash is never exported, nor are archived Collections when
+ `skip-archived?`. When `collection-set` is non-empty only those Collections are exported (nil in the set counts as
+ the root collection); otherwise every non-personal Collection is. `filter-column` and `filter-ids`, when given,
+ further restrict the export to the rows whose `filter-column` is one of `filter-ids`."
+ [collection-set :- [:maybe [:or [:set [:maybe ::lib.schema.id/collection]] [:sequential [:maybe ::lib.schema.id/collection]]]]
+ skip-archived? :- [:maybe :boolean]
+ filter-column :- [:maybe :keyword]
+ filter-ids :- [:maybe [:sequential [:maybe [:or :int :string]]]]]
+ (t2/reducible-select :model/Collection
+ {:where [:and
+ (when skip-archived? [:not :archived])
+ (if (seq collection-set)
+ [:or
+ [:in :id collection-set]
+ (when (some nil? collection-set)
+ [:= :id nil])]
+ [:= :personal_owner_id nil])
+ [:or
+ [:= :type nil]
+ [:not= :type collections.schema/trash-collection-type]]
+ (when filter-column
+ [:in filter-column filter-ids])]
+ :order-by serdes/stable-storage-order}))
(mu/defn collection-count-by-ids
"The number of Collections among `collection-ids`."
diff --git a/src/metabase/collections/models/collection.clj b/src/metabase/collections/models/collection.clj
index 587827335ee..804ec3fc2c2 100644
--- a/src/metabase/collections/models/collection.clj
+++ b/src/metabase/collections/models/collection.clj
@@ -12,6 +12,7 @@
[metabase.audit-app.core :as audit]
[metabase.collections.db :as collections.db]
[metabase.collections.models.collection.root :as collection.root]
+ [metabase.collections.schema :as collections.schema]
[metabase.config.core :as config :refer [*request-id*]]
[metabase.events.core :as events]
[metabase.models.interface :as mi]
@@ -60,7 +61,7 @@
(def ^:constant trash-collection-type
"The value of the `:type` field for the Trash collection that holds archived items."
- "trash")
+ collections.schema/trash-collection-type)
(def ^:constant library-collection-type
"The value of the `:type` field for library collections."
@@ -1121,13 +1122,17 @@
other users' Personal Collections, regardless of whether we should actually be allowed to see them (e.g. admins
have perms for all Collections) -- this is done to keep the Root Collection View for admins from getting crazily
cluttered with Personal Collections belonging to other users. `archived?`, if given (true or false), restricts
- to Collections with that archived status. `additional-honeysql-where-clauses` are for permission-filter builders
- like [[visible-collection-filter-clause]]."
+ to Collections with that archived status. `visibility-clause`, if given, is
+ a [[visible-collection-filter-clause]] to restrict the descendants further."
([collection :- CollectionWithLocationAndIDOrRoot]
(descendants-flat collection nil))
- ([collection :- CollectionWithLocationAndIDOrRoot, archived? :- [:maybe :boolean], & additional-honeysql-where-clauses]
+ ([collection :- CollectionWithLocationAndIDOrRoot, archived? :- [:maybe :boolean]]
+ (descendants-flat collection archived? nil))
+ ([collection :- CollectionWithLocationAndIDOrRoot
+ archived? :- [:maybe :boolean]
+ visibility-clause :- [:maybe vector?]]
(collections.db/descendant-summaries
- (children-location collection) api/*current-user-id* archived? additional-honeysql-where-clauses)))
+ (children-location collection) api/*current-user-id* archived? visibility-clause)))
(mu/defn descendants-flat-for :- [:sequential CollectionWithLocationAndIDOrRoot]
"Like [[descendants-flat]], but returns the descendants of *any* of
@@ -1215,20 +1220,18 @@
:where (apply effective-children-where-clause collection :col visibility-config additional-honeysql-where-clauses)})
(mu/defn- effective-children* :- [:set (ms/InstanceOf :model/Collection)]
- [collection :- CollectionWithLocationAndIDOrRoot & additional-honeysql-where-clauses]
- (set (collections.db/effective-children-where (apply effective-children-where-clause
- collection
- (t2/table-name :model/Collection)
- default-visibility-config
- additional-honeysql-where-clauses))))
+ [collection :- CollectionWithLocationAndIDOrRoot]
+ (set (collections.db/effective-children (effective-children-where-clause collection
+ (t2/table-name :model/Collection)
+ default-visibility-config))))
(mi/define-simple-hydration-method effective-children
:effective_children
"Get the descendant Collections of `collection` that should be presented to the current User as direct children of
this Collection. See documentation for [[metabase.collections.models.collection/effective-children-query]] for more
details."
- [collection & additional-honeysql-where-clauses]
- (apply effective-children* collection additional-honeysql-where-clauses))
+ [collection]
+ (effective-children* collection))
;;; -------------------------------------------------- Remote Sync -------------------------------------------------------
@@ -2033,26 +2036,8 @@
[:not= (maybe-alias :namespace) ^:allow-raw-sql [:inline "analytics"]]]
[:not (maybe-alias :is_sample)]]))
-(defmethod serdes/extract-query "Collection" [_model {:keys [collection-set where skip-archived]}]
- (let [not-trash-clause [:or
- [:= :type nil]
- [:not= :type trash-collection-type]]]
- (if (seq collection-set)
- ;; stable filename de-dup suffixes across exports, see GHY-3754
- (collections.db/collections-for-serdes-reducible
- [:and
- (when skip-archived [:not :archived])
- [:or
- [:in :id collection-set]
- (when (some nil? collection-set) [:= :id nil])]
- not-trash-clause
- (or where true)])
- (collections.db/collections-for-serdes-reducible
- [:and
- (when skip-archived [:not :archived])
- [:= :personal_owner_id nil]
- not-trash-clause
- (or where true)]))))
+(defmethod serdes/extract-query "Collection" [_model {:keys [collection-set filter-column filter-ids skip-archived]}]
+ (collections.db/collections-for-serdes-reducible collection-set skip-archived filter-column filter-ids))
(defmethod serdes/deserialization-dependencies "Collection"
[{:keys [parent_id]}]
diff --git a/src/metabase/collections/schema.clj b/src/metabase/collections/schema.clj
index 71525758310..4a3863c4aa9 100644
--- a/src/metabase/collections/schema.clj
+++ b/src/metabase/collections/schema.clj
@@ -4,6 +4,10 @@
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]))
+(def ^:constant trash-collection-type
+ "The value of the `:type` field for the Trash collection that holds archived items."
+ "trash")
+
(mr/def ::CollectionContentModel [:enum "card" "dataset" "metric"])
(mr/def ::CollectionType [:enum
diff --git a/src/metabase/documents/db.clj b/src/metabase/documents/db.clj
index 450191487d7..16511ee7cd1 100644
--- a/src/metabase/documents/db.clj
+++ b/src/metabase/documents/db.clj
@@ -30,6 +30,32 @@
[:= :archived false]
[:= :exploration_id nil]]}))
+(mu/defn documents-for-serdes-reducible
+ "A reducible of the Documents to export via serdes: those whose `:collection_id` is in `collection-set` (nil in the
+ set counts as the root collection; an empty or nil set means every collection), further restricted to the rows
+ whose `filter-column` is one of `filter-ids` when `filter-column` is given, and ordered ascending by
+ `order-columns` (unordered when empty).
+
+ Exploration documents are always excluded: such a document is not first-class content, it is reachable only through
+ its owning exploration, its body embeds values computed under its creator's data-access lens, and
+ `:exploration_id` is in the serdes spec's `:skip` list — so an exported document would import as an ordinary,
+ ungated document detached from any exploration."
+ [collection-set :- [:maybe [:or [:set [:maybe ::lib.schema.id/collection]] [:sequential [:maybe ::lib.schema.id/collection]]]]
+ filter-column :- [:maybe :keyword]
+ filter-ids :- [:maybe [:sequential [:maybe [:or :int :string]]]]
+ order-columns :- [:maybe [:sequential :keyword]]]
+ (t2/reducible-select :model/Document
+ (cond-> {:where [:and
+ (when (seq collection-set)
+ [:or
+ [:in :collection_id collection-set]
+ (when (some nil? collection-set)
+ [:= :collection_id nil])])
+ (when filter-column
+ [:in filter-column filter-ids])
+ [:= :exploration_id nil]]}
+ (seq order-columns) (assoc :order-by (mapv (fn [column] [column :asc]) order-columns)))))
+
(mu/defn insert-document!
"Insert the Document `row` and return its id."
[row :- ::documents.schema/document.update]
diff --git a/src/metabase/documents/models/document.clj b/src/metabase/documents/models/document.clj
index f7d429decf0..eb3926a6f26 100644
--- a/src/metabase/documents/models/document.clj
+++ b/src/metabase/documents/models/document.clj
@@ -344,16 +344,11 @@
:archived_directly false}})
(defmethod serdes/extract-query "Document"
- [model-name opts]
- ;; An exploration document is not first-class content: it is reachable only through its owning
- ;; exploration, its body embeds values computed under its creator's data-access lens, and
- ;; `:exploration_id` is in this spec's `:skip` list — so an exported document would import as an
- ;; ordinary, ungated document detached from any exploration.
- ((get-method serdes/extract-query :default)
- model-name
- (update opts :where (fn [where]
- (let [clause [:= :exploration_id nil]]
- (if where [:and where clause] clause))))))
+ [model-name {:keys [collection-set filter-column filter-ids] :as opts}]
+ (documents.db/documents-for-serdes-reducible collection-set
+ filter-column
+ filter-ids
+ (serdes/extract-order-columns model-name opts)))
(defn- document-deps
[{:keys [content_type] :as document}]
diff --git a/src/metabase/metrics/api.clj b/src/metabase/metrics/api.clj
index 34f07b692fa..ffe18200adb 100644
--- a/src/metabase/metrics/api.clj
+++ b/src/metabase/metrics/api.clj
@@ -11,7 +11,6 @@
[metabase.metrics.db :as metrics.db]
[metabase.metrics.dimension :as metrics.dimension]
[metabase.metrics.permissions :as metrics.perms]
- [metabase.queries.core :as queries]
[metabase.query-processor.core :as qp]
[metabase.query-processor.middleware.permissions :as qp.perms]
[metabase.query-processor.pipeline :as qp.pipeline]
@@ -52,10 +51,10 @@
[:result_column_name {:optional true} [:maybe :string]]]])
(defn- count-metrics []
- (metrics.db/metric-card-count (queries/visible-metric-cards-where-clause)))
+ (metrics.db/visible-metric-card-count))
(defn- select-metrics [limit offset]
- (-> (metrics.db/metric-cards-page (queries/visible-metric-cards-where-clause) limit offset)
+ (-> (metrics.db/visible-metric-cards-page limit offset)
(t2/hydrate :collection)))
(api.macros/defendpoint :get "/"
diff --git a/src/metabase/metrics/db.clj b/src/metabase/metrics/db.clj
index f165acf3988..02631488e6f 100644
--- a/src/metabase/metrics/db.clj
+++ b/src/metabase/metrics/db.clj
@@ -2,24 +2,36 @@
"Application database queries for the metrics module. Every function here is a direct Toucan 2 call with no
additional logic, so the rest of the module only touches `toucan2.core` for hydration."
(:require
+ [metabase.collections.models.collection :as collection]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
[toucan2.core :as t2]))
-(mu/defn metric-card-count
- "The number of Cards matching the Honey SQL `where` clause."
- [where :- [:maybe vector?]]
- (t2/count :model/Card {:where where}))
+(defn- visible-metric-cards-where
+ "The `:where` clause selecting the unarchived metric Cards the current user can read. Mirrors
+ `metabase.queries.core/visible-metric-cards-where-clause`, which this namespace cannot call: the queries module
+ requires the metrics module, so requiring it back here would close a load cycle."
+ []
+ [:and
+ [:= :type "metric"]
+ [:= :archived false]
+ (collection/visible-collection-filter-clause :collection_id {:include-trash-collection? false
+ :include-archived-items :exclude
+ :permission-level :read})])
-(mu/defn metric-cards-page
- "Up to `limit` id, name, description, and Collection id rows from `offset` of the Cards matching the Honey SQL
- `where` clause, in name order."
- [where :- [:maybe vector?]
- limit :- ms/PositiveInt
+(mu/defn visible-metric-card-count
+ "The number of unarchived metric Cards the current user can read."
+ []
+ (t2/count :model/Card {:where (visible-metric-cards-where)}))
+
+(mu/defn visible-metric-cards-page
+ "Up to `limit` id, name, description, and Collection id rows from `offset` of the unarchived metric Cards the
+ current user can read, in name order."
+ [limit :- ms/PositiveInt
offset :- ms/IntGreaterThanOrEqualToZero]
(t2/select [:model/Card :id :name :description :collection_id]
- {:where where
+ {:where (visible-metric-cards-where)
:order-by [[:name :asc]]
:limit limit
:offset offset}))
diff --git a/src/metabase/models/db.clj b/src/metabase/models/db.clj
index a45dd038f62..81dcb75a171 100644
--- a/src/metabase/models/db.clj
+++ b/src/metabase/models/db.clj
@@ -53,32 +53,34 @@
(t2/select-one model (t2.identity-query/identity-query [row-map])))
(mu/defn entities-reducible
- "A reducible of the `model` rows additionally matching the Honey SQL `extra-where` when given — the serdes
- extract-query nested-fetch hook passes a dynamic foreign-key-column + id-list condition here that varies per
- model/transform and can't be expressed as fixed data — ordered by `order-by` (or unordered when nil)."
- [model :- [:or :keyword symbol?]
- extra-where :- [:maybe vector?]
- order-by :- [:maybe vector?]]
+ "A reducible of the `model` rows whose `filter-column` is one of `filter-ids` (every row when `filter-column` is
+ nil), ordered ascending by `order-columns` (unordered when empty)."
+ [model :- [:or :keyword symbol?]
+ filter-column :- [:maybe :keyword]
+ filter-ids :- [:maybe [:sequential [:maybe [:or :int :string]]]]
+ order-columns :- [:maybe [:sequential :keyword]]]
(t2/reducible-select model (cond-> {}
- extra-where (assoc :where extra-where)
- order-by (assoc :order-by order-by))))
+ filter-column (assoc :where [:in filter-column filter-ids])
+ (seq order-columns) (assoc :order-by (mapv (fn [column] [column :asc]) order-columns)))))
(mu/defn entities-in-collections-reducible
"A reducible of the `model` rows whose `:collection_id` is in `collection-set` (nil in the set counts as the root
- collection), additionally matching the Honey SQL `extra-where` when given (see [[entities-reducible]] for why this
- stays a raw clause), ordered by `order-by` (or unordered when nil)."
+ collection) and whose `filter-column` is one of `filter-ids` (unrestricted when `filter-column` is nil), ordered
+ ascending by `order-columns` (unordered when empty)."
[model :- [:or :keyword symbol?]
collection-set :- [:or [:set [:maybe ms/PositiveInt]] [:sequential [:maybe ms/PositiveInt]]]
- extra-where :- [:maybe vector?]
- order-by :- [:maybe vector?]]
+ filter-column :- [:maybe :keyword]
+ filter-ids :- [:maybe [:sequential [:maybe [:or :int :string]]]]
+ order-columns :- [:maybe [:sequential :keyword]]]
(t2/reducible-select model
(cond-> {:where [:and
[:or
[:in :collection_id collection-set]
(when (some nil? collection-set)
[:= :collection_id nil])]
- extra-where]}
- order-by (assoc :order-by order-by))))
+ (when filter-column
+ [:in filter-column filter-ids])]}
+ (seq order-columns) (assoc :order-by (mapv (fn [column] [column :asc]) order-columns)))))
(mu/defn table-names-reducible
"A reducible of the id, name, and display name of every Table."
diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.clj
index a9f638d391c..85aa3371e3b 100644
--- a/src/metabase/models/serialization.clj
+++ b/src/metabase/models/serialization.clj
@@ -452,7 +452,10 @@
(defn- transform->nested [transform opts batch]
(let [backward-fk (:backward-fk transform)
entities (-> (extract-query (name (:model transform))
- (assoc opts :where [:in backward-fk (map :id batch)] ::nested-fetch true))
+ (assoc opts
+ :filter-column backward-fk
+ :filter-ids (mapv :id batch)
+ ::nested-fetch true))
t2.realize/realize)]
(group-by backward-fk entities)))
@@ -492,26 +495,33 @@
(let [serialized? (into (set (:copy spec)) (keys (:transform spec)))]
(not-empty (filterv (comp serialized? first) stable-storage-order))))
+(defn extract-order-columns
+ "The ascending order columns for `model-name`'s extract query: [[stable-storage-order]] restricted to the columns the
+ model's spec actually serializes.
+
+ Empty for a nested fetch (e.g. a Dashboard's DashboardCards): those are embedded as lists inside the parent's file
+ rather than written to their own files, so they keep their natural order and are left untouched."
+ [model-name opts]
+ (when-not (::nested-fetch opts)
+ (mapv first (stable-storage-order-by (*make-spec* model-name opts)))))
+
(defn extract-query-collections
"Helper for the common (but not default) [[extract-query]] case of fetching everything that isn't in a personal
collection."
- [model {:keys [collection-set where] :as opts}]
- (let [spec (*make-spec* (name model) opts)
- ;; Nested fetches (e.g. a Dashboard's DashboardCards) are embedded as lists inside the parent's file
- ;; rather than written to their own files, so they keep their natural order and are left untouched.
- order-by (when-not (::nested-fetch opts)
- (stable-storage-order-by spec))]
+ [model {:keys [collection-set filter-column filter-ids] :as opts}]
+ (let [spec (*make-spec* (name model) opts)
+ order-columns (extract-order-columns (name model) opts)]
(if (or (empty? collection-set)
(nil? (-> spec :transform :collection_id)))
;; either no collections specified or our model has no collection
- (models.db/entities-reducible model where order-by)
- (models.db/entities-in-collections-reducible model collection-set where order-by))))
+ (models.db/entities-reducible model filter-column filter-ids order-columns)
+ (models.db/entities-in-collections-reducible model collection-set filter-column filter-ids order-columns))))
(defmethod extract-query :default [model-name opts]
(let [spec (*make-spec* model-name opts)
nested? (some ::nested (vals (:transform spec)))]
(cond->> (extract-query-collections (keyword "model" model-name) opts)
- nested? (extract-reducible-nested model-name (dissoc opts :where)))))
+ nested? (extract-reducible-nested model-name (dissoc opts :filter-column :filter-ids)))))
(defmulti descendants
"Returns map of `{[model-name database-id] {initiating-model id}}` for all entities contained or used by this
diff --git a/src/metabase/native_query_snippets/db.clj b/src/metabase/native_query_snippets/db.clj
index 56a9f8e1320..9314cfe94d8 100644
--- a/src/metabase/native_query_snippets/db.clj
+++ b/src/metabase/native_query_snippets/db.clj
@@ -53,14 +53,15 @@
(t2/select-one-fn :collection_id :model/NativeQuerySnippet :id id))
(mu/defn exportable-snippets
- "A reducible of the NativeQuerySnippets to export via serdes: unarchived when `skip-archived?`, and either
- in one of `collection-ids`, uncollected when `include-root?`, or matching the serdes-supplied
- `extra-condition` — an additional condition the caller widens the export scope with (e.g. also export as
- a Card dependency, regardless of collection), or nil — in stable export order."
+ "A reducible of the NativeQuerySnippets to export via serdes: unarchived when `skip-archived?`, and either in one of
+ `collection-ids`, uncollected when `include-root?`, or — when `filter-column` is given — one of the rows whose
+ `filter-column` is in `filter-ids`, which widens the export scope past the collections (e.g. a snippet exported as
+ a Card dependency, regardless of collection). In stable export order."
[collection-ids :- [:maybe [:sequential ::lib.schema.id/collection]]
- include-root? :- :boolean
+ include-root? :- :boolean
skip-archived? :- [:maybe :boolean]
- extra-condition :- [:maybe vector?]]
+ filter-column :- [:maybe :keyword]
+ filter-ids :- [:maybe [:sequential [:maybe [:or :int :string]]]]]
(t2/reducible-select :model/NativeQuerySnippet
(cond-> {:where [:and
(when skip-archived? [:not :archived])
@@ -68,4 +69,4 @@
(when (seq collection-ids) [:in :collection_id collection-ids])
(when include-root? [:= :collection_id nil])]]
:order-by serdes/stable-storage-order}
- extra-condition (sql.helpers/where :or extra-condition))))
+ filter-column (sql.helpers/where :or [:in filter-column filter-ids]))))
diff --git a/src/metabase/native_query_snippets/models/native_query_snippet.clj b/src/metabase/native_query_snippets/models/native_query_snippet.clj
index e2954ee218a..82408c7281a 100644
--- a/src/metabase/native_query_snippets/models/native_query_snippet.clj
+++ b/src/metabase/native_query_snippets/models/native_query_snippet.clj
@@ -156,7 +156,7 @@
;;; ------------------------------------------------- Serialization --------------------------------------------------
-(defmethod serdes/extract-query "NativeQuerySnippet" [_ {:keys [collection-set where skip-archived]}]
+(defmethod serdes/extract-query "NativeQuerySnippet" [_ {:keys [collection-set filter-column filter-ids skip-archived]}]
;; NativeQuerySnippets live in their own special collections, so the logic is the following:
;; - you either are exporting one of those
;; - or it was requested as a dependency of some Card, so export it regardless of collection
@@ -164,7 +164,8 @@
(not-empty (remove nil? collection-set))
(boolean (some nil? collection-set))
skip-archived
- where))
+ filter-column
+ filter-ids))
(defmethod serdes/make-spec "NativeQuerySnippet" [_model-name _opts]
{:copy [:archived :content :description :entity_id :name]
diff --git a/src/metabase/permissions/db.clj b/src/metabase/permissions/db.clj
index 3c7f1d46af6..3f40df968c1 100644
--- a/src/metabase/permissions/db.clj
+++ b/src/metabase/permissions/db.clj
@@ -357,9 +357,10 @@
(t2/select-fn-set :name :model/PermissionsGroup :name [:like pattern]))
(mu/defn group-members
- "The active Users in the PermissionsGroups with `group-ids`, with the optional extra `group-manager-column`."
- [group-ids :- [:sequential ms/PositiveInt]
- group-manager-column :- [:maybe vector?]]
+ "The active Users in the PermissionsGroups with `group-ids`. When `include-group-manager?` is true each row also
+ carries the membership's `:is_group_manager` flag."
+ [group-ids :- [:sequential ms/PositiveInt]
+ include-group-manager? :- :boolean]
(t2/select :model/User {:select (cond-> [:u.id
[:u.id :user_id]
:u.first_name
@@ -369,7 +370,8 @@
:u.type
:pgm.group_id
[:pgm.id :membership_id]]
- group-manager-column (conj group-manager-column))
+ include-group-manager?
+ (conj [:pgm.is_group_manager :is_group_manager]))
:from [[:core_user :u]]
:left-join [[:permissions_group_membership :pgm] [:= :u.id :pgm.user_id]]
:where [:and
diff --git a/src/metabase/permissions/models/permissions_group.clj b/src/metabase/permissions/models/permissions_group.clj
index 4334f36db0c..da0f53a9f1a 100644
--- a/src/metabase/permissions/models/permissions_group.clj
+++ b/src/metabase/permissions/models/permissions_group.clj
@@ -156,8 +156,7 @@
groups k
;; `user_id` in the result is for legacy reasons, we should remove it
#(group-by :group_id (permissions.db/group-members (map :id groups)
- (when (premium-features/enable-advanced-permissions?)
- [:pgm.is_group_manager :is_group_manager])))
+ (boolean (premium-features/enable-advanced-permissions?))))
:id
{:default []}))
diff --git a/src/metabase/queries/db.clj b/src/metabase/queries/db.clj
index ef4af5ec333..8b345243485 100644
--- a/src/metabase/queries/db.clj
+++ b/src/metabase/queries/db.clj
@@ -14,6 +14,41 @@
;;; ------------------------------------------------ Cards ------------------------------------------------
+(def ^:private not-in-exploration-document
+ "The `:where` fragment excluding Cards that belong to an exploration Summary document.
+
+ Such a Card is materialized by the Summary itself — its `name` and `dataset_query` are copied from the
+ `ExplorationQuery` it renders, so they carry dimension values discovered under the creator's data-access lens. Its
+ parent Document is never serialized (see `metabase.documents.models.document`'s `extract-query`), and this Card's
+ `deserialization-dependencies` name that Document, so exporting the Card without it would leave a dangling
+ reference even setting the lens question aside."
+ [:or
+ [:= :document_id nil]
+ [:in :document_id ^:allow-subquery {:select [:id]
+ :from [:document]
+ :where [:= :exploration_id nil]}]])
+
+(mu/defn cards-for-serdes-reducible
+ "A reducible of the Cards to export via serdes: those whose `:collection_id` is in `collection-set` (nil in the set
+ counts as the root collection; an empty or nil set means every collection), further restricted to the rows whose
+ `filter-column` is one of `filter-ids` when `filter-column` is given, never a Card materialized by an exploration
+ Summary document, and ordered ascending by `order-columns` (unordered when empty)."
+ [collection-set :- [:maybe [:or [:set [:maybe ::lib.schema.id/collection]] [:sequential [:maybe ::lib.schema.id/collection]]]]
+ filter-column :- [:maybe :keyword]
+ filter-ids :- [:maybe [:sequential [:maybe [:or :int :string]]]]
+ order-columns :- [:maybe [:sequential :keyword]]]
+ (t2/reducible-select :model/Card
+ (cond-> {:where [:and
+ (when (seq collection-set)
+ [:or
+ [:in :collection_id collection-set]
+ (when (some nil? collection-set)
+ [:= :collection_id nil])])
+ (when filter-column
+ [:in filter-column filter-ids])
+ not-in-exploration-document]}
+ (seq order-columns) (assoc :order-by (mapv (fn [column] [column :asc]) order-columns)))))
+
(mu/defn card
"The Card with `card-id`, or nil."
[card-id :- ::lib.schema.id/card]
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index 2fcd9f5ea4b..954dacf118a 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -1476,29 +1476,12 @@
(for [snippet-id snippets]
{["NativeQuerySnippet" snippet-id] {"Card" id}})))))
-(def ^:private not-in-exploration-document
- "HoneySQL predicate: this Card does not belong to an exploration Summary document.
-
- Such a Card is materialized by the Summary itself — its `name` and `dataset_query` are copied
- from the `ExplorationQuery` it renders, so they carry dimension values discovered under the
- creator's data-access lens. Its parent Document is never serialized (see
- `metabase.documents.models.document`'s `extract-query`), and this Card's
- `deserialization-dependencies` name that Document, so exporting the Card without it would leave a
- dangling reference even setting the lens question aside."
- [:or
- [:= :document_id nil]
- [:in :document_id ^:allow-subquery {:select [:id]
- :from [:document]
- :where [:= :exploration_id nil]}]])
-
(defmethod serdes/extract-query "Card"
- [model-name opts]
- ((get-method serdes/extract-query :default)
- model-name
- (update opts :where (fn [where]
- (if where
- [:and where not-in-exploration-document]
- not-in-exploration-document)))))
+ [model-name {:keys [collection-set filter-column filter-ids] :as opts}]
+ (queries.db/cards-for-serdes-reducible collection-set
+ filter-column
+ filter-ids
+ (serdes/extract-order-columns model-name opts)))
(defmethod serdes/serialization-dependencies "Card" [_model-name card]
(card-deps true card))
diff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.clj
index 933907c263a..911fb4527c4 100644
--- a/src/metabase/search/appdb/index.clj
+++ b/src/metabase/search/appdb/index.clj
@@ -8,7 +8,6 @@
[metabase.search.appdb.specialization.api :as specialization]
[metabase.search.appdb.specialization.h2 :as h2]
[metabase.search.appdb.specialization.postgres :as postgres]
- [metabase.search.config :as search.config]
[metabase.search.db :as search.db]
[metabase.search.engine :as search.engine]
[metabase.search.ingestion :as search.ingestion]
@@ -140,54 +139,13 @@
(catch Exception e
(log/warnf "Failed to clean up obsolete indexes: %s" (ex-message e))))))
-(defn- ->db-type [t]
- (get {:pk :int, :timestamp :timestamp-with-time-zone} t t))
-
-(defn- ->db-column [c]
- (or (get {:id :model_id
- :created-at :model_created_at
- :updated-at :model_updated_at}
- c)
- (keyword (u/->snake_case_en (name c)))))
-
-(def ^:private not-null
- #{:archived :name})
-
-(def ^:private default
- {:archived false})
-
-;; If this fails, we'll need to increase the size of :model below
-(assert (>= 32 (transduce (map (comp count name)) max 0 search.config/all-models)))
-
-(def ^:private base-schema
- (into [[:model [:varchar 32] :not-null]
- [:display_data :text :not-null]
- [:legacy_input :text :not-null]
- ;; useful for tracking the speed and age of the index
- [:created_at :timestamp-with-time-zone
- [:default ^:allow-raw-sql [:raw "CURRENT_TIMESTAMP"]]
- :not-null]
- [:updated_at :timestamp-with-time-zone :not-null]]
- (keep (fn [[k t]]
- (when t
- (into [(->db-column k) (->db-type t)]
- (concat
- (when (not-null k)
- [:not-null])
- (when-some [d (default k)]
- [[:default d]]))))))
- search.spec/attr-types))
-
(defn create-table!
"Create an index table with the given name. Should fail if it already exists."
[table-name]
;; Create with a separate transaction so that postgresql will complete the index creations before returning,
;; even when already running in a transaction
(t2/with-transaction [_ (mdb/app-db)]
- (search.db/create-search-index-table! table-name (specialization/table-schema base-schema))
- (let [table-name (name table-name)]
- (doseq [stmt (specialization/post-create-statements table-name table-name)]
- (search.db/run-search-index-statement! stmt)))))
+ (search.db/create-search-index-table! table-name)))
(defn maybe-create-pending!
"Create a search index table if one doesn't exist. Record and return the name of the table, regardless."
diff --git a/src/metabase/search/appdb/index_schema.clj b/src/metabase/search/appdb/index_schema.clj
new file mode 100644
index 00000000000..f90a1db4a0d
--- /dev/null
+++ b/src/metabase/search/appdb/index_schema.clj
@@ -0,0 +1,46 @@
+(ns metabase.search.appdb.index-schema
+ "The engine-independent column definitions of the search index table. Lives below `metabase.search.db`, which
+ creates the table, so it can't be part of `metabase.search.appdb.index`."
+ (:require
+ [metabase.search.config :as search.config]
+ [metabase.search.spec :as search.spec]
+ [metabase.util :as u]))
+
+(defn- ->db-type [t]
+ (get {:pk :int, :timestamp :timestamp-with-time-zone} t t))
+
+(defn- ->db-column [c]
+ (or (get {:id :model_id
+ :created-at :model_created_at
+ :updated-at :model_updated_at}
+ c)
+ (keyword (u/->snake_case_en (name c)))))
+
+(def ^:private not-null
+ #{:archived :name})
+
+(def ^:private default
+ {:archived false})
+
+;; If this fails, we'll need to increase the size of :model below
+(assert (>= 32 (transduce (map (comp count name)) max 0 search.config/all-models)))
+
+(def base-schema
+ "The columns of the search index table, before the active engine specialization adds its own."
+ (into [[:model [:varchar 32] :not-null]
+ [:display_data :text :not-null]
+ [:legacy_input :text :not-null]
+ ;; useful for tracking the speed and age of the index
+ [:created_at :timestamp-with-time-zone
+ [:default ^:allow-raw-sql [:raw "CURRENT_TIMESTAMP"]]
+ :not-null]
+ [:updated_at :timestamp-with-time-zone :not-null]]
+ (keep (fn [[k t]]
+ (when t
+ (into [(->db-column k) (->db-type t)]
+ (concat
+ (when (not-null k)
+ [:not-null])
+ (when-some [d (default k)]
+ [[:default d]]))))))
+ search.spec/attr-types))
diff --git a/src/metabase/search/db.clj b/src/metabase/search/db.clj
index 3d486009005..22a382a5caf 100644
--- a/src/metabase/search/db.clj
+++ b/src/metabase/search/db.clj
@@ -5,6 +5,7 @@
[honey.sql.helpers :as sql.helpers]
[metabase.app-db.core :as mdb]
[metabase.lib.schema.id :as lib.schema.id]
+ [metabase.search.appdb.index-schema :as index-schema]
[metabase.search.appdb.query :as appdb.query]
[metabase.search.appdb.scoring :as search.scoring]
[metabase.search.appdb.specialization.api :as specialization]
@@ -18,8 +19,13 @@
(mu/defn spec-index-reducible-rows
"A reducible of the indexable rows of `search-model` (see `metabase.search.ingestion.query/spec-index-query`)
- matching `where-clause`: the search-spec generated `:where` fragment of `metabase.search.spec/search-models-to-update`,
- or nil for every row."
+ matching `where-clause`, or every row when it is nil.
+
+ `where-clause` is the one Honey SQL argument left in this namespace. It is the search-spec generated `:where`
+ fragment of `metabase.search.spec/search-models-to-update`, which is the payload of the ingestion queue itself:
+ `metabase.search.util/impossible-condition?` drops entries by inspecting it, `metabase.search.ingestion` ORs the
+ distinct clauses of a batch together and parses `[model id]` pairs back out of them to decide what to purge.
+ Turning it into plain data means redesigning that queue, not restructuring a caller."
[search-model :- :string
where-clause :- [:maybe vector?]]
(mdb/streaming-reducible-query (ingestion.query/spec-index-query-where search-model where-clause)))
@@ -107,17 +113,15 @@
(t2/query (sql.helpers/drop-table table-name)))
(mu/defn create-search-index-table!
- "Create the search index table named `table-name` with `columns` (the Honey SQL column definitions built by the
- active search engine specialization)."
- [table-name :- [:or :keyword :string]
- columns :- [:sequential vector?]]
+ "Create the search index table named `table-name`: the columns of `metabase.search.appdb.index-schema/base-schema`
+ as shaped by the active search engine specialization, then that specialization's post-creation statements (index
+ creation and the like)."
+ [table-name :- [:or :keyword :string]]
(t2/query (-> (sql.helpers/create-table table-name)
- (sql.helpers/with-columns columns))))
-
-(mu/defn run-search-index-statement!
- "Run a single post-creation SQL statement (e.g. an index creation) for a search index table."
- [statement :- [:or :string vector? :map]]
- (t2/query statement))
+ (sql.helpers/with-columns (specialization/table-schema index-schema/base-schema))))
+ (let [table-name (name table-name)]
+ (doseq [statement (specialization/post-create-statements table-name table-name)]
+ (t2/query statement))))
(mu/defn analyze-search-index-table!
"Run `ANALYZE` on the search index table `table-name` (Postgres only)."
diff --git a/src/metabase/timeline/api/timeline.clj b/src/metabase/timeline/api/timeline.clj
index 788f4aca828..fa89cf7f7d3 100644
--- a/src/metabase/timeline/api/timeline.clj
+++ b/src/metabase/timeline/api/timeline.clj
@@ -50,7 +50,7 @@
([]
(list-timelines false))
([archived :- ms/BooleanValue]
- (timeline.db/timelines-in-collections archived (collection/visible-collection-filter-clause))))
+ (timeline.db/timelines-in-visible-collections archived)))
(mu/defn get-timeline :- [:maybe (ms/InstanceOf :model/Timeline)]
"Fetch a single timeline by ID. Checks read permissions but does not hydrate."
diff --git a/src/metabase/timeline/db.clj b/src/metabase/timeline/db.clj
index a963e9892f1..d82b30514c9 100644
--- a/src/metabase/timeline/db.clj
+++ b/src/metabase/timeline/db.clj
@@ -3,6 +3,7 @@
additional logic, so no other namespace in the module runs a query itself (model definitions still use `toucan2.core`)."
(:require
[malli.util :as mut]
+ [metabase.collections.models.collection :as collection]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.timeline.schema :as timeline.schema]
[metabase.util.honey-sql-2 :as h2x]
@@ -15,15 +16,14 @@
[row :- (mut/select-keys ::timeline.schema/timeline.update [:name :creator_id :default :description :icon :collection_id :archived])]
(t2/insert-returning-instance! :model/Timeline row))
-(mu/defn timelines-in-collections
- "The Timelines whose archived flag is `archived` in the Collections matching the Honey SQL `collection-clause`, in
+(mu/defn timelines-in-visible-collections
+ "The Timelines whose archived flag is `archived` in the Collections visible to the current user, in
case-insensitive name order."
- [archived :- :boolean
- collection-clause :- [:maybe vector?]]
+ [archived :- :boolean]
(t2/select :model/Timeline
{:where [:and
[:= :archived archived]
- collection-clause]
+ (collection/visible-collection-filter-clause)]
:order-by [[:%lower.name :asc]]}))
(mu/defn timeline
diff --git a/src/metabase/warehouses/db.clj b/src/metabase/warehouses/db.clj
index f016f959bf0..b75faa3026a 100644
--- a/src/metabase/warehouses/db.clj
+++ b/src/metabase/warehouses/db.clj
@@ -135,10 +135,22 @@
[table-ids :- [:set ::lib.schema.id/table]]
(t2/select :model/Field, :table_id [:in table-ids], :semantic_type (mdb/isa :type/PK)))
-(mu/defn databases-reducible
- "A reducible of the Databases matching the Honey SQL `where` clause."
- [where :- [:maybe vector?]]
- (t2/reducible-select :model/Database {:where where}))
+(mu/defn databases-for-serdes-reducible
+ "A reducible of the Databases to export via serdes: routing destinations and the sample database are always
+ excluded, H2 databases unless `include-h2?`, and the export is restricted to the rows whose `filter-column` is one
+ of `filter-ids` when `filter-column` is given."
+ [filter-column :- [:maybe :keyword]
+ filter-ids :- [:maybe [:sequential [:maybe [:or :int :string]]]]
+ include-h2? :- :boolean]
+ (t2/reducible-select :model/Database
+ {:where [:and
+ (when filter-column
+ [:in filter-column filter-ids])
+ [:= :router_database_id nil]
+ ;; never export the sample database, regardless of its driver
+ [:not= :is_sample true]
+ (when-not include-h2?
+ [:not= :engine "h2"])]}))
(mu/defn table-database-id
"The Database id of the Table with `table-id`, or nil."
diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.clj
index b86ea0fcdda..bf79193e94b 100644
--- a/src/metabase/warehouses/models/database.clj
+++ b/src/metabase/warehouses/models/database.clj
@@ -718,14 +718,8 @@
false)
(defmethod serdes/extract-query "Database"
- [_model-name {:keys [where]}]
- (warehouses.db/databases-reducible (cond-> [:and
- (or where true)
- [:= :router_database_id nil]
- ;; never export the sample database, regardless of its driver
- [:not= :is_sample true]]
- (not *include-h2-in-extract?*)
- (conj [:not= :engine "h2"]))))
+ [_model-name {:keys [filter-column filter-ids]}]
+ (warehouses.db/databases-for-serdes-reducible filter-column filter-ids (boolean *include-h2-in-extract?*)))
(defmethod serdes/entity-id "Database"
[_ {:keys [name]}]
diff --git a/test/metabase/documents/models/document_test.clj b/test/metabase/documents/models/document_test.clj
index 0767dbc9c7f..00649a0b666 100644
--- a/test/metabase/documents/models/document_test.clj
+++ b/test/metabase/documents/models/document_test.clj
@@ -856,11 +856,12 @@
:collection_id coll-id
:exploration_id expl-id}]
(let [eid #(t2/select-one-fn :entity_id :model/Document :id %)
- ;; A caller-supplied `:where` must still compose — this is also the shape a full
+ ;; A caller-supplied id filter must still compose — this is also the shape a full
;; (untargeted) export takes, where the Collection descendants filter never runs.
extracted (into #{}
(map :entity_id)
- (serdes/extract-all "Document" {:where [:in :id [plain-id summary-id]]}))]
+ (serdes/extract-all "Document" {:filter-column :id
+ :filter-ids [plain-id summary-id]}))]
(is (contains? extracted (eid plain-id))
"an ordinary document is still exported")
(is (not (contains? extracted (eid summary-id)))
diff --git a/test/metabase/indexes/serdes_test.clj b/test/metabase/indexes/serdes_test.clj
index 8e7df2dfc58..bf4f95009f0 100644
--- a/test/metabase/indexes/serdes_test.clj
+++ b/test/metabase/indexes/serdes_test.clj
@@ -22,7 +22,7 @@
:columns [{:name "category" :direction :asc}]}
:status :succeeded
:error_message "boom"}]
- (let [hydrated (u/rfirst (serdes/extract-query "Transform" {:where [:= :id transform-id]}))]
+ (let [hydrated (u/rfirst (serdes/extract-query "Transform" {:filter-column :id :filter-ids [transform-id]}))]
(testing "extract-query hydrates the index under :indexes"
(is (= [idx-id] (map :id (:indexes hydrated)))))
(testing "the serialized index keeps its definition and drops runtime/local state"
diff --git a/test/metabase/queries/models/card_test.clj b/test/metabase/queries/models/card_test.clj
index 17ef415dad7..488d7c8cb96 100644
--- a/test/metabase/queries/models/card_test.clj
+++ b/test/metabase/queries/models/card_test.clj
@@ -1764,7 +1764,8 @@
(let [eid #(t2/select-one-fn :entity_id :model/Card :id %)
extracted (into #{}
(map :entity_id)
- (serdes/extract-all "Card" {:where [:in :id [plain-card doc-card summary-card]]}))]
+ (serdes/extract-all "Card" {:filter-column :id
+ :filter-ids [plain-card doc-card summary-card]}))]
(is (contains? extracted (eid plain-card))
"an ordinary card is still exported")
(is (contains? extracted (eid doc-card))