Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix lib-related N+1 issues with DB calls in parameters (#75888)

Alexander Polyankine025170e85f7
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 5121440d068..72f2bc27a34 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1486,8 +1486,7 @@
models
query-processor
util
- warehouse-schema
- warehouses}
+ warehouse-schema}
:model-imports #{:model/Card
:model/Dashboard
:model/Field
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
index eaf0a13fe74..ea4e64a92c9 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
@@ -15,7 +15,6 @@
[metabase.collections.models.collection.root :as collection.root]
[metabase.documents.schema :as documents.schema]
[metabase.graph.core :as graph]
- [metabase.lib-be.core :as lib-be]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.models.interface :as mi]
@@ -496,16 +495,15 @@
[:id {:optional true} ms/PositiveInt]
[:type {:optional true} ::deps.dependency-types/dependency-types]]]
(api/read-check (deps.dependency-types/dependency-type->model type) id)
- (lib-be/with-metadata-provider-cache
- (let [starting-nodes [[type id]]
- upstream-graph (readable-graph-dependencies {:include-archived-items :all})
- downstream-graph (graph/cached-graph (readable-graph-dependents))
- edge-graph (graph/cached-graph (readable-graph-dependents {:include-archived-items :all}))
- nodes (into (set starting-nodes)
- (graph/transitive upstream-graph starting-nodes))
- edges (graph/edges-between edge-graph nodes)]
- {:nodes (expanded-nodes downstream-graph nodes {:include-errors? false})
- :edges edges})))
+ (let [starting-nodes [[type id]]
+ upstream-graph (readable-graph-dependencies {:include-archived-items :all})
+ downstream-graph (graph/cached-graph (readable-graph-dependents))
+ edge-graph (graph/cached-graph (readable-graph-dependents {:include-archived-items :all}))
+ nodes (into (set starting-nodes)
+ (graph/transitive upstream-graph starting-nodes))
+ edges (graph/edges-between edge-graph nodes)]
+ {:nodes (expanded-nodes downstream-graph nodes {:include-errors? false})
+ :edges edges}))
(def ^:private sort-directions
"Valid sort directions for dependency item endpoints."
@@ -609,37 +607,36 @@
sort-column :name
sort-direction :asc}} :- dependents-args]
(api/read-check (deps.dependency-types/dependency-type->model type) id)
- (lib-be/with-metadata-provider-cache
- (let [downstream-graph (graph/cached-graph (readable-graph-dependents {:broken broken}))
- nodes (-> (graph/children-of downstream-graph [[type id]])
- (get [type id]))
- dep-types-set (cond
- (nil? dependent-types) deps.dependency-types/dependency-types
- (sequential? dependent-types) (set dependent-types)
- :else #{dependent-types})
- card-types-set (cond
- (nil? dependent-card-types) lib.schema.metadata/card-types
- (sequential? dependent-card-types) (set dependent-card-types)
- :else #{dependent-card-types})
- dependents-filter
- (comp
- ;; Filter by dependent types and card types
- (filter (fn [node]
- (and (or (nil? dep-types-set)
- (contains? dep-types-set (:type node)))
- (or (not= (:type node) :card)
- (nil? card-types-set)
- (contains? card-types-set (-> node :data :type))))))
- ;; Filter out personal collections unless explicitly included
- (if include-personal-collections
- identity
- (remove in-personal-collection?))
- ;; Filter by query (sandboxes are excluded since they have no name or location)
- (if query
- (filter #(entity-matches-query? % query))
- identity))]
- (-> (into [] dependents-filter (expanded-nodes downstream-graph nodes {:include-errors? false}))
- (sort-dependents sort-column sort-direction)))))
+ (let [downstream-graph (graph/cached-graph (readable-graph-dependents {:broken broken}))
+ nodes (-> (graph/children-of downstream-graph [[type id]])
+ (get [type id]))
+ dep-types-set (cond
+ (nil? dependent-types) deps.dependency-types/dependency-types
+ (sequential? dependent-types) (set dependent-types)
+ :else #{dependent-types})
+ card-types-set (cond
+ (nil? dependent-card-types) lib.schema.metadata/card-types
+ (sequential? dependent-card-types) (set dependent-card-types)
+ :else #{dependent-card-types})
+ dependents-filter
+ (comp
+ ;; Filter by dependent types and card types
+ (filter (fn [node]
+ (and (or (nil? dep-types-set)
+ (contains? dep-types-set (:type node)))
+ (or (not= (:type node) :card)
+ (nil? card-types-set)
+ (contains? card-types-set (-> node :data :type))))))
+ ;; Filter out personal collections unless explicitly included
+ (if include-personal-collections
+ identity
+ (remove in-personal-collection?))
+ ;; Filter by query (sandboxes are excluded since they have no name or location)
+ (if query
+ (filter #(entity-matches-query? % query))
+ identity))]
+ (-> (into [] dependents-filter (expanded-nodes downstream-graph nodes {:include-errors? false}))
+ (sort-dependents sort-column sort-direction))))
(defn- entity-type-config
[entity-type]
@@ -1011,51 +1008,50 @@
sort-column :name
sort-direction :asc}} :- broken-dependents-args]
(api/read-check (deps.dependency-types/dependency-type->model entity-type) id)
- (lib-be/with-metadata-provider-cache
- (let [normalize-types (fn normalize-types [types]
- (if (keyword? types)
- [(name types)]
- (not-empty (map name types))))
- dep-types (normalize-types dependent-types)
- card-types (normalize-types dependent-card-types)
- where-clause (cond-> [:and
- [:= :afe.source_entity_type (name entity-type)]
- [:= :afe.source_entity_id id]
- [:= :af.result false]
- (visible-entities-filter-clause
- :afe.analyzed_entity_type
- :afe.analyzed_entity_id
- {:include-archived-items :exclude})]
- dep-types (conj [:in :afe.analyzed_entity_type dep-types])
- card-types (conj [:or
- [:!= :afe.analyzed_entity_type [:inline "card"]]
- [:in :rc.type card-types]]))
- broken-entity-pairs
- (t2/query (cond-> {:select-distinct [[:afe.analyzed_entity_type :entity_type]
- [:afe.analyzed_entity_id :entity_id]]
- :from [[:analysis_finding_error :afe]]
- :join [[:analysis_finding :af]
- [:and
- [:= :af.analyzed_entity_type :afe.analyzed_entity_type]
- [:= :af.analyzed_entity_id :afe.analyzed_entity_id]]]
- :where where-clause}
- card-types (assoc :left-join [[:report_card :rc]
- [:and
- [:= :afe.analyzed_entity_type [:inline "card"]]
- [:= :rc.id :afe.analyzed_entity_id]]])))
- nodes (map (fn [{:keys [entity_type entity_id]}]
- [(keyword entity_type) entity_id])
- broken-entity-pairs)
- nodes-by-type (-> (group-by first nodes)
- (update-vals #(map second %)))]
- (-> (into [] (cond-> (map (fn [[[entity-type entity-id] entity]]
- {:id entity-id
- :type entity-type
- :data (-> (select-keys entity (entity-keys entity-type))
- (update-vals format-subentity))}))
- (not include-personal-collections) (comp (remove in-personal-collection?)))
- (fetch-and-hydrate-nodes nodes-by-type))
- (sort-dependents sort-column sort-direction)))))
+ (let [normalize-types (fn normalize-types [types]
+ (if (keyword? types)
+ [(name types)]
+ (not-empty (map name types))))
+ dep-types (normalize-types dependent-types)
+ card-types (normalize-types dependent-card-types)
+ where-clause (cond-> [:and
+ [:= :afe.source_entity_type (name entity-type)]
+ [:= :afe.source_entity_id id]
+ [:= :af.result false]
+ (visible-entities-filter-clause
+ :afe.analyzed_entity_type
+ :afe.analyzed_entity_id
+ {:include-archived-items :exclude})]
+ dep-types (conj [:in :afe.analyzed_entity_type dep-types])
+ card-types (conj [:or
+ [:!= :afe.analyzed_entity_type [:inline "card"]]
+ [:in :rc.type card-types]]))
+ broken-entity-pairs
+ (t2/query (cond-> {:select-distinct [[:afe.analyzed_entity_type :entity_type]
+ [:afe.analyzed_entity_id :entity_id]]
+ :from [[:analysis_finding_error :afe]]
+ :join [[:analysis_finding :af]
+ [:and
+ [:= :af.analyzed_entity_type :afe.analyzed_entity_type]
+ [:= :af.analyzed_entity_id :afe.analyzed_entity_id]]]
+ :where where-clause}
+ card-types (assoc :left-join [[:report_card :rc]
+ [:and
+ [:= :afe.analyzed_entity_type [:inline "card"]]
+ [:= :rc.id :afe.analyzed_entity_id]]])))
+ nodes (map (fn [{:keys [entity_type entity_id]}]
+ [(keyword entity_type) entity_id])
+ broken-entity-pairs)
+ nodes-by-type (-> (group-by first nodes)
+ (update-vals #(map second %)))]
+ (-> (into [] (cond-> (map (fn [[[entity-type entity-id] entity]]
+ {:id entity-id
+ :type entity-type
+ :data (-> (select-keys entity (entity-keys entity-type))
+ (update-vals format-subentity))}))
+ (not include-personal-collections) (comp (remove in-personal-collection?)))
+ (fetch-and-hydrate-nodes nodes-by-type))
+ (sort-dependents sort-column sort-direction))))
(api.macros/defendpoint :get "/backfill-status" :- [:map
[:complete :boolean]]
diff --git a/enterprise/backend/src/metabase_enterprise/replacement/runner.clj b/enterprise/backend/src/metabase_enterprise/replacement/runner.clj
index facc151bc2d..83bea598b64 100644
--- a/enterprise/backend/src/metabase_enterprise/replacement/runner.clj
+++ b/enterprise/backend/src/metabase_enterprise/replacement/runner.clj
@@ -84,7 +84,7 @@
(vals measures))))]
(when (seq queries)
;; Extract all referenced entity IDs across all queries
- (let [referenced-ids (lib/all-referenced-entity-ids queries)]
+ (let [referenced-ids (lib/all-referenced-entity-ids queries {:include-implicitly-joinable? true})]
;; Bulk load all metadata at once
(lib-be/bulk-load-query-metadata! metadata-provider referenced-ids)))
(merge {} cards tables dashboards transforms segments measures))))
diff --git a/src/metabase/content_verification/impl.clj b/src/metabase/content_verification/impl.clj
index 4ba32aac292..7c24b6da1c9 100644
--- a/src/metabase/content_verification/impl.clj
+++ b/src/metabase/content_verification/impl.clj
@@ -31,10 +31,14 @@
;; item that comes into this comes out. The nested hydration is positional, not by an id so everything that comes in
;; must go out in the same order
(when (seq items)
- (let [item-ids (not-empty (keep :id items))
+ (let [items* (keep identity items)
+ item-ids (not-empty (keep :id items*))
+ ;; constrain on `:moderated_item_type` too so the `(moderated_item_type, moderated_item_id)` index is used
+ item-types (not-empty (into #{} (map (comp keyword object->type)) items*))
all-reviews (when item-ids
(group-by (juxt :moderated_item_type :moderated_item_id)
(t2/select :model/ModerationReview
+ :moderated_item_type [:in item-types]
:moderated_item_id [:in item-ids]
{:order-by [[:id :desc]]})))]
(for [item items]
@@ -58,9 +62,13 @@
"Hydrate moderation status onto a seq of items"
[items]
(when (seq items)
- (let [item-ids (seq (keep :id items))
+ (let [items* (keep identity items)
+ item-ids (seq (keep :id items*))
+ ;; constrain on `:moderated_item_type` too so the `(moderated_item_type, moderated_item_id)` index is used
+ item-types (not-empty (into #{} (map (comp keyword object->type)) items*))
type+id->status (when item-ids
(->> (t2/select [:model/ModerationReview :moderated_item_id :moderated_item_type :status]
+ :moderated_item_type [:in item-types]
:moderated_item_id [:in item-ids]
:most_recent true
{:order-by [[:id :desc]]})
diff --git a/src/metabase/dashboards/models/dashboard.clj b/src/metabase/dashboards/models/dashboard.clj
index d61f51cc05f..428282b6ae1 100644
--- a/src/metabase/dashboards/models/dashboard.clj
+++ b/src/metabase/dashboards/models/dashboard.clj
@@ -407,7 +407,7 @@
:dashcard ...
:target [:dimension [:field-id 264]]}}}}"
[_model k dashboards]
- (let [dashboards-with-cards (t2/hydrate dashboards [:dashcards :card])]
+ (let [dashboards-with-cards (t2/hydrate dashboards [:dashcards :card :series])]
(map #(assoc %1 k %2) dashboards (map dashboard->resolved-params dashboards-with-cards))))
(defmethod mi/exclude-internal-content-hsql :model/Dashboard
diff --git a/src/metabase/dashboards_rest/api.clj b/src/metabase/dashboards_rest/api.clj
index 5857b728c43..6c131e25166 100644
--- a/src/metabase/dashboards_rest/api.clj
+++ b/src/metabase/dashboards_rest/api.clj
@@ -139,28 +139,27 @@
[:collection_position {:optional true} [:maybe ms/PositiveInt]]]]
;; if we're trying to save the new dashboard in a Collection make sure we have permissions to do that
(api/create-check :model/Dashboard {:collection_id collection_id})
- (lib-be/with-metadata-provider-cache
- (let [dashboard-data {:name name
- :description description
- :parameters (or parameters [])
- :creator_id api/*current-user-id*
- :cache_ttl cache_ttl
- :collection_id collection_id
- :collection_position collection_position}
- dash (t2/with-transaction [_conn]
- ;; Adding a new dashboard at `collection_position` could cause other dashboards in this
- ;; collection to change position, check that and fix up if needed
- (api/maybe-reconcile-collection-position! dashboard-data)
- ;; Ok, now save the Dashboard
- (first (t2/insert-returning-instances! :model/Dashboard dashboard-data)))]
- (events/publish-event! :event/dashboard-create {:object dash :user-id api/*current-user-id*})
- (analytics/track-event! :snowplow/dashboard
- {:event :dashboard-created
- :dashboard-id (u/the-id dash)})
- (-> dash
- hydrate-dashboard-details
- collection.root/hydrate-root-collection
- (assoc :last-edit-info (revisions/edit-information-for-user @api/*current-user*))))))
+ (let [dashboard-data {:name name
+ :description description
+ :parameters (or parameters [])
+ :creator_id api/*current-user-id*
+ :cache_ttl cache_ttl
+ :collection_id collection_id
+ :collection_position collection_position}
+ dash (t2/with-transaction [_conn]
+ ;; Adding a new dashboard at `collection_position` could cause other dashboards in this
+ ;; collection to change position, check that and fix up if needed
+ (api/maybe-reconcile-collection-position! dashboard-data)
+ ;; Ok, now save the Dashboard
+ (first (t2/insert-returning-instances! :model/Dashboard dashboard-data)))]
+ (events/publish-event! :event/dashboard-create {:object dash :user-id api/*current-user-id*})
+ (analytics/track-event! :snowplow/dashboard
+ {:event :dashboard-created
+ :dashboard-id (u/the-id dash)})
+ (-> dash
+ hydrate-dashboard-details
+ collection.root/hydrate-root-collection
+ (assoc :last-edit-info (revisions/edit-information-for-user @api/*current-user*)))))
;;; -------------------------------------------- Hiding Unreadable Cards ---------------------------------------------
@@ -1249,11 +1248,10 @@
[:id ms/PositiveInt]
[:param-key ms/NonBlankString]]
constraint-param-key->value :- [:map-of string? any?]]
- (lib-be/with-metadata-provider-cache
- (let [dashboard (hydrate-dashboard-details (api/read-check :model/Dashboard id))]
- ;; If a user can read the dashboard, then they can lookup filters. This also works with sandboxing.
- (binding [qp.perms/*param-values-query* true]
- (parameters.dashboard/param-values dashboard param-key constraint-param-key->value)))))
+ (let [dashboard (api/read-check :model/Dashboard id)]
+ ;; If a user can read the dashboard, then they can lookup filters. This also works with sandboxing.
+ (binding [qp.perms/*param-values-query* true]
+ (parameters.dashboard/param-values dashboard param-key constraint-param-key->value))))
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
@@ -1273,12 +1271,11 @@
[:param-key ms/NonBlankString]
[:query ms/NonBlankString]]
constraint-param-key->value :- [:map-of string? any?]]
- (lib-be/with-metadata-provider-cache
- (let [dashboard (api/read-check :model/Dashboard id)]
- ;; If a user can read the dashboard, then they can lookup filters. This also works with sandboxing.
- (binding [qp.perms/*param-values-query* true
- chain-filter/*allow-implicit-uuid-field-remapping* false]
- (parameters.dashboard/param-values dashboard param-key constraint-param-key->value query)))))
+ (let [dashboard (api/read-check :model/Dashboard id)]
+ ;; If a user can read the dashboard, then they can lookup filters. This also works with sandboxing.
+ (binding [qp.perms/*param-values-query* true
+ chain-filter/*allow-implicit-uuid-field-remapping* false]
+ (parameters.dashboard/param-values dashboard param-key constraint-param-key->value query))))
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
diff --git a/src/metabase/embedding_rest/api/common.clj b/src/metabase/embedding_rest/api/common.clj
index 45c39037078..e2df18af088 100644
--- a/src/metabase/embedding_rest/api/common.clj
+++ b/src/metabase/embedding_rest/api/common.clj
@@ -526,8 +526,9 @@
dashboard-id (unsigned-token->dashboard-id unsigned-token)
_ (when-not preview (check-embedding-enabled-for-dashboard dashboard-id))
slug-token-params (embed/get-in-unsigned-token-or-throw unsigned-token [:params])
+ dashboard (t2/select-one :model/Dashboard :id dashboard-id)
{parameters :parameters
- published-embedding-params :embedding_params} (t2/select-one :model/Dashboard :id dashboard-id)
+ published-embedding-params :embedding_params} dashboard
;; when previewing an embed, embedding-params should come from the token,
;; since a user may be changing them prior to publishing the Embed, which is what actually persists
;; the settings to the Appdb.
@@ -552,7 +553,7 @@
(try
(binding [api/*current-user-permissions-set* (atom #{"/"})
api/*is-superuser?* true]
- (parameters.dashboard/param-values (t2/select-one :model/Dashboard :id dashboard-id) searched-param-id merged-id-params prefix))
+ (parameters.dashboard/param-values dashboard searched-param-id merged-id-params prefix))
(catch Throwable e
(throw (ex-info (.getMessage e)
{:merged-id-params merged-id-params}
diff --git a/src/metabase/lib/metadata.cljc b/src/metabase/lib/metadata.cljc
index 0a31ea61368..f0363cd9106 100644
--- a/src/metabase/lib/metadata.cljc
+++ b/src/metabase/lib/metadata.cljc
@@ -276,6 +276,17 @@
(keep id->result)
ids))))
+(mu/defn metadatas :- [:maybe [:sequential :metabase.lib.metadata.protocols/metadata]]
+ "Return a sequence of metadata objects matching `metadata-spec` (see
+ `:metabase.lib.metadata.protocols/metadata-spec`). A thin wrapper over [[lib.metadata.protocols/metadatas]] that
+ resolves the MetadataProvider from `metadata-providerable` for you, so callers don't need to reach for
+ [[->metadata-provider]] or the protocol namespace directly.
+
+ Like the underlying method, can be called for side-effects to warm the cache."
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ metadata-spec :- :metabase.lib.metadata.protocols/metadata-spec]
+ (lib.metadata.protocols/metadatas (->metadata-provider metadata-providerable) metadata-spec))
+
(defn- missing-bulk-metadata-error [metadata-type id]
(ex-info (i18n/tru "Failed to fetch {0} {1}: either it does not exist, or it belongs to a different Database"
(pr-str metadata-type)
diff --git a/src/metabase/lib/walk/util.cljc b/src/metabase/lib/walk/util.cljc
index 32c5bbd6137..00c9d426f0a 100644
--- a/src/metabase/lib/walk/util.cljc
+++ b/src/metabase/lib/walk/util.cljc
@@ -7,6 +7,7 @@
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.lib.schema.template-tag :as lib.schema.template-tag]
[metabase.lib.walk :as lib.walk]
[metabase.util.malli :as mu]
@@ -258,26 +259,65 @@
[:segment [:set ::lib.schema.id/segment]]
[:snippet [:set ::lib.schema.id/snippet]]])
+(mr/def ::referenced-entity-ids.options
+ [:map
+ [:include-implicitly-joinable? {:optional true} :boolean]])
+
+(mu/defn- implicitly-joinable-table-ids :- [:set ::lib.schema.id/table]
+ "The ids of Tables reachable by one FK hop out of the columns of `source-table-ids` and the result metadata of
+ `source-card-ids` -- i.e. the Tables a caller exposing implicitly-joinable columns would need on top of the directly
+ referenced ones. Loads those columns/Fields and the FK-target Fields into `metadata-providerable`'s cache while
+ discovering the ids, so that caller hits the cache instead of fetching one entity at a time.
+
+ Note `implicitly-joined-field-ids` above means Fields *already* joined in a query; this is about Fields that are
+ *joinable* (reachable via an FK) but not joined yet."
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ source-table-ids :- [:set ::lib.schema.id/table]
+ source-card-ids :- [:set ::lib.schema.id/card]]
+ (let [table-columns (when (seq source-table-ids)
+ (lib.metadata/metadatas metadata-providerable {:lib/type :metadata/column, :table-ids source-table-ids}))
+ cards (lib.metadata/bulk-metadata metadata-providerable :metadata/card source-card-ids)
+ result-columns (mapcat :result-metadata cards)
+ card-columns (lib.metadata/bulk-metadata metadata-providerable :metadata/column
+ (into #{} (keep :id) result-columns))
+ fk-fields (lib.metadata/bulk-metadata metadata-providerable :metadata/column
+ (into #{} (keep :fk-target-field-id)
+ (concat table-columns card-columns result-columns)))]
+ (into #{} (keep :table-id) fk-fields)))
+
(mu/defn all-referenced-entity-ids :- ::referenced-entity-ids
- "Return a map of all referenced entity IDs in `queries`."
- [queries :- [:sequential ::lib.schema/query]]
- (let [source-table-ids (into #{} (mapcat all-source-table-ids) queries)
- source-card-ids (into #{} (mapcat all-source-card-ids) queries)
- implicitly-joined-field-ids (into #{} (mapcat all-implicitly-joined-field-ids) queries)
- template-tag-field-ids (into #{} (mapcat all-template-tag-field-ids) queries)
- template-tag-table-ids (into #{} (mapcat all-template-tag-table-ids) queries)
- template-tag-card-ids (into #{} (mapcat all-template-tag-card-ids) queries)
- template-tag-snippet-ids (into #{} (mapcat all-template-tag-snippet-ids) queries)
- metric-ids (into #{} (mapcat all-metric-ids) queries)
- measure-ids (into #{} (mapcat all-measure-ids) queries)
- segment-ids (into #{} (mapcat all-segment-ids) queries)
- all-field-ids* (set/union implicitly-joined-field-ids template-tag-field-ids)
- all-field-table-ids (when (seq queries)
- (->> (lib.metadata/bulk-metadata (first queries) :metadata/column all-field-ids*)
- (into #{} (keep :table-id))))]
- {:table (set/union source-table-ids all-field-table-ids template-tag-table-ids)
- :card (set/union source-card-ids template-tag-card-ids)
- :metric metric-ids
- :measure measure-ids
- :segment segment-ids
- :snippet template-tag-snippet-ids}))
+ "Return a map of all referenced entity IDs in `queries`.
+
+ With `:include-implicitly-joinable?` true, `:table` additionally includes the Tables reachable by one FK hop out of
+ the source Tables' columns and the source Cards' result metadata (see [[implicitly-joinable-table-ids]]); discovering
+ them warms those columns/Fields in `(first queries)`'s cache. Defaults to false."
+ ([queries :- [:sequential ::lib.schema/query]]
+ (all-referenced-entity-ids queries nil))
+
+ ([queries :- [:sequential ::lib.schema/query]
+ {:keys [include-implicitly-joinable?]} :- [:maybe ::referenced-entity-ids.options]]
+ (let [source-table-ids (into #{} (mapcat all-source-table-ids) queries)
+ source-card-ids (into #{} (mapcat all-source-card-ids) queries)
+ implicitly-joined-field-ids (into #{} (mapcat all-implicitly-joined-field-ids) queries)
+ template-tag-field-ids (into #{} (mapcat all-template-tag-field-ids) queries)
+ template-tag-table-ids (into #{} (mapcat all-template-tag-table-ids) queries)
+ template-tag-card-ids (into #{} (mapcat all-template-tag-card-ids) queries)
+ template-tag-snippet-ids (into #{} (mapcat all-template-tag-snippet-ids) queries)
+ metric-ids (into #{} (mapcat all-metric-ids) queries)
+ measure-ids (into #{} (mapcat all-measure-ids) queries)
+ segment-ids (into #{} (mapcat all-segment-ids) queries)
+ all-card-ids (set/union source-card-ids template-tag-card-ids)
+ field-ids (set/union implicitly-joined-field-ids template-tag-field-ids)
+ all-field-table-ids (if (seq queries)
+ (->> (lib.metadata/bulk-metadata (first queries) :metadata/column field-ids)
+ (into #{} (keep :table-id)))
+ #{})
+ joinable-table-ids (if (and include-implicitly-joinable? (seq queries))
+ (implicitly-joinable-table-ids (first queries) source-table-ids all-card-ids)
+ #{})]
+ {:table (set/union source-table-ids all-field-table-ids template-tag-table-ids joinable-table-ids)
+ :card all-card-ids
+ :metric metric-ids
+ :measure measure-ids
+ :segment segment-ids
+ :snippet template-tag-snippet-ids})))
diff --git a/src/metabase/lib_be/query.clj b/src/metabase/lib_be/query.clj
index 6d005300cdb..11755bc4536 100644
--- a/src/metabase/lib_be/query.clj
+++ b/src/metabase/lib_be/query.clj
@@ -1,28 +1,20 @@
(ns metabase.lib-be.query
(:require
[metabase.lib.metadata :as lib.metadata]
- [metabase.lib.metadata.protocols :as lib.metadata.protocols]
- [metabase.lib.metadata.util :as lib.metadata.util]
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.lib.walk.util :as lib.walk.util]
[metabase.util.malli :as mu]))
-(mu/defn bulk-load-query-metadata!
+(mu/defn bulk-load-query-metadata! :- :nil
"Bulk-load all metadata referenced by `referenced-entity-ids` into `metadata-providerable`'s cache."
[metadata-providerable :- ::lib.schema.metadata/metadata-providerable
{:keys [table card metric measure segment snippet]} :- ::lib.walk.util/referenced-entity-ids]
+ (lib.metadata/bulk-metadata metadata-providerable :metadata/table table)
(when (seq table)
- (lib.metadata/bulk-metadata metadata-providerable :metadata/table table)
- (lib.metadata.protocols/metadatas (lib.metadata.util/->metadata-provider metadata-providerable)
- {:lib/type :metadata/column, :table-ids (set table)}))
- (when (seq card)
- (lib.metadata/bulk-metadata metadata-providerable :metadata/card card))
- (when (seq metric)
- (lib.metadata/bulk-metadata metadata-providerable :metadata/metric metric))
- (when (seq measure)
- (lib.metadata/bulk-metadata metadata-providerable :metadata/measure measure))
- (when (seq segment)
- (lib.metadata/bulk-metadata metadata-providerable :metadata/segment segment))
- (when (seq snippet)
- (lib.metadata/bulk-metadata metadata-providerable :metadata/native-query-snippet snippet))
+ (lib.metadata/metadatas metadata-providerable {:lib/type :metadata/column, :table-ids table}))
+ (lib.metadata/bulk-metadata metadata-providerable :metadata/card card)
+ (lib.metadata/bulk-metadata metadata-providerable :metadata/metric metric)
+ (lib.metadata/bulk-metadata metadata-providerable :metadata/measure measure)
+ (lib.metadata/bulk-metadata metadata-providerable :metadata/segment segment)
+ (lib.metadata/bulk-metadata metadata-providerable :metadata/native-query-snippet snippet)
nil)
diff --git a/src/metabase/parameters/chain_filter.clj b/src/metabase/parameters/chain_filter.clj
index e2f9a9fa0c1..717510338bf 100644
--- a/src/metabase/parameters/chain_filter.clj
+++ b/src/metabase/parameters/chain_filter.clj
@@ -72,6 +72,7 @@
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.id :as lib.schema.id]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.lib.types.isa :as lib.types.isa]
[metabase.lib.util :as lib.util]
[metabase.parameters.chain-filter.dedupe-joins :as dedupe]
@@ -93,7 +94,6 @@
[metabase.warehouse-schema.metadata-queries :as schema.metadata-queries]
[metabase.warehouse-schema.models.field :as field]
[metabase.warehouse-schema.models.field-values :as field-values]
- [metabase.warehouses.models.database :as database]
[toucan2.core :as t2]))
;; so the hydration method for name_field is loaded
@@ -121,38 +121,31 @@
(defn- joined-table-alias [table-id]
(format "table_%d" table-id))
-(def ^:private ^{:arglists '([field-id])} memoized-field-types-by-id
- "Return field types by id. Cached for 10 minutes to avoid hitting the DB too much since this is unlike to change
- often, if ever."
- (memoize/ttl
- ^{::memoize/args-fn (fn [[field-id]]
- [(mdb/unique-identifier) field-id])}
- (fn [field-id]
- (t2/select-one [:model/Field :base_type :semantic_type] :id field-id))
- :ttl/threshold (u/minutes->ms 10)))
-
(mu/defn- add-filter :- ::lib.schema/query
- "Generate a single MBQL `:filter` clause for a Field and `value` (or multiple values, if `value` is a collection)."
+ "Generate a single MBQL `:filter` clause for a Field and `value` (or multiple values, if `value` is a collection).
+ `id->field` looks up prefetched Field metadata by id (see [[add-filters]])."
[query :- ::lib.schema/query
source-table-id :- ::lib.schema.id/table
+ id->field :- [:map-of ::lib.schema.id/field ::lib.schema.metadata/column]
{:keys [field-id op value options]} :- ::constraint]
- (let [field (let [this-field-table-id (field/field-id->table-id field-id)]
- (cond-> (lib.metadata/field query field-id)
- (not= this-field-table-id source-table-id)
- (lib/with-join-alias (joined-table-alias this-field-table-id))))
- filter-op (if (and (lib.types.isa/temporal? field)
- (string? value))
- (try
- (params.dates/date-string->filter value field-id)
- (catch Throwable e
- (log/error e "Error creating filter for date string")
- nil))
- ;; we don't want to skip our value, even if its nil
- (let [values (if (nil? value) [nil] (u/one-or-many value))]
- {:lib/type :lib/external-op
- :operator op
- :options options
- :args (cons field values)}))]
+ (let [field (id->field field-id)
+ field-table-id (:table-id field)
+ field (cond-> field
+ (not= field-table-id source-table-id)
+ (lib/with-join-alias (joined-table-alias field-table-id)))
+ filter-op (if (and (lib.types.isa/temporal? field)
+ (string? value))
+ (try
+ (params.dates/date-string->filter value field-id)
+ (catch Throwable e
+ (log/error e "Error creating filter for date string")
+ nil))
+ ;; we don't want to skip our value, even if its nil
+ (let [values (if (nil? value) [nil] (u/one-or-many value))]
+ {:lib/type :lib/external-op
+ :operator op
+ :options options
+ :args (cons field values)}))]
(when filter-op
(log/tracef "Adding filter clause: %s" (pr-str filter-op)))
(cond-> query
@@ -179,27 +172,29 @@
source-table-id :- ::lib.schema.id/table
joined-table-ids :- [:set ::lib.schema.id/table]
constraints :- [:maybe ::constraints]]
- (reduce
- (fn [query {:keys [field-id] :as constraint}]
- ;; only add a where clause for the Field if it's part of the source Table or if we're actually joining against
- ;; the Table it belongs to. This Field might not even be part of the same Database in which case we can ignore
- ;; it.
- (let [field-table-id (field/field-id->table-id field-id)]
- (if (or (= field-table-id source-table-id)
- (contains? joined-table-ids field-table-id))
- (do
- (log/tracef "Added filter clause for %s %s with constraint %s"
- (name-for-logging :model/Table field-table-id)
- (name-for-logging :model/Field field-id)
- (pr-str constraint))
- (add-filter query source-table-id constraint))
- (do
- (log/tracef "Not adding filter clause for %s %s because we did not join against its Table"
- (name-for-logging :model/Table field-table-id)
- (name-for-logging :model/Field field-id))
- query))))
- query
- constraints))
+ (let [id->field (u/index-by :id (lib.metadata/bulk-metadata query :metadata/column
+ (into #{} (map :field-id) constraints)))]
+ (reduce
+ (fn [query {:keys [field-id] :as constraint}]
+ ;; only add a where clause for the Field if it's part of the source Table or if we're actually joining against
+ ;; the Table it belongs to. This Field might not even be part of the same Database in which case we can ignore
+ ;; it.
+ (let [field-table-id (:table-id (id->field field-id))]
+ (if (or (= field-table-id source-table-id)
+ (contains? joined-table-ids field-table-id))
+ (do
+ (log/tracef "Added filter clause for %s %s with constraint %s"
+ (name-for-logging :model/Table field-table-id)
+ (name-for-logging :model/Field field-id)
+ (pr-str constraint))
+ (add-filter query source-table-id id->field constraint))
+ (do
+ (log/tracef "Not adding filter clause for %s %s because we did not join against its Table"
+ (name-for-logging :model/Table field-table-id)
+ (name-for-logging :model/Field field-id))
+ query))))
+ query
+ constraints)))
(def ^:private find-joins-cache-duration-ms
"Amount of time to cache results of `find-joins`. Since FK relationships in Tables are unlikely to change very
@@ -339,13 +334,12 @@
(f database-id source-table-id other-table-id enable-reverse-joins?)))
(meta f))))
-(def ^:private ^{:arglists '([source-table field-ids other-table-ids enable-reverse-joins?])} find-all-joins*
+(def ^:private ^{:arglists '([db-id source-table field-ids other-table-ids enable-reverse-joins?])} find-all-joins*
(memoize/ttl
- ^{::memoize/args-fn (fn [[source-table-id field-ids other-table-ids enable-reverse-joins?]]
+ ^{::memoize/args-fn (fn [[_db-id source-table-id field-ids other-table-ids enable-reverse-joins?]]
[(mdb/unique-identifier) source-table-id field-ids other-table-ids enable-reverse-joins?])}
- (fn [source-table-id field-ids other-table-ids enable-reverse-joins?]
- (let [db-id (database/table-id->database-id source-table-id)
- all-joins (mapcat #(find-joins db-id source-table-id % enable-reverse-joins?)
+ (fn [db-id source-table-id field-ids other-table-ids enable-reverse-joins?]
+ (let [all-joins (mapcat #(find-joins db-id source-table-id % enable-reverse-joins?)
other-table-ids)]
(when (seq all-joins)
(log/tracef "Deduplicating for source %s; Tables to keep: %s\n%s"
@@ -360,11 +354,15 @@
(mu/defn- find-all-joins
"Find the complete set of joins we need to do for `source-table-id` to join against Fields in `field-ids`."
- [source-table-id :- ::lib.schema.id/table
- field-ids :- [:set ::lib.schema.id/field]]
- (when-let [other-table-ids (not-empty (disj (set (map field/field-id->table-id (set field-ids)))
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ database-id :- ::lib.schema.id/database
+ source-table-id :- ::lib.schema.id/table
+ field-ids :- [:set ::lib.schema.id/field]]
+ (when-let [other-table-ids (not-empty (disj (into #{}
+ (map :table-id)
+ (lib.metadata/bulk-metadata metadata-providerable :metadata/column field-ids))
source-table-id))]
- (find-all-joins* source-table-id field-ids other-table-ids *enable-reverse-joins*)))
+ (find-all-joins* database-id source-table-id field-ids other-table-ids *enable-reverse-joins*)))
(mu/defn- add-joins :- ::lib.schema/query
"Add joins to the MBQL `query` we're generating. The Field for which we are returning values is the \"source Field\",
@@ -385,24 +383,30 @@
[query :- ::lib.schema/query
source-table-id :- ::lib.schema.id/table
joins]
- (reduce
- (fn [query {{lhs-table-id :table, lhs-field-id :field} :lhs, {rhs-table-id :table, rhs-field-id :field} :rhs}]
- (let [lhs-field (lib.metadata/field query lhs-field-id)
- rhs-field (lib.metadata/field query rhs-field-id)
- rhs-table (lib.metadata/table query rhs-table-id)
- join (-> (lib/join-clause rhs-table)
- (lib/with-join-alias (joined-table-alias rhs-table-id))
- (lib/with-join-conditions [(lib/=
- (cond-> lhs-field
- (not= lhs-table-id source-table-id)
- (lib/with-join-alias (joined-table-alias lhs-table-id)))
- (-> rhs-field
- (lib/with-join-alias (joined-table-alias rhs-table-id))))]))]
- (log/tracef "Adding join against %s\n%s"
- (name-for-logging :model/Table rhs-table-id) (u/cprint-to-str join))
- (lib/join query join)))
- query
- joins))
+ (let [id->field (u/index-by :id (lib.metadata/bulk-metadata query :metadata/column
+ (into #{} (mapcat (juxt #(get-in % [:lhs :field])
+ #(get-in % [:rhs :field])))
+ joins)))
+ id->table (u/index-by :id (lib.metadata/bulk-metadata query :metadata/table
+ (into #{} (map #(get-in % [:rhs :table])) joins)))]
+ (reduce
+ (fn [query {{lhs-table-id :table, lhs-field-id :field} :lhs, {rhs-table-id :table, rhs-field-id :field} :rhs}]
+ (let [lhs-field (id->field lhs-field-id)
+ rhs-field (id->field rhs-field-id)
+ rhs-table (id->table rhs-table-id)
+ join (-> (lib/join-clause rhs-table)
+ (lib/with-join-alias (joined-table-alias rhs-table-id))
+ (lib/with-join-conditions [(lib/=
+ (cond-> lhs-field
+ (not= lhs-table-id source-table-id)
+ (lib/with-join-alias (joined-table-alias lhs-table-id)))
+ (-> rhs-field
+ (lib/with-join-alias (joined-table-alias rhs-table-id))))]))]
+ (log/tracef "Adding join against %s\n%s"
+ (name-for-logging :model/Table rhs-table-id) (u/cprint-to-str join))
+ (lib/join query join)))
+ query
+ joins)))
(mu/defn- tighten-join-projections :- ::lib.schema/query
"Narrow each join's inner-stage `:fields` to exactly the field-ids `query` references whose `:table-id` matches the
@@ -445,17 +449,19 @@
constraints :- [:maybe ::constraints]
{:keys [original-field-id limit]} :- [:maybe ::options]]
(log/tracef "Chain filter %s with constraints %s" (name-for-logging :model/Field field-id) (u/cprint-to-str constraints))
+ ;; `field-id->database-id` is the one place we still bootstrap from a raw field-id: we need the Database before we
+ ;; can build a (per-Database) metadata provider. Every other table-id/db-id below comes from `mp`.
(let [database-id (field/field-id->database-id field-id)
mp (lib-be/application-database-metadata-provider database-id)
- field-table-id (field/field-id->table-id field-id)
- original-table-id (when original-field-id (field/field-id->table-id original-field-id))
+ field-table-id (:table-id (lib.metadata/field mp field-id))
+ original-table-id (when original-field-id (:table-id (lib.metadata/field mp original-field-id)))
;; When the original (FK) field lives on a different table, reverse the join direction:
;; make the original field's table the source and join the label table. The original table
;; is typically the large fact table; putting it on the right side of a JOIN can OOM on
;; engines like ClickHouse that materialize the right side in memory.
reversed? (and original-table-id (not= original-table-id field-table-id))
source-table-id (if reversed? original-table-id field-table-id)
- joins (find-all-joins source-table-id
+ joins (find-all-joins mp database-id source-table-id
(cond-> (set (map :field-id constraints))
reversed? (conj field-id)
(not reversed?) (cond-> original-field-id (conj original-field-id))))
@@ -813,12 +819,14 @@
(assert (even? (count options)))
(let [{:as options} options
v->human-readable (delay (schema.metadata-queries/human-readable-remapping-map field-id))
- the-remapped-field-id (delay (let [{:keys [base_type effective_type]} (memoized-field-types-by-id field-id)]
+ the-remapped-field-id (delay (let [metadata-provider (lib-be/application-database-metadata-provider
+ (field/field-id->database-id field-id))
+ {:keys [base-type effective-type]} (lib.metadata/field metadata-provider field-id)]
(binding [*allow-implicit-uuid-field-remapping*
;; For the details on following condition see the dynamic var's
;; docstring.
(or *allow-implicit-uuid-field-remapping*
- (not (isa? (or effective_type base_type) :type/UUID)))]
+ (not (isa? (or effective-type base-type) :type/UUID)))]
(remapped-field-id field-id))))]
(cond
(str/blank? query-string)
diff --git a/src/metabase/parameters/custom_values.clj b/src/metabase/parameters/custom_values.clj
index 82ef6a72773..7c35b873636 100644
--- a/src/metabase/parameters/custom_values.clj
+++ b/src/metabase/parameters/custom_values.clj
@@ -8,6 +8,7 @@
(:require
[clojure.string :as str]
[medley.core :as m]
+ [metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.schema :as lib.schema]
@@ -74,48 +75,55 @@
;; remapped label for a single selected value)
[:exact-value {:optional true} :any]])
-(defn- card-query
- [{query :dataset_query, :keys [id], :as _card}]
+(mu/defn- card-query :- [:maybe ::lib.schema/query]
+ "Build the lib query for the value-source Card identified by `card-id`. `query` is the Card's own `:dataset_query`,
+ which is metadata-providerable, so we resolve the Card as a Lib `:metadata/card` through it. Bulk-loads the metadata
+ it references up front so the column resolution that follows ([[lib/visible-columns]] etc.) hits the provider cache
+ instead of fetching one entity at a time."
+ [card-id :- ::lib.schema.id/card
+ query :- [:maybe ::lib.schema/query]]
(when (seq query)
- (lib/query query (lib.metadata/card query id))))
+ (let [value-source-query (lib/query query (lib.metadata/card query card-id))]
+ (lib-be/bulk-load-query-metadata! query
+ (lib/all-referenced-entity-ids [value-source-query]
+ {:include-implicitly-joinable? true}))
+ value-source-query)))
(mu/defn- values-from-card-query :- [:maybe ::lib.schema/query]
- [{:keys [id], :as card} :- [:and
- :metabase.queries.schema/card
- [:map
- [:id ::lib.schema.id/card]]]
- field-ref :- [:or :mbql.clause/field :mbql.clause/expression]
+ [query :- [:maybe ::lib.schema/query]
+ field-ref :- [:or :mbql.clause/field :mbql.clause/expression]
{:keys [query-string label-field exact-value] :as _opts} :- [:maybe ::values-from-card-query.options]]
- (when-let [query (card-query card)]
- (when-let [visible-columns (or (not-empty (lib/visible-columns query))
- (log/warnf "Cannot get values from Card %d: Card query has no visible columns"
- id))]
- (when-let [value-column (or (lib/find-matching-column query -1 field-ref visible-columns)
- (log/warnf "Cannot get values from Card %d: failed to find column for ref %s\nFound: %s"
- id
- (pr-str field-ref)
- (pr-str (map (some-fn :lib/source-column-alias :name) visible-columns))))]
- (let [label-column (when label-field
- (or (lib/find-matching-column query -1 label-field visible-columns)
- (log/warnf "Cannot get labels from Card %d: failed to find column for ref %s"
- id
- (pr-str label-field))))
- search-column (or label-column value-column)
- value-textual? (lib.types.isa/string? value-column)
- search-textual? (lib.types.isa/string? search-column)
- nonempty ((if value-textual? lib/not-empty lib/not-null) value-column)
- query-filter (cond
- (some? exact-value) (lib/= value-column exact-value)
- query-string (if search-textual?
- (lib/ignore-case (lib/contains search-column query-string))
- (lib/= search-column query-string)))]
- (-> query
- (lib/limit *max-rows*)
- (lib/filter nonempty)
- (cond-> query-filter (lib/filter query-filter))
- (lib/breakout value-column)
- ;; add the label as a second breakout so each row is a [value label] pair
- (cond-> label-column (lib/breakout label-column))))))))
+ (when query
+ (let [card-id (lib/primary-source-card-id query)]
+ (when-let [visible-columns (or (not-empty (lib/visible-columns query))
+ (log/warnf "Cannot get values from Card %d: Card query has no visible columns"
+ card-id))]
+ (when-let [value-column (or (lib/find-matching-column query -1 field-ref visible-columns)
+ (log/warnf "Cannot get values from Card %d: failed to find column for ref %s\nFound: %s"
+ card-id
+ (pr-str field-ref)
+ (pr-str (map (some-fn :lib/source-column-alias :name) visible-columns))))]
+ (let [label-column (when label-field
+ (or (lib/find-matching-column query -1 label-field visible-columns)
+ (log/warnf "Cannot get labels from Card %d: failed to find column for ref %s"
+ card-id
+ (pr-str label-field))))
+ search-column (or label-column value-column)
+ value-textual? (lib.types.isa/string? value-column)
+ search-textual? (lib.types.isa/string? search-column)
+ nonempty ((if value-textual? lib/not-empty lib/not-null) value-column)
+ query-filter (cond
+ (some? exact-value) (lib/= value-column exact-value)
+ query-string (if search-textual?
+ (lib/ignore-case (lib/contains search-column query-string))
+ (lib/= search-column query-string)))]
+ (-> query
+ (lib/limit *max-rows*)
+ (lib/filter nonempty)
+ (cond-> query-filter (lib/filter query-filter))
+ (lib/breakout value-column)
+ ;; add the label as a second breakout so each row is a [value label] pair
+ (cond-> label-column (lib/breakout label-column)))))))))
(defn- result->rows
"Extract rows from a QP result, dropping values for any display columns the QP injected for
@@ -130,6 +138,19 @@
(let [keep-idxs (into [] (keep-indexed (fn [i c] (when-not (drop-names (:name c)) i))) cols)]
(perf/mapv (fn [row] (perf/mapv #(nth row %) keep-idxs)) rows)))))
+(mu/defn- values-from-card* :- ms/FieldValuesResult
+ "Core of [[values-from-card]], working off a prebuilt value-source `query`."
+ [query :- [:maybe ::lib.schema/query]
+ field-ref :- [:or :mbql.clause/field :mbql.clause/expression]
+ opts :- [:maybe ::values-from-card-query.options]]
+ (let [mbql-query (values-from-card-query query field-ref opts)
+ result (some-> mbql-query qp/process-query)
+ values (some-> result result->rows)]
+ {:values (or values [])
+ ;; If the row_count returned = the limit we specified, then it's probably has more than that.
+ ;; If the query has its own limit smaller than *max-rows*, then there's no more values.
+ :has_more_values (= (:row_count result) *max-rows*)}))
+
(mu/defn values-from-card
"Get distinct values of a field from a card.
@@ -150,32 +171,25 @@
([card :- :metabase.queries.schema/card
field-ref :- [:or :mbql.clause/field :mbql.clause/expression]
opts :- [:maybe ::values-from-card-query.options]]
- (let [mbql-query (values-from-card-query card field-ref opts)
- result (some-> mbql-query qp/process-query)
- values (some-> result result->rows)]
- {:values (or values [])
- ;; If the row_count returned = the limit we specified, then it's probably has more than that.
- ;; If the query has its own limit smaller than *max-rows*, then there's no more values.
- :has_more_values (= (:row_count result) *max-rows*)})))
-
-(mu/defn card-values
- "Given a param and query returns the values."
- [{config :values_source_config :as _param} :- ::parameters.schema/parameter
- query-string :- [:maybe ms/NonBlankString]]
- (let [card-id (:card_id config)
- card (t2/select-one :model/Card :id card-id)]
- (values-from-card card
- (lib/->mbql5 (:value_field config))
- (cond-> {:query-string query-string}
- (:label_field config) (assoc :label-field (lib/->mbql5 (:label_field config)))))))
+ (values-from-card* (card-query (:id card) (not-empty (:dataset_query card))) field-ref opts)))
(defn- can-get-card-values?
- [card value-field]
+ "Whether the prebuilt value-source `query` exposes the `value-field` column."
+ [query value-field]
(boolean
- (and (not (:archived card))
- (when-let [query (card-query card)]
- (some? (lib/find-matching-column query -1 (lib/->mbql5 value-field)
- (lib/visible-columns query)))))))
+ (and query
+ (some? (lib/find-matching-column query -1 (lib/->mbql5 value-field)
+ (lib/visible-columns query))))))
+
+(mu/defn- card-values :- ms/FieldValuesResult
+ "Given a prebuilt value-source `query`, the parameter's `config`, and a `query-string`, return the values."
+ [query :- [:maybe ::lib.schema/query]
+ config :- ::parameters.schema/values-source-config
+ query-string :- [:maybe ms/NonBlankString]]
+ (values-from-card* query
+ (lib/->mbql5 (:value_field config))
+ (cond-> {:query-string query-string}
+ (:label_field config) (assoc :label-field (lib/->mbql5 (:label_field config))))))
;;; --------------------------------------------- Putting it together ----------------------------------------------
@@ -190,12 +204,15 @@
default-case-thunk :- [:=> [:cat :any] ms/FieldValuesResult]]
(case (:values_source_type parameter)
:static-list (static-list-values parameter query-string)
- :card (let [card (t2/select-one :model/Card :id (get-in parameter [:values_source_config :card_id]))]
+ :card (let [config (:values_source_config parameter)
+ card (t2/select-one :model/Card :id (:card_id config))]
(when-not (mi/can-read? card)
(throw (ex-info "You don't have permissions to do that." {:status-code 403})))
- (if (can-get-card-values? card (get-in parameter [:values_source_config :value_field]))
- (card-values parameter query-string)
- (default-case-thunk)))
+ (or (when-not (:archived card)
+ (when-let [query (card-query (:id card) (not-empty (:dataset_query card)))]
+ (when (can-get-card-values? query (:value_field config))
+ (card-values query config query-string))))
+ (default-case-thunk)))
nil (default-case-thunk)
(throw (ex-info (tru "Invalid parameter source {0}" (:values_source_type parameter))
{:status-code 400
@@ -239,13 +256,13 @@
[{config :values_source_config :as _param} value]
(when-let [label-field (:label_field config)]
(when-let [card (t2/select-one :model/Card :id (:card_id config))]
- (when (and (not (:archived card))
- (mi/can-read? card)
- (can-get-card-values? card (:value_field config)))
- (first (:values (values-from-card card
- (lib/->mbql5 (:value_field config))
- {:exact-value value
- :label-field (lib/->mbql5 label-field)})))))))
+ (when (and (not (:archived card)) (mi/can-read? card))
+ (when-let [query (card-query (:id card) (not-empty (:dataset_query card)))]
+ (when (can-get-card-values? query (:value_field config))
+ (first (:values (values-from-card* query
+ (lib/->mbql5 (:value_field config))
+ {:exact-value value
+ :label-field (lib/->mbql5 label-field)})))))))))
(mu/defn parameter-remapped-value
"Fetch the remapped value for the given `value` of parameter `param` with default values provided by
diff --git a/src/metabase/parameters/params.clj b/src/metabase/parameters/params.clj
index a7438e671fe..35bfb49aeb2 100644
--- a/src/metabase/parameters/params.clj
+++ b/src/metabase/parameters/params.clj
@@ -17,7 +17,9 @@
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.schema :as lib.schema]
[metabase.lib.schema.id :as lib.schema.id]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.lib.schema.parameter :as lib.schema.parameter]
[metabase.lib.schema.template-tag :as lib.schema.template-tag]
[metabase.parameters.schema :as parameters.schema]
@@ -172,17 +174,34 @@
;;; | DASHBOARD-SPECIFIC |
;;; +----------------------------------------------------------------------------------------------------------------+
-(defn- filterable-columns-for-query
- "Get filterable columns for query."
- [database-id card stage-number]
- (let [metadata-provider (lib-be/application-database-metadata-provider database-id)
- ;; Regular questions are used directly. If a model or metric has been used directly in this card, wrap it into
- ;; a query against that model or metric.
- query (lib/query metadata-provider (if (= :question (:type card))
- (:dataset_query card)
- (lib.metadata/card metadata-provider (:id card))))
- ;; for backward compatibility, append a filter stage only with explicit stage numbers
- query (cond-> query (>= stage-number 0) lib/ensure-filter-stage)]
+(mr/def ::param-dashcard-info
+ "Internal bookkeeping for one parameter `mapping` on a hydrated `dashcard`: the dashcard, the mapping, and the
+ parameter target's Field ID when the target is already field-id-based (nil when it must be resolved from filterable
+ columns)."
+ [:map
+ [:dashcard :map]
+ [:param-mapping ::parameters.schema/parameter-mapping]
+ [:param-target-field-id [:maybe ::lib.schema.id/field]]])
+
+(mu/defn- card->filterable-columns-query :- [:maybe ::lib.schema/query]
+ "Build the lib query whose filterable columns we want for `card` at `stage-number`, or nil when `card` has no query."
+ [card :- :metabase.queries.schema/card
+ stage-number :- :int]
+ (when (and (seq (:dataset_query card)) (pos-int? (:database_id card)))
+ (let [metadata-provider (lib-be/application-database-metadata-provider (:database_id card))
+ ;; Regular questions are used directly. If a model or metric has been used directly in this card, wrap it
+ ;; into a query against that model or metric.
+ query (lib/query metadata-provider (if (= :question (:type card))
+ (:dataset_query card)
+ (lib.metadata/card metadata-provider (:id card))))]
+ ;; for backward compatibility, append a filter stage only with explicit stage numbers
+ (cond-> query (>= stage-number 0) lib/ensure-filter-stage))))
+
+(mu/defn- filterable-columns-for-query :- [:maybe [:sequential ::lib.schema.metadata/column]]
+ "Get the filterable columns of `card`'s query at `stage-number`."
+ [card :- :metabase.queries.schema/card
+ stage-number :- :int]
+ (when-let [query (card->filterable-columns-query card stage-number)]
(when (and (>= stage-number -1) (< stage-number (lib/stage-count query)))
(lib/filterable-columns query stage-number))))
@@ -198,7 +217,7 @@
(seq dataset-query)
(pos-int? database-id))
(assoc-in [:card-id->filterable-columns card-id stage-number]
- (or (filterable-columns-for-query database-id card stage-number) []))))
+ (or (filterable-columns-for-query card stage-number) []))))
(defn- field-id-from-dashcards-filterable-columns
"Update the `ctx` with `field-id`. This function is supposed to be used on params where target is a name field, in
@@ -234,6 +253,21 @@
{:card-id->filterable-columns {}
:param-id->field-ids {}})
+(mu/defn- param-dashcard-info->card :- [:maybe :metabase.queries.schema/card]
+ "The Card on `param-dashcard-info`'s dashcard that its parameter mapping targets: matched by the mapping's `:card_id`
+ against the dashcard's main Card and series Cards, falling back to the main Card when the mapping has no `:card_id`."
+ [{:keys [param-mapping dashcard] :as _param-dashcard-info} :- ::param-dashcard-info]
+ (if-let [card-id (:card_id param-mapping)]
+ (m/find-first #(= (:id %) card-id)
+ (cons (:card dashcard) (:series dashcard)))
+ (:card dashcard)))
+
+(mu/defn- param-dashcard-info->stage-number :- :int
+ "The query stage of `param-dashcard-info`'s parameter target whose filterable columns we need, or -1 when the target
+ doesn't pin a stage."
+ [{:keys [param-mapping] :as _param-dashcard-info} :- ::param-dashcard-info]
+ (get-in param-mapping [:target 2 :stage-number] -1))
+
(mu/defn- field-id-into-context-rf
"Reducing function that generates _field id_ corresponding to `:parameter` of `param-dashcard-info` if possible,
and returns new _context_ (`ctx`) with the _field id_ added.
@@ -255,14 +289,7 @@
merge (:card-id->filterable-columns ctx)))
(:param-id->field-ids ctx))
([ctx {:keys [param-mapping param-target-field-id] :as param-dashcard-info}]
- (let [card-id (:card_id param-mapping)
- card (if card-id
- (m/find-first #(= (:id %) card-id)
- (cons (get-in param-dashcard-info [:dashcard :card])
- (get-in param-dashcard-info [:dashcard :series])))
- (get-in param-dashcard-info [:dashcard :card]))
- param-id (:parameter_id param-mapping)
- stage-number (get-in param-mapping [:target 2 :stage-number] -1)]
+ (let [param-id (:parameter_id param-mapping)]
;; Get the field id from the field-clause if it contains it. This is the common case
;; for mbql queries.
(if param-target-field-id
@@ -270,9 +297,11 @@
;; In case the card doesn't have the same result_metadata columns as filterable columns (a question that
;; aggregates a native query model with a field that was mapped to a db field), we need to load metadata in
;; [[ensure-filterable-columns-for-card]] to find the originating field. (#42829)
- (-> ctx
- (ensure-filterable-columns-for-card card stage-number)
- (field-id-from-dashcards-filterable-columns param-dashcard-info stage-number))))))
+ (let [card (param-dashcard-info->card param-dashcard-info)
+ stage-number (param-dashcard-info->stage-number param-dashcard-info)]
+ (-> ctx
+ (ensure-filterable-columns-for-card card stage-number)
+ (field-id-from-dashcards-filterable-columns param-dashcard-info stage-number)))))))
(defn- find-card-for-mapping
"Find the card that a parameter mapping refers to. Looks up the card by `:card_id` from both
@@ -287,19 +316,48 @@
(string? (:type card)) (update :type keyword)
(seq (:dataset_query card)) (update :dataset_query lib-be/normalize-query))))
+(mu/defn- mapping->param-dashcard-info :- ::param-dashcard-info
+ "Build the `param-dashcard-info` for a parameter `mapping` on `dashcard`, resolving `:param-target-field-id` when the
+ target is already field-id-based."
+ [dashcard :- :map
+ mapping :- ::parameters.schema/parameter-mapping]
+ (let [card (find-card-for-mapping dashcard mapping)]
+ {:dashcard dashcard
+ :param-mapping mapping
+ :param-target-field-id (when (:target mapping)
+ (param-target->field-id (:target mapping) card))}))
+
+(mu/defn- preload-param-filterable-columns! :- :nil
+ "Bulk-load up front the metadata that the per-Card [[filterable-columns-for-query]] computations (run lazily by
+ [[field-id-into-context-rf]]) will need, so those computations hit the metadata-provider cache instead of fetching
+ objects one at a time. Only Cards whose parameter target is not field-id-based need filterable columns; we build
+ their queries here (the same way [[ensure-filterable-columns-for-card]] does, via [[card->filterable-columns-query]]),
+ gather the entities they reference, and bulk-load them per Database. Requires a metadata-provider cache scope (asserted
+ by [[card->filterable-columns-query]]), since there'd otherwise be no shared cache to warm."
+ [param-dashcard-infos :- [:sequential ::param-dashcard-info]]
+ (doseq [[database-id infos] (->> param-dashcard-infos
+ ;; field-id-based targets are resolved without filterable columns, so nothing to preload
+ (remove :param-target-field-id)
+ (group-by (comp :database_id param-dashcard-info->card))
+ (filter (fn [[database-id _infos]] (pos-int? database-id))))]
+ (when-let [queries (not-empty (into [] (keep (fn [info]
+ (card->filterable-columns-query
+ (param-dashcard-info->card info)
+ (param-dashcard-info->stage-number info))))
+ infos))]
+ (lib-be/bulk-load-query-metadata! (lib-be/application-database-metadata-provider database-id)
+ (lib/all-referenced-entity-ids queries {:include-implicitly-joinable? true})))))
+
(mu/defn dashcards->param-id->field-ids* :- [:map-of ::lib.schema.parameter/id [:set ::lib.schema.id/field]]
"Return map of parameter ids to mapped field ids."
[dashcards]
- (letfn [(dashcard->param-dashcard-info [dashcard]
- (for [mapping (:parameter_mappings dashcard)]
- (let [card (find-card-for-mapping dashcard mapping)]
- {:dashcard dashcard
- :param-mapping mapping
- :param-target-field-id (when (:target mapping)
- (param-target->field-id (:target mapping) card))})))]
- (transduce (mapcat dashcard->param-dashcard-info)
- field-id-into-context-rf
- dashcards)))
+ (let [param-dashcard-infos (into []
+ (mapcat (fn [dashcard]
+ (for [mapping (:parameter_mappings dashcard)]
+ (mapping->param-dashcard-info dashcard mapping))))
+ dashcards)]
+ (preload-param-filterable-columns! param-dashcard-infos)
+ (transduce identity field-id-into-context-rf param-dashcard-infos)))
(declare card->template-tag-id->field-ids)
@@ -322,16 +380,11 @@
(mu/defn dashboard-param->field-ids :- [:set ::lib.schema.id/field]
"Return field ids mapped to the parameter. `dashcard` and `card` must be present for each mapping."
[{:keys [mappings]} :- ::parameters.schema/parameter]
- (let [param-id->field-ids (transduce (map (fn [mapping]
- (let [card (find-card-for-mapping (:dashcard mapping) mapping)]
- {:dashcard (:dashcard mapping)
- :param-mapping mapping
- :param-target-field-id (param-target->field-id
- (:target mapping)
- card)})))
- field-id-into-context-rf
- mappings)]
- (into #{} cat (vals param-id->field-ids))))
+ (let [param-dashcard-infos (mapv (fn [mapping]
+ (mapping->param-dashcard-info (:dashcard mapping) mapping))
+ mappings)]
+ (preload-param-filterable-columns! param-dashcard-infos)
+ (into #{} cat (vals (transduce identity field-id-into-context-rf param-dashcard-infos)))))
(defn get-linked-field-ids
"Retrieve a map relating parameter ids to field ids."
diff --git a/src/metabase/public_sharing_rest/api.clj b/src/metabase/public_sharing_rest/api.clj
index f9ebf8ef133..1ccb78d69bd 100644
--- a/src/metabase/public_sharing_rest/api.clj
+++ b/src/metabase/public_sharing_rest/api.clj
@@ -263,6 +263,14 @@
(defn- dashboard-with-uuid [uuid] (public-dashboard :public_uuid uuid))
+(defn- dashboard-with-uuid-for-param-values
+ "Light Dashboard fetch for the public param-value endpoints. Unlike [[dashboard-with-uuid]]/[[public-dashboard]] it
+ skips the full hydration (dashcards/cards/series/tabs/param_fields/query-average-durations) -- `param-values` and
+ `dashboard-param-remapped-value` only need `:resolved-params`, which they hydrate themselves. Matches how the embed
+ param-value endpoints fetch the Dashboard."
+ [uuid]
+ (api/check-404 (t2/select-one :model/Dashboard :public_uuid uuid, :archived false)))
+
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
;;
@@ -532,7 +540,7 @@
[:param-key ms/NonBlankString]]
constraint-param-key->value :- [:map-of string? any?]]
(public-sharing.validation/check-public-sharing-enabled)
- (let [dashboard (dashboard-with-uuid uuid)]
+ (let [dashboard (dashboard-with-uuid-for-param-values uuid)]
(request/as-admin
(binding [qp.perms/*param-values-query* true]
(parameters.dashboard/param-values dashboard param-key constraint-param-key->value)))))
@@ -549,7 +557,7 @@
[:query ms/NonBlankString]]
constraint-param-key->value]
(public-sharing.validation/check-public-sharing-enabled)
- (let [dashboard (dashboard-with-uuid uuid)]
+ (let [dashboard (dashboard-with-uuid-for-param-values uuid)]
(request/as-admin
(binding [qp.perms/*param-values-query* true]
(parameters.dashboard/param-values dashboard param-key constraint-param-key->value query)))))
@@ -565,7 +573,7 @@
[:param-key ms/NonBlankString]]
{:keys [value]} :- [:map [:value :any]]]
(public-sharing.validation/check-public-sharing-enabled)
- (let [dashboard (dashboard-with-uuid uuid)]
+ (let [dashboard (dashboard-with-uuid-for-param-values uuid)]
(request/as-admin
(binding [qp.perms/*param-values-query* true]
(parameters.dashboard/dashboard-param-remapped-value dashboard param-key (codec/url-decode value))))))
diff --git a/src/metabase/queries_rest/api/card.clj b/src/metabase/queries_rest/api/card.clj
index edda84f8d54..160eb4801d7 100644
--- a/src/metabase/queries_rest/api/card.clj
+++ b/src/metabase/queries_rest/api/card.clj
@@ -370,12 +370,18 @@
(def ^:private supported-series-display-type (set (keys (methods series-are-compatible?))))
-(defn- fetch-compatible-series*
+(mu/defn- fetch-compatible-series* :- [:sequential :map]
"Implementation of `fetch-compatible-series`.
Provide `page-size` to limit the number of cards returned, it does not guaranteed to return exactly `page-size` cards.
Use `fetch-compatible-series` for that."
- [card database-id->metadata-provider {:keys [query last-cursor page-size exclude-ids] :as _options}]
+ [card :- :map
+ {:keys [query last-cursor page-size exclude-ids] :as _options}
+ :- [:map {:closed true}
+ [:query {:optional true} [:maybe ms/NonBlankString]]
+ [:last-cursor {:optional true} [:maybe ms/PositiveInt]]
+ [:page-size {:optional true} [:maybe ms/PositiveInt]]
+ [:exclude-ids {:optional true} [:maybe [:sequential ms/PositiveInt]]]]]
(let [matching-cards (t2/select :model/Card
:archived false
:display [:in supported-series-display-type]
@@ -396,15 +402,6 @@
;; this is just a heuristic, but it should be good enough
page-size
(assoc :limit (+ 10 page-size))))
- database-ids (set (keys database-id->metadata-provider))
- database-id->metadata-provider (->> matching-cards
- (filter #(or (nil? (get-in % [:visualization_settings :graph.metrics]))
- (nil? (get-in % [:visualization_settings :graph.dimensions]))))
- (keep :database_id)
- (set)
- (remove #(contains? database-ids %))
- (into database-id->metadata-provider
- (map (juxt identity lib-be/application-database-metadata-provider))))
compatible-cards (->> matching-cards
(filter mi/can-read?)
(filter #(or
@@ -414,8 +411,8 @@
(= (:query_type %) :native)
(series-are-compatible? card %))))]
(if page-size
- [database-id->metadata-provider (take page-size compatible-cards)]
- [database-id->metadata-provider compatible-cards])))
+ (take page-size compatible-cards)
+ compatible-cards)))
(defn- fetch-compatible-series
"Fetch a list of compatible series for `card`.
@@ -426,14 +423,10 @@
- last-cursor: the id of the last card from the previous page
- page-size: is nullable, it'll try to fetches exactly `page-size` cards if there are enough cards."
([card options]
- (fetch-compatible-series
- card
- options
- {(:database_id card) (lib-be/application-database-metadata-provider (:database_id card))}
- []))
-
- ([card {:keys [page-size] :as options} database-id->metadata-provider current-cards]
- (let [[database-id->metadata-provider cards] (fetch-compatible-series* card database-id->metadata-provider options)
+ (fetch-compatible-series card options []))
+
+ ([card {:keys [page-size] :as options} current-cards]
+ (let [cards (fetch-compatible-series* card options)
new-cards (concat current-cards cards)]
;; if the total card fetches is less than page-size and there are still more, continue fetching
(if (and (some? page-size)
@@ -443,7 +436,6 @@
(merge options
{:page-size (- page-size (count cards))
:last-cursor (:id (last cards))})
- database-id->metadata-provider
new-cards)
new-cards))))
@@ -784,9 +776,8 @@
"Get all of the required query metadata for a card."
[{:keys [id]} :- [:map
[:id [:or ms/PositiveInt ms/NanoIdString]]]]
- (lib-be/with-metadata-provider-cache
- (let [resolved-id (eid-translation/->id-or-404 :card id)]
- (queries/batch-fetch-card-metadata [(get-card resolved-id)]))))
+ (let [resolved-id (eid-translation/->id-or-404 :card id)]
+ (queries/batch-fetch-card-metadata [(get-card resolved-id)])))
;;; ------------------------------------------------- Deleting Cards -------------------------------------------------
diff --git a/src/metabase/query_processor/api.clj b/src/metabase/query_processor/api.clj
index 966650473d0..7071f550a94 100644
--- a/src/metabase/query_processor/api.clj
+++ b/src/metabase/query_processor/api.clj
@@ -186,11 +186,10 @@
[:database ms/PositiveInt]
[:settings {:optional true} [:maybe [:map
[:include_sensitive_fields {:optional true} :boolean]]]]]]
- (lib-be/with-metadata-provider-cache
- (queries/batch-fetch-query-metadata
- [query]
- (when-some [include-sensitive-fields (get-in query [:settings :include_sensitive_fields])]
- {:include-sensitive-fields? include-sensitive-fields}))))
+ (queries/batch-fetch-query-metadata
+ [query]
+ (when-some [include-sensitive-fields (get-in query [:settings :include_sensitive_fields])]
+ {:include-sensitive-fields? include-sensitive-fields})))
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
diff --git a/test/metabase/lib/walk/util_test.cljc b/test/metabase/lib/walk/util_test.cljc
index 90d4a5386d3..1fea88c95b7 100644
--- a/test/metabase/lib/walk/util_test.cljc
+++ b/test/metabase/lib/walk/util_test.cljc
@@ -345,3 +345,51 @@
:segment #{}
:snippet #{snippet-id}}
(lib/all-referenced-entity-ids [query])))))
+
+(deftest ^:parallel all-referenced-entity-ids-implicitly-joinable-table-test
+ (testing ":include-implicitly-joinable? adds the source Table's columns' FK-target Tables to :table"
+ (let [query (lib/query meta/metadata-provider (meta/table-metadata :orders))]
+ (is (= #{(meta/id :orders)}
+ (:table (lib/all-referenced-entity-ids [query])))
+ "without the option, only the source Table is referenced")
+ (is (= #{(meta/id :orders) (meta/id :people) (meta/id :products)}
+ (:table (lib/all-referenced-entity-ids [query] {:include-implicitly-joinable? true})))
+ "with the option, the FK-target Tables of the source columns are included too"))))
+
+(deftest ^:parallel all-referenced-entity-ids-implicitly-joinable-card-test
+ (testing ":include-implicitly-joinable? adds a source Card's result-metadata columns' FK-target Tables to :table"
+ (let [card-id 1
+ mp (lib.tu/metadata-provider-with-card-from-query
+ meta/metadata-provider card-id
+ (lib/query meta/metadata-provider (meta/table-metadata :orders)))
+ query (lib/query mp (lib.metadata/card mp card-id))]
+ (is (= #{}
+ (:table (lib/all-referenced-entity-ids [query])))
+ "without the option, the Card source references no Tables")
+ (is (= #{(meta/id :people) (meta/id :products)}
+ (:table (lib/all-referenced-entity-ids [query] {:include-implicitly-joinable? true})))
+ "with the option, the Card columns' FK-target Tables are included"))))
+
+(deftest ^:parallel all-referenced-entity-ids-implicitly-joinable-result-metadata-fk-override-test
+ (testing ":include-implicitly-joinable? follows an FK target set on a Card's result metadata even when the raw Field
+ has no such FK"
+ (let [products-query (lib/query meta/metadata-provider (meta/table-metadata :products))
+ returned (lib/returned-columns products-query)
+ plain-mp (lib.tu/metadata-provider-with-card-from-query meta/metadata-provider 1 products-query)
+ override-mp (lib.tu/metadata-provider-with-card-from-query
+ meta/metadata-provider 2 products-query
+ {:result-metadata (mapv (fn [col]
+ (cond-> col
+ (= (:id col) (meta/id :products :price))
+ (assoc :fk-target-field-id (meta/id :people :id))))
+ returned)})]
+ (is (= #{}
+ (:table (lib/all-referenced-entity-ids
+ [(lib/query plain-mp (lib.metadata/card plain-mp 1))]
+ {:include-implicitly-joinable? true})))
+ "PRODUCTS columns have no raw FKs, so nothing is implicitly joinable")
+ (is (= #{(meta/id :people)}
+ (:table (lib/all-referenced-entity-ids
+ [(lib/query override-mp (lib.metadata/card override-mp 2))]
+ {:include-implicitly-joinable? true})))
+ "the result-metadata FK-target override pulls in PEOPLE, which the raw PRODUCTS.PRICE Field does not reference"))))
diff --git a/test/metabase/parameters/chain_filter_test.clj b/test/metabase/parameters/chain_filter_test.clj
index 159c7be7a23..aa6ebac9646 100644
--- a/test/metabase/parameters/chain_filter_test.clj
+++ b/test/metabase/parameters/chain_filter_test.clj
@@ -2,6 +2,7 @@
(:require
[clojure.set :as set]
[clojure.test :refer :all]
+ [metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.util :as lib.util]
@@ -63,6 +64,7 @@
(are [op] (some? (#'chain-filter/add-filter
(lib/query mp (lib.metadata/table mp (mt/id :venues)))
(mt/id :venues)
+ {(mt/id :venues :name) (lib.metadata/field mp (mt/id :venues :name))}
{:field-id (mt/id :venues :name)
:op op
:value value
@@ -294,7 +296,7 @@
(binding [chain-filter/*enable-reverse-joins* false]
(mt/$ids nil
(is (= [{:lhs {:table $$venues, :field %venues.category_id}, :rhs {:table $$categories, :field %categories.id}}]
- (#'chain-filter/find-all-joins $$venues #{%categories.name %users.id})))))))
+ (#'chain-filter/find-all-joins (mt/metadata-provider) (mt/id) $$venues #{%categories.name %users.id})))))))
(deftest ^:parallel find-all-joins-test-2
(mt/dataset airports
@@ -305,7 +307,7 @@
:rhs {:table $$municipality, :field %municipality.id}}
{:lhs {:table $$municipality, :field %municipality.region_id}
:rhs {:table $$region, :field %region.id}}]
- (#'chain-filter/find-all-joins $$airport #{%region.name %municipality.name %region.id}))))))))
+ (#'chain-filter/find-all-joins (mt/metadata-provider) (mt/id) $$airport #{%region.name %municipality.name %region.id}))))))))
(deftest ^:parallel multi-hop-test
(mt/dataset airports
@@ -881,13 +883,14 @@
[:field {} (mt/id :checkins :date)]
-32
:week]]}]}
- (#'chain-filter/add-filter
- (let [mp (mt/metadata-provider)]
- (lib/query mp (lib.metadata/table mp (mt/id :checkins))))
- $$checkins
- {:field-id %checkins.date
- :op :=
- :value "past32weeks"}))))))
+ (let [mp (mt/metadata-provider)]
+ (#'chain-filter/add-filter
+ (lib/query mp (lib.metadata/table mp (mt/id :checkins)))
+ $$checkins
+ {%checkins.date (lib.metadata/field mp %checkins.date)}
+ {:field-id %checkins.date
+ :op :=
+ :value "past32weeks"})))))))
(mt/defdataset nil-values-dataset
[["tbl"
@@ -1045,3 +1048,93 @@
(t2/update! :model/Field {:id %users.id} {:active false})
(testing "there are no connections when PK is inactive"
(is (nil? (#'chain-filter/find-joins (mt/id) $$messages $$users)))))))))
+
+;;; --------------------- chain-filter-mbql-query metadata app-DB call counts (no metadata N+1) ---------------------
+;;;
+;;; `chain-filter-mbql-query` bulk-loads all the Field/Table metadata it needs while building the query (see
+;;; `find-all-joins`, `add-joins`, `add-filters`), so the number of app-DB calls is bounded: it does NOT grow with the
+;;; number of constraints, nor with the number/depth of joins. A second build reuses the request-scoped
+;;; metadata-provider cache and makes no app-DB calls at all. If a metadata fetch regresses to
+;;; one-object-at-a-time, these exact counts change.
+
+(defn- chain-filter-mbql-query-call-counts
+ "Warm the process-global memoized caches (FK graph, `field-id->database-id`) for `field-id`/`constraints`, then
+ return the app-DB call counts of a first build (fresh provider cache) and a second build (warm cache)."
+ [field-id constraints]
+ (#'chain-filter/chain-filter-mbql-query field-id constraints nil)
+ (lib-be/with-metadata-provider-cache
+ {:first (t2/with-call-count [call-count] (#'chain-filter/chain-filter-mbql-query field-id constraints nil) (call-count))
+ :second (t2/with-call-count [call-count] (#'chain-filter/chain-filter-mbql-query field-id constraints nil) (call-count))}))
+
+(deftest ^:parallel chain-filter-mbql-query-no-constraints-test
+ (mt/dataset test-data
+ (mt/$ids
+ (let [{:keys [first second]} (chain-filter-mbql-query-call-counts %venues.name [])]
+ ;; - fetch the source Field (the one we return values of)
+ ;; - fetch the source Table
+ ;; - fetch the Database
+ (is (= 3 first))
+ (testing "a second build reuses the cached metadata and makes no app-DB calls"
+ (is (= 0 second)))))))
+
+(deftest ^:parallel chain-filter-mbql-query-one-constraint-test
+ (mt/dataset test-data
+ (mt/$ids
+ ;; a constraint on the source Table does not add a join
+ (let [{:keys [first second]} (chain-filter-mbql-query-call-counts %venues.name [(shorthand->constraint %venues.price 3)])]
+ ;; - fetch the source Field
+ ;; - fetch the source Table
+ ;; - fetch the Database
+ ;; - bulk-fetch the constraint Field(s)
+ (is (= 4 first))
+ (testing "a second build reuses the cached metadata and makes no app-DB calls"
+ (is (= 0 second)))))))
+
+(deftest ^:parallel chain-filter-mbql-query-three-constraints-test
+ (mt/dataset test-data
+ (mt/$ids
+ (let [{:keys [first second]} (chain-filter-mbql-query-call-counts
+ %venues.name
+ [(shorthand->constraint %venues.price 3)
+ (shorthand->constraint %venues.id 1)
+ (shorthand->constraint %venues.latitude 1)])]
+ ;; same as the one-constraint case -- the constraint Fields are bulk-loaded together, not one at a time:
+ ;; - fetch the source Field
+ ;; - fetch the source Table
+ ;; - fetch the Database
+ ;; - bulk-fetch the (three) constraint Fields
+ (is (= 4 first))
+ (testing "a second build reuses the cached metadata and makes no app-DB calls"
+ (is (= 0 second)))))))
+
+(deftest ^:parallel chain-filter-mbql-query-one-join-test
+ (mt/dataset test-data
+ (mt/$ids
+ ;; a constraint on another Table (categories) forces a venues -> categories join
+ (let [{:keys [first second]} (chain-filter-mbql-query-call-counts %venues.name [(shorthand->constraint %categories.name "BBQ")])]
+ ;; - fetch the source Field
+ ;; - fetch the source Table
+ ;; - fetch the Database
+ ;; - bulk-fetch the constraint Field(s)
+ ;; - bulk-fetch the join Fields
+ ;; - bulk-fetch the join Table(s)
+ (is (= 6 first))
+ (testing "a second build reuses the cached metadata and makes no app-DB calls"
+ (is (= 0 second)))))))
+
+(deftest ^:parallel chain-filter-mbql-query-multi-hop-joins-test
+ (mt/dataset airports
+ (mt/$ids
+ ;; airport -> municipality -> region is two joins
+ (let [{:keys [first second]} (chain-filter-mbql-query-call-counts %airport.name [(shorthand->constraint %region.name "x")])]
+ ;; same count as the single-join case -- the join Fields and Tables are each bulk-loaded in one call,
+ ;; regardless of how many joins there are:
+ ;; - fetch the source Field
+ ;; - fetch the source Table
+ ;; - fetch the Database
+ ;; - bulk-fetch the constraint Field(s)
+ ;; - bulk-fetch the join Fields (across both joins)
+ ;; - bulk-fetch the join Tables (both of them)
+ (is (= 6 first))
+ (testing "a second build reuses the cached metadata and makes no app-DB calls"
+ (is (= 0 second)))))))
diff --git a/test/metabase/parameters/custom_values_test.clj b/test/metabase/parameters/custom_values_test.clj
index f70363f77e0..ce78c83bed2 100644
--- a/test/metabase/parameters/custom_values_test.clj
+++ b/test/metabase/parameters/custom_values_test.clj
@@ -2,11 +2,13 @@
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.parameters.custom-values-test]}}}}}}
(:require
[clojure.test :refer :all]
+ [metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.parameters.custom-values :as custom-values]
[metabase.test :as mt]
- [metabase.test.fixtures :as fixtures]))
+ [metabase.test.fixtures :as fixtures]
+ [toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -629,3 +631,29 @@
(deftest ^:parallel pk-of-fk-pk-field-ids-test-6
(testing "just a non-key"
(is (nil? (custom-values/pk-of-fk-pk-field-ids [(mt/id :people :name)])))))
+
+(deftest ^:parallel card-query-bulk-loads-metadata-test
+ (testing "card-query bulk-loads the value-source Card's metadata up front, so resolving the value field's column hits
+ the provider cache instead of fetching one entity at a time (no N+1)"
+ (let [mp (mt/metadata-provider)
+ orders-query (lib/query mp (lib.metadata/table mp (mt/id :orders)))]
+ (mt/with-temp [:model/Card card {:database_id (mt/id)
+ :table_id (mt/id :orders)
+ :type :question
+ :dataset_query orders-query
+ :result_metadata (lib/returned-columns orders-query)}]
+ (let [card (t2/select-one :model/Card :id (:id card))
+ value-field [:field "TOTAL" {:base-type :type/Float}]]
+ ;; The 6 batched loads are:
+ ;; - the source Card
+ ;; - the source Database
+ ;; - the Card's result-metadata Fields
+ ;; - those Fields' FK targets
+ ;; - the FK-target Tables
+ ;; - those Tables' columns
+ ;; This count is constant -- it must NOT grow with the number of columns/Tables (that would be the N+1 this
+ ;; bulk-loading exists to prevent).
+ (is (= 6 (lib-be/with-metadata-provider-cache
+ (t2/with-call-count [call-count]
+ (#'custom-values/can-get-card-values? (#'custom-values/card-query (:id card) (:dataset_query card)) value-field)
+ (call-count))))))))))
diff --git a/test/metabase/parameters/params_test.clj b/test/metabase/parameters/params_test.clj
index 884d751c7fd..d134b2cf0d5 100644
--- a/test/metabase/parameters/params_test.clj
+++ b/test/metabase/parameters/params_test.clj
@@ -3,6 +3,9 @@
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.parameters.params-test]}}}}}}
(:require
[clojure.test :refer :all]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.parameters.params :as params]
[metabase.public-sharing-rest.api-test :as public-test]
[metabase.test :as mt]
@@ -169,7 +172,8 @@
:target [:dimension
[:field "CATEGORY" {:base-type :type/Text}]
{:stage-number 2}]}]}]
- (let [param-fields (-> (t2/hydrate dashboard :param_fields) :param_fields)]
+ (let [param-fields (lib-be/with-metadata-provider-cache
+ (-> (t2/hydrate dashboard :param_fields) :param_fields))]
(is (=? {"p1" [{:id (mt/id :products :id)}]
"p2" [{:id (mt/id :products :id)}]
"p3" [{:id (mt/id :products :id)}]
@@ -266,3 +270,44 @@
(is (= #{(mt/id :orders :id)
(mt/id :products :id)}
(params/card->template-tag-field-ids card))))))))
+
+(deftest ^:parallel dashcards->param-field-ids-bulk-loads-metadata-test
+ (testing "name-based parameter targets are resolved via filterable-columns, whose metadata is bulk-loaded across all
+ the dashboard's cards up front instead of one card at a time (no N+1)"
+ (mt/dataset test-data
+ (let [mp (mt/metadata-provider)
+ orders-query (lib/query mp (lib.metadata/table mp (mt/id :orders)))
+ products-query (lib/query mp (lib.metadata/table mp (mt/id :products)))]
+ (mt/with-temp
+ [:model/Card src1 {:database_id (mt/id) :table_id (mt/id :orders)
+ :dataset_query orders-query
+ :result_metadata (lib/returned-columns orders-query)}
+ :model/Card agg1 {:database_id (mt/id)
+ :dataset_query {:database (mt/id) :type :query
+ :query {:source-table (str "card__" (:id src1)) :aggregation [[:count]]}}}
+ :model/Card src2 {:database_id (mt/id) :table_id (mt/id :products)
+ :dataset_query products-query
+ :result_metadata (lib/returned-columns products-query)}
+ :model/Card agg2 {:database_id (mt/id)
+ :dataset_query {:database (mt/id) :type :query
+ :query {:source-table (str "card__" (:id src2)) :aggregation [[:count]]}}}
+ :model/Dashboard {dash-id :id} {}
+ :model/DashboardCard dc1 {:dashboard_id dash-id :card_id (:id agg1)
+ :parameter_mappings [{:parameter_id "p1" :card_id (:id agg1)
+ :target [:dimension [:field "PRODUCT_ID" {:base-type :type/Integer}]]}]}
+ :model/DashboardCard dc2 {:dashboard_id dash-id :card_id (:id agg2)
+ :parameter_mappings [{:parameter_id "p2" :card_id (:id agg2)
+ :target [:dimension [:field "CATEGORY" {:base-type :type/Text}]]}]}]
+ (let [dashcards (-> (t2/select :model/DashboardCard :id [:in [(:id dc1) (:id dc2)]])
+ (t2/hydrate :card :series))]
+ ;; Each is a single set load, so this count is constant in the number of cards -- it must NOT grow with more
+ ;; dashcards/Cards/Tables (that would be the N+1 this preloading exists to prevent).
+ ;; - one bulk fetch of the source Cards
+ ;; - one bulk fetch of those Cards' result-metadata Fields
+ ;; - one bulk fetch of those Fields' FK targets
+ ;; - one bulk fetch of the FK-target Tables
+ ;; - one bulk fetch of those Tables' columns
+ (is (= 5 (lib-be/with-metadata-provider-cache
+ (t2/with-call-count [call-count]
+ (params/dashcards->param-field-ids dashcards)
+ (call-count)))))))))))