The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix lib-related N+1 issues with DB calls in parameters (#75888)
Alexander Polyankine025170e85f7
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 5121440d068..72f2bc27a34 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1486,8 +1486,7 @@modelsquery-processorutil- warehouse-schema- warehouses}+ warehouse-schema}:model-imports #{:model/Card:model/Dashboard:model/Fielddiff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.cljindex eaf0a13fe74..ea4e64a92c9 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj@@ -15,7 +15,6 @@[metabase.collections.models.collection.root :as collection.root][metabase.documents.schema :as documents.schema][metabase.graph.core :as graph]- [metabase.lib-be.core :as lib-be][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.models.interface :as mi]@@ -496,16 +495,15 @@[:id {:optional true} ms/PositiveInt][:type {:optional true} ::deps.dependency-types/dependency-types]]](api/read-check (deps.dependency-types/dependency-type->model type) id)- (lib-be/with-metadata-provider-cache- (let [starting-nodes [[type id]]- upstream-graph (readable-graph-dependencies {:include-archived-items :all})- downstream-graph (graph/cached-graph (readable-graph-dependents))- edge-graph (graph/cached-graph (readable-graph-dependents {:include-archived-items :all}))- nodes (into (set starting-nodes)- (graph/transitive upstream-graph starting-nodes))- edges (graph/edges-between edge-graph nodes)]- {:nodes (expanded-nodes downstream-graph nodes {:include-errors? false})- :edges edges})))+ (let [starting-nodes [[type id]]+ upstream-graph (readable-graph-dependencies {:include-archived-items :all})+ downstream-graph (graph/cached-graph (readable-graph-dependents))+ edge-graph (graph/cached-graph (readable-graph-dependents {:include-archived-items :all}))+ nodes (into (set starting-nodes)+ (graph/transitive upstream-graph starting-nodes))+ edges (graph/edges-between edge-graph nodes)]+ {:nodes (expanded-nodes downstream-graph nodes {:include-errors? false})+ :edges edges}))(def ^:private sort-directions"Valid sort directions for dependency item endpoints."@@ -609,37 +607,36 @@sort-column :namesort-direction :asc}} :- dependents-args](api/read-check (deps.dependency-types/dependency-type->model type) id)- (lib-be/with-metadata-provider-cache- (let [downstream-graph (graph/cached-graph (readable-graph-dependents {:broken broken}))- nodes (-> (graph/children-of downstream-graph [[type id]])- (get [type id]))- dep-types-set (cond- (nil? dependent-types) deps.dependency-types/dependency-types- (sequential? dependent-types) (set dependent-types)- :else #{dependent-types})- card-types-set (cond- (nil? dependent-card-types) lib.schema.metadata/card-types- (sequential? dependent-card-types) (set dependent-card-types)- :else #{dependent-card-types})- dependents-filter- (comp- ;; Filter by dependent types and card types- (filter (fn [node]- (and (or (nil? dep-types-set)- (contains? dep-types-set (:type node)))- (or (not= (:type node) :card)- (nil? card-types-set)- (contains? card-types-set (-> node :data :type))))))- ;; Filter out personal collections unless explicitly included- (if include-personal-collections- identity- (remove in-personal-collection?))- ;; Filter by query (sandboxes are excluded since they have no name or location)- (if query- (filter #(entity-matches-query? % query))- identity))]- (-> (into [] dependents-filter (expanded-nodes downstream-graph nodes {:include-errors? false}))- (sort-dependents sort-column sort-direction)))))+ (let [downstream-graph (graph/cached-graph (readable-graph-dependents {:broken broken}))+ nodes (-> (graph/children-of downstream-graph [[type id]])+ (get [type id]))+ dep-types-set (cond+ (nil? dependent-types) deps.dependency-types/dependency-types+ (sequential? dependent-types) (set dependent-types)+ :else #{dependent-types})+ card-types-set (cond+ (nil? dependent-card-types) lib.schema.metadata/card-types+ (sequential? dependent-card-types) (set dependent-card-types)+ :else #{dependent-card-types})+ dependents-filter+ (comp+ ;; Filter by dependent types and card types+ (filter (fn [node]+ (and (or (nil? dep-types-set)+ (contains? dep-types-set (:type node)))+ (or (not= (:type node) :card)+ (nil? card-types-set)+ (contains? card-types-set (-> node :data :type))))))+ ;; Filter out personal collections unless explicitly included+ (if include-personal-collections+ identity+ (remove in-personal-collection?))+ ;; Filter by query (sandboxes are excluded since they have no name or location)+ (if query+ (filter #(entity-matches-query? % query))+ identity))]+ (-> (into [] dependents-filter (expanded-nodes downstream-graph nodes {:include-errors? false}))+ (sort-dependents sort-column sort-direction))))(defn- entity-type-config[entity-type]@@ -1011,51 +1008,50 @@sort-column :namesort-direction :asc}} :- broken-dependents-args](api/read-check (deps.dependency-types/dependency-type->model entity-type) id)- (lib-be/with-metadata-provider-cache- (let [normalize-types (fn normalize-types [types]- (if (keyword? types)- [(name types)]- (not-empty (map name types))))- dep-types (normalize-types dependent-types)- card-types (normalize-types dependent-card-types)- where-clause (cond-> [:and- [:= :afe.source_entity_type (name entity-type)]- [:= :afe.source_entity_id id]- [:= :af.result false]- (visible-entities-filter-clause- :afe.analyzed_entity_type- :afe.analyzed_entity_id- {:include-archived-items :exclude})]- dep-types (conj [:in :afe.analyzed_entity_type dep-types])- card-types (conj [:or- [:!= :afe.analyzed_entity_type [:inline "card"]]- [:in :rc.type card-types]]))- broken-entity-pairs- (t2/query (cond-> {:select-distinct [[:afe.analyzed_entity_type :entity_type]- [:afe.analyzed_entity_id :entity_id]]- :from [[:analysis_finding_error :afe]]- :join [[:analysis_finding :af]- [:and- [:= :af.analyzed_entity_type :afe.analyzed_entity_type]- [:= :af.analyzed_entity_id :afe.analyzed_entity_id]]]- :where where-clause}- card-types (assoc :left-join [[:report_card :rc]- [:and- [:= :afe.analyzed_entity_type [:inline "card"]]- [:= :rc.id :afe.analyzed_entity_id]]])))- nodes (map (fn [{:keys [entity_type entity_id]}]- [(keyword entity_type) entity_id])- broken-entity-pairs)- nodes-by-type (-> (group-by first nodes)- (update-vals #(map second %)))]- (-> (into [] (cond-> (map (fn [[[entity-type entity-id] entity]]- {:id entity-id- :type entity-type- :data (-> (select-keys entity (entity-keys entity-type))- (update-vals format-subentity))}))- (not include-personal-collections) (comp (remove in-personal-collection?)))- (fetch-and-hydrate-nodes nodes-by-type))- (sort-dependents sort-column sort-direction)))))+ (let [normalize-types (fn normalize-types [types]+ (if (keyword? types)+ [(name types)]+ (not-empty (map name types))))+ dep-types (normalize-types dependent-types)+ card-types (normalize-types dependent-card-types)+ where-clause (cond-> [:and+ [:= :afe.source_entity_type (name entity-type)]+ [:= :afe.source_entity_id id]+ [:= :af.result false]+ (visible-entities-filter-clause+ :afe.analyzed_entity_type+ :afe.analyzed_entity_id+ {:include-archived-items :exclude})]+ dep-types (conj [:in :afe.analyzed_entity_type dep-types])+ card-types (conj [:or+ [:!= :afe.analyzed_entity_type [:inline "card"]]+ [:in :rc.type card-types]]))+ broken-entity-pairs+ (t2/query (cond-> {:select-distinct [[:afe.analyzed_entity_type :entity_type]+ [:afe.analyzed_entity_id :entity_id]]+ :from [[:analysis_finding_error :afe]]+ :join [[:analysis_finding :af]+ [:and+ [:= :af.analyzed_entity_type :afe.analyzed_entity_type]+ [:= :af.analyzed_entity_id :afe.analyzed_entity_id]]]+ :where where-clause}+ card-types (assoc :left-join [[:report_card :rc]+ [:and+ [:= :afe.analyzed_entity_type [:inline "card"]]+ [:= :rc.id :afe.analyzed_entity_id]]])))+ nodes (map (fn [{:keys [entity_type entity_id]}]+ [(keyword entity_type) entity_id])+ broken-entity-pairs)+ nodes-by-type (-> (group-by first nodes)+ (update-vals #(map second %)))]+ (-> (into [] (cond-> (map (fn [[[entity-type entity-id] entity]]+ {:id entity-id+ :type entity-type+ :data (-> (select-keys entity (entity-keys entity-type))+ (update-vals format-subentity))}))+ (not include-personal-collections) (comp (remove in-personal-collection?)))+ (fetch-and-hydrate-nodes nodes-by-type))+ (sort-dependents sort-column sort-direction))))(api.macros/defendpoint :get "/backfill-status" :- [:map[:complete :boolean]]diff --git a/enterprise/backend/src/metabase_enterprise/replacement/runner.clj b/enterprise/backend/src/metabase_enterprise/replacement/runner.cljindex facc151bc2d..83bea598b64 100644--- a/enterprise/backend/src/metabase_enterprise/replacement/runner.clj+++ b/enterprise/backend/src/metabase_enterprise/replacement/runner.clj@@ -84,7 +84,7 @@(vals measures))))](when (seq queries);; Extract all referenced entity IDs across all queries- (let [referenced-ids (lib/all-referenced-entity-ids queries)]+ (let [referenced-ids (lib/all-referenced-entity-ids queries {:include-implicitly-joinable? true})];; Bulk load all metadata at once(lib-be/bulk-load-query-metadata! metadata-provider referenced-ids)))(merge {} cards tables dashboards transforms segments measures))))diff --git a/src/metabase/content_verification/impl.clj b/src/metabase/content_verification/impl.cljindex 4ba32aac292..7c24b6da1c9 100644--- a/src/metabase/content_verification/impl.clj+++ b/src/metabase/content_verification/impl.clj@@ -31,10 +31,14 @@;; item that comes into this comes out. The nested hydration is positional, not by an id so everything that comes in;; must go out in the same order(when (seq items)- (let [item-ids (not-empty (keep :id items))+ (let [items* (keep identity items)+ item-ids (not-empty (keep :id items*))+ ;; constrain on `:moderated_item_type` too so the `(moderated_item_type, moderated_item_id)` index is used+ item-types (not-empty (into #{} (map (comp keyword object->type)) items*))all-reviews (when item-ids(group-by (juxt :moderated_item_type :moderated_item_id)(t2/select :model/ModerationReview+ :moderated_item_type [:in item-types]:moderated_item_id [:in item-ids]{:order-by [[:id :desc]]})))](for [item items]@@ -58,9 +62,13 @@"Hydrate moderation status onto a seq of items"[items](when (seq items)- (let [item-ids (seq (keep :id items))+ (let [items* (keep identity items)+ item-ids (seq (keep :id items*))+ ;; constrain on `:moderated_item_type` too so the `(moderated_item_type, moderated_item_id)` index is used+ item-types (not-empty (into #{} (map (comp keyword object->type)) items*))type+id->status (when item-ids(->> (t2/select [:model/ModerationReview :moderated_item_id :moderated_item_type :status]+ :moderated_item_type [:in item-types]:moderated_item_id [:in item-ids]:most_recent true{:order-by [[:id :desc]]})diff --git a/src/metabase/dashboards/models/dashboard.clj b/src/metabase/dashboards/models/dashboard.cljindex d61f51cc05f..428282b6ae1 100644--- a/src/metabase/dashboards/models/dashboard.clj+++ b/src/metabase/dashboards/models/dashboard.clj@@ -407,7 +407,7 @@:dashcard ...:target [:dimension [:field-id 264]]}}}}"[_model k dashboards]- (let [dashboards-with-cards (t2/hydrate dashboards [:dashcards :card])]+ (let [dashboards-with-cards (t2/hydrate dashboards [:dashcards :card :series])](map #(assoc %1 k %2) dashboards (map dashboard->resolved-params dashboards-with-cards))))(defmethod mi/exclude-internal-content-hsql :model/Dashboarddiff --git a/src/metabase/dashboards_rest/api.clj b/src/metabase/dashboards_rest/api.cljindex 5857b728c43..6c131e25166 100644--- a/src/metabase/dashboards_rest/api.clj+++ b/src/metabase/dashboards_rest/api.clj@@ -139,28 +139,27 @@[:collection_position {:optional true} [:maybe ms/PositiveInt]]]];; if we're trying to save the new dashboard in a Collection make sure we have permissions to do that(api/create-check :model/Dashboard {:collection_id collection_id})- (lib-be/with-metadata-provider-cache- (let [dashboard-data {:name name- :description description- :parameters (or parameters [])- :creator_id api/*current-user-id*- :cache_ttl cache_ttl- :collection_id collection_id- :collection_position collection_position}- dash (t2/with-transaction [_conn]- ;; Adding a new dashboard at `collection_position` could cause other dashboards in this- ;; collection to change position, check that and fix up if needed- (api/maybe-reconcile-collection-position! dashboard-data)- ;; Ok, now save the Dashboard- (first (t2/insert-returning-instances! :model/Dashboard dashboard-data)))]- (events/publish-event! :event/dashboard-create {:object dash :user-id api/*current-user-id*})- (analytics/track-event! :snowplow/dashboard- {:event :dashboard-created- :dashboard-id (u/the-id dash)})- (-> dash- hydrate-dashboard-details- collection.root/hydrate-root-collection- (assoc :last-edit-info (revisions/edit-information-for-user @api/*current-user*))))))+ (let [dashboard-data {:name name+ :description description+ :parameters (or parameters [])+ :creator_id api/*current-user-id*+ :cache_ttl cache_ttl+ :collection_id collection_id+ :collection_position collection_position}+ dash (t2/with-transaction [_conn]+ ;; Adding a new dashboard at `collection_position` could cause other dashboards in this+ ;; collection to change position, check that and fix up if needed+ (api/maybe-reconcile-collection-position! dashboard-data)+ ;; Ok, now save the Dashboard+ (first (t2/insert-returning-instances! :model/Dashboard dashboard-data)))]+ (events/publish-event! :event/dashboard-create {:object dash :user-id api/*current-user-id*})+ (analytics/track-event! :snowplow/dashboard+ {:event :dashboard-created+ :dashboard-id (u/the-id dash)})+ (-> dash+ hydrate-dashboard-details+ collection.root/hydrate-root-collection+ (assoc :last-edit-info (revisions/edit-information-for-user @api/*current-user*)))));;; -------------------------------------------- Hiding Unreadable Cards ---------------------------------------------@@ -1249,11 +1248,10 @@[:id ms/PositiveInt][:param-key ms/NonBlankString]]constraint-param-key->value :- [:map-of string? any?]]- (lib-be/with-metadata-provider-cache- (let [dashboard (hydrate-dashboard-details (api/read-check :model/Dashboard id))]- ;; If a user can read the dashboard, then they can lookup filters. This also works with sandboxing.- (binding [qp.perms/*param-values-query* true]- (parameters.dashboard/param-values dashboard param-key constraint-param-key->value)))))+ (let [dashboard (api/read-check :model/Dashboard id)]+ ;; If a user can read the dashboard, then they can lookup filters. This also works with sandboxing.+ (binding [qp.perms/*param-values-query* true]+ (parameters.dashboard/param-values dashboard param-key constraint-param-key->value))));; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to;; use our API + we will need it when we make auto-TypeScript-signature generation happen@@ -1273,12 +1271,11 @@[:param-key ms/NonBlankString][:query ms/NonBlankString]]constraint-param-key->value :- [:map-of string? any?]]- (lib-be/with-metadata-provider-cache- (let [dashboard (api/read-check :model/Dashboard id)]- ;; If a user can read the dashboard, then they can lookup filters. This also works with sandboxing.- (binding [qp.perms/*param-values-query* true- chain-filter/*allow-implicit-uuid-field-remapping* false]- (parameters.dashboard/param-values dashboard param-key constraint-param-key->value query)))))+ (let [dashboard (api/read-check :model/Dashboard id)]+ ;; If a user can read the dashboard, then they can lookup filters. This also works with sandboxing.+ (binding [qp.perms/*param-values-query* true+ chain-filter/*allow-implicit-uuid-field-remapping* false]+ (parameters.dashboard/param-values dashboard param-key constraint-param-key->value query))));; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to;; use our API + we will need it when we make auto-TypeScript-signature generation happendiff --git a/src/metabase/embedding_rest/api/common.clj b/src/metabase/embedding_rest/api/common.cljindex 45c39037078..e2df18af088 100644--- a/src/metabase/embedding_rest/api/common.clj+++ b/src/metabase/embedding_rest/api/common.clj@@ -526,8 +526,9 @@dashboard-id (unsigned-token->dashboard-id unsigned-token)_ (when-not preview (check-embedding-enabled-for-dashboard dashboard-id))slug-token-params (embed/get-in-unsigned-token-or-throw unsigned-token [:params])+ dashboard (t2/select-one :model/Dashboard :id dashboard-id){parameters :parameters- published-embedding-params :embedding_params} (t2/select-one :model/Dashboard :id dashboard-id)+ published-embedding-params :embedding_params} dashboard;; when previewing an embed, embedding-params should come from the token,;; since a user may be changing them prior to publishing the Embed, which is what actually persists;; the settings to the Appdb.@@ -552,7 +553,7 @@(try(binding [api/*current-user-permissions-set* (atom #{"/"})api/*is-superuser?* true]- (parameters.dashboard/param-values (t2/select-one :model/Dashboard :id dashboard-id) searched-param-id merged-id-params prefix))+ (parameters.dashboard/param-values dashboard searched-param-id merged-id-params prefix))(catch Throwable e(throw (ex-info (.getMessage e){:merged-id-params merged-id-params}diff --git a/src/metabase/lib/metadata.cljc b/src/metabase/lib/metadata.cljcindex 0a31ea61368..f0363cd9106 100644--- a/src/metabase/lib/metadata.cljc+++ b/src/metabase/lib/metadata.cljc@@ -276,6 +276,17 @@(keep id->result)ids))))+(mu/defn metadatas :- [:maybe [:sequential :metabase.lib.metadata.protocols/metadata]]+ "Return a sequence of metadata objects matching `metadata-spec` (see+ `:metabase.lib.metadata.protocols/metadata-spec`). A thin wrapper over [[lib.metadata.protocols/metadatas]] that+ resolves the MetadataProvider from `metadata-providerable` for you, so callers don't need to reach for+ [[->metadata-provider]] or the protocol namespace directly.++ Like the underlying method, can be called for side-effects to warm the cache."+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable+ metadata-spec :- :metabase.lib.metadata.protocols/metadata-spec]+ (lib.metadata.protocols/metadatas (->metadata-provider metadata-providerable) metadata-spec))+(defn- missing-bulk-metadata-error [metadata-type id](ex-info (i18n/tru "Failed to fetch {0} {1}: either it does not exist, or it belongs to a different Database"(pr-str metadata-type)diff --git a/src/metabase/lib/walk/util.cljc b/src/metabase/lib/walk/util.cljcindex 32c5bbd6137..00c9d426f0a 100644--- a/src/metabase/lib/walk/util.cljc+++ b/src/metabase/lib/walk/util.cljc@@ -7,6 +7,7 @@[metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.schema.template-tag :as lib.schema.template-tag][metabase.lib.walk :as lib.walk][metabase.util.malli :as mu]@@ -258,26 +259,65 @@[:segment [:set ::lib.schema.id/segment]][:snippet [:set ::lib.schema.id/snippet]]])+(mr/def ::referenced-entity-ids.options+ [:map+ [:include-implicitly-joinable? {:optional true} :boolean]])++(mu/defn- implicitly-joinable-table-ids :- [:set ::lib.schema.id/table]+ "The ids of Tables reachable by one FK hop out of the columns of `source-table-ids` and the result metadata of+ `source-card-ids` -- i.e. the Tables a caller exposing implicitly-joinable columns would need on top of the directly+ referenced ones. Loads those columns/Fields and the FK-target Fields into `metadata-providerable`'s cache while+ discovering the ids, so that caller hits the cache instead of fetching one entity at a time.++ Note `implicitly-joined-field-ids` above means Fields *already* joined in a query; this is about Fields that are+ *joinable* (reachable via an FK) but not joined yet."+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable+ source-table-ids :- [:set ::lib.schema.id/table]+ source-card-ids :- [:set ::lib.schema.id/card]]+ (let [table-columns (when (seq source-table-ids)+ (lib.metadata/metadatas metadata-providerable {:lib/type :metadata/column, :table-ids source-table-ids}))+ cards (lib.metadata/bulk-metadata metadata-providerable :metadata/card source-card-ids)+ result-columns (mapcat :result-metadata cards)+ card-columns (lib.metadata/bulk-metadata metadata-providerable :metadata/column+ (into #{} (keep :id) result-columns))+ fk-fields (lib.metadata/bulk-metadata metadata-providerable :metadata/column+ (into #{} (keep :fk-target-field-id)+ (concat table-columns card-columns result-columns)))]+ (into #{} (keep :table-id) fk-fields)))+(mu/defn all-referenced-entity-ids :- ::referenced-entity-ids- "Return a map of all referenced entity IDs in `queries`."- [queries :- [:sequential ::lib.schema/query]]- (let [source-table-ids (into #{} (mapcat all-source-table-ids) queries)- source-card-ids (into #{} (mapcat all-source-card-ids) queries)- implicitly-joined-field-ids (into #{} (mapcat all-implicitly-joined-field-ids) queries)- template-tag-field-ids (into #{} (mapcat all-template-tag-field-ids) queries)- template-tag-table-ids (into #{} (mapcat all-template-tag-table-ids) queries)- template-tag-card-ids (into #{} (mapcat all-template-tag-card-ids) queries)- template-tag-snippet-ids (into #{} (mapcat all-template-tag-snippet-ids) queries)- metric-ids (into #{} (mapcat all-metric-ids) queries)- measure-ids (into #{} (mapcat all-measure-ids) queries)- segment-ids (into #{} (mapcat all-segment-ids) queries)- all-field-ids* (set/union implicitly-joined-field-ids template-tag-field-ids)- all-field-table-ids (when (seq queries)- (->> (lib.metadata/bulk-metadata (first queries) :metadata/column all-field-ids*)- (into #{} (keep :table-id))))]- {:table (set/union source-table-ids all-field-table-ids template-tag-table-ids)- :card (set/union source-card-ids template-tag-card-ids)- :metric metric-ids- :measure measure-ids- :segment segment-ids- :snippet template-tag-snippet-ids}))+ "Return a map of all referenced entity IDs in `queries`.++ With `:include-implicitly-joinable?` true, `:table` additionally includes the Tables reachable by one FK hop out of+ the source Tables' columns and the source Cards' result metadata (see [[implicitly-joinable-table-ids]]); discovering+ them warms those columns/Fields in `(first queries)`'s cache. Defaults to false."+ ([queries :- [:sequential ::lib.schema/query]]+ (all-referenced-entity-ids queries nil))++ ([queries :- [:sequential ::lib.schema/query]+ {:keys [include-implicitly-joinable?]} :- [:maybe ::referenced-entity-ids.options]]+ (let [source-table-ids (into #{} (mapcat all-source-table-ids) queries)+ source-card-ids (into #{} (mapcat all-source-card-ids) queries)+ implicitly-joined-field-ids (into #{} (mapcat all-implicitly-joined-field-ids) queries)+ template-tag-field-ids (into #{} (mapcat all-template-tag-field-ids) queries)+ template-tag-table-ids (into #{} (mapcat all-template-tag-table-ids) queries)+ template-tag-card-ids (into #{} (mapcat all-template-tag-card-ids) queries)+ template-tag-snippet-ids (into #{} (mapcat all-template-tag-snippet-ids) queries)+ metric-ids (into #{} (mapcat all-metric-ids) queries)+ measure-ids (into #{} (mapcat all-measure-ids) queries)+ segment-ids (into #{} (mapcat all-segment-ids) queries)+ all-card-ids (set/union source-card-ids template-tag-card-ids)+ field-ids (set/union implicitly-joined-field-ids template-tag-field-ids)+ all-field-table-ids (if (seq queries)+ (->> (lib.metadata/bulk-metadata (first queries) :metadata/column field-ids)+ (into #{} (keep :table-id)))+ #{})+ joinable-table-ids (if (and include-implicitly-joinable? (seq queries))+ (implicitly-joinable-table-ids (first queries) source-table-ids all-card-ids)+ #{})]+ {:table (set/union source-table-ids all-field-table-ids template-tag-table-ids joinable-table-ids)+ :card all-card-ids+ :metric metric-ids+ :measure measure-ids+ :segment segment-ids+ :snippet template-tag-snippet-ids})))diff --git a/src/metabase/lib_be/query.clj b/src/metabase/lib_be/query.cljindex 6d005300cdb..11755bc4536 100644--- a/src/metabase/lib_be/query.clj+++ b/src/metabase/lib_be/query.clj@@ -1,28 +1,20 @@(ns metabase.lib-be.query(:require[metabase.lib.metadata :as lib.metadata]- [metabase.lib.metadata.protocols :as lib.metadata.protocols]- [metabase.lib.metadata.util :as lib.metadata.util][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.walk.util :as lib.walk.util][metabase.util.malli :as mu]))-(mu/defn bulk-load-query-metadata!+(mu/defn bulk-load-query-metadata! :- :nil"Bulk-load all metadata referenced by `referenced-entity-ids` into `metadata-providerable`'s cache."[metadata-providerable :- ::lib.schema.metadata/metadata-providerable{:keys [table card metric measure segment snippet]} :- ::lib.walk.util/referenced-entity-ids]+ (lib.metadata/bulk-metadata metadata-providerable :metadata/table table)(when (seq table)- (lib.metadata/bulk-metadata metadata-providerable :metadata/table table)- (lib.metadata.protocols/metadatas (lib.metadata.util/->metadata-provider metadata-providerable)- {:lib/type :metadata/column, :table-ids (set table)}))- (when (seq card)- (lib.metadata/bulk-metadata metadata-providerable :metadata/card card))- (when (seq metric)- (lib.metadata/bulk-metadata metadata-providerable :metadata/metric metric))- (when (seq measure)- (lib.metadata/bulk-metadata metadata-providerable :metadata/measure measure))- (when (seq segment)- (lib.metadata/bulk-metadata metadata-providerable :metadata/segment segment))- (when (seq snippet)- (lib.metadata/bulk-metadata metadata-providerable :metadata/native-query-snippet snippet))+ (lib.metadata/metadatas metadata-providerable {:lib/type :metadata/column, :table-ids table}))+ (lib.metadata/bulk-metadata metadata-providerable :metadata/card card)+ (lib.metadata/bulk-metadata metadata-providerable :metadata/metric metric)+ (lib.metadata/bulk-metadata metadata-providerable :metadata/measure measure)+ (lib.metadata/bulk-metadata metadata-providerable :metadata/segment segment)+ (lib.metadata/bulk-metadata metadata-providerable :metadata/native-query-snippet snippet)nil)diff --git a/src/metabase/parameters/chain_filter.clj b/src/metabase/parameters/chain_filter.cljindex e2f9a9fa0c1..717510338bf 100644--- a/src/metabase/parameters/chain_filter.clj+++ b/src/metabase/parameters/chain_filter.clj@@ -72,6 +72,7 @@[metabase.lib.metadata :as lib.metadata][metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.types.isa :as lib.types.isa][metabase.lib.util :as lib.util][metabase.parameters.chain-filter.dedupe-joins :as dedupe]@@ -93,7 +94,6 @@[metabase.warehouse-schema.metadata-queries :as schema.metadata-queries][metabase.warehouse-schema.models.field :as field][metabase.warehouse-schema.models.field-values :as field-values]- [metabase.warehouses.models.database :as database][toucan2.core :as t2]));; so the hydration method for name_field is loaded@@ -121,38 +121,31 @@(defn- joined-table-alias [table-id](format "table_%d" table-id))-(def ^:private ^{:arglists '([field-id])} memoized-field-types-by-id- "Return field types by id. Cached for 10 minutes to avoid hitting the DB too much since this is unlike to change- often, if ever."- (memoize/ttl- ^{::memoize/args-fn (fn [[field-id]]- [(mdb/unique-identifier) field-id])}- (fn [field-id]- (t2/select-one [:model/Field :base_type :semantic_type] :id field-id))- :ttl/threshold (u/minutes->ms 10)))-(mu/defn- add-filter :- ::lib.schema/query- "Generate a single MBQL `:filter` clause for a Field and `value` (or multiple values, if `value` is a collection)."+ "Generate a single MBQL `:filter` clause for a Field and `value` (or multiple values, if `value` is a collection).+ `id->field` looks up prefetched Field metadata by id (see [[add-filters]])."[query :- ::lib.schema/querysource-table-id :- ::lib.schema.id/table+ id->field :- [:map-of ::lib.schema.id/field ::lib.schema.metadata/column]{:keys [field-id op value options]} :- ::constraint]- (let [field (let [this-field-table-id (field/field-id->table-id field-id)]- (cond-> (lib.metadata/field query field-id)- (not= this-field-table-id source-table-id)- (lib/with-join-alias (joined-table-alias this-field-table-id))))- filter-op (if (and (lib.types.isa/temporal? field)- (string? value))- (try- (params.dates/date-string->filter value field-id)- (catch Throwable e- (log/error e "Error creating filter for date string")- nil))- ;; we don't want to skip our value, even if its nil- (let [values (if (nil? value) [nil] (u/one-or-many value))]- {:lib/type :lib/external-op- :operator op- :options options- :args (cons field values)}))]+ (let [field (id->field field-id)+ field-table-id (:table-id field)+ field (cond-> field+ (not= field-table-id source-table-id)+ (lib/with-join-alias (joined-table-alias field-table-id)))+ filter-op (if (and (lib.types.isa/temporal? field)+ (string? value))+ (try+ (params.dates/date-string->filter value field-id)+ (catch Throwable e+ (log/error e "Error creating filter for date string")+ nil))+ ;; we don't want to skip our value, even if its nil+ (let [values (if (nil? value) [nil] (u/one-or-many value))]+ {:lib/type :lib/external-op+ :operator op+ :options options+ :args (cons field values)}))](when filter-op(log/tracef "Adding filter clause: %s" (pr-str filter-op)))(cond-> query@@ -179,27 +172,29 @@source-table-id :- ::lib.schema.id/tablejoined-table-ids :- [:set ::lib.schema.id/table]constraints :- [:maybe ::constraints]]- (reduce- (fn [query {:keys [field-id] :as constraint}]- ;; only add a where clause for the Field if it's part of the source Table or if we're actually joining against- ;; the Table it belongs to. This Field might not even be part of the same Database in which case we can ignore- ;; it.- (let [field-table-id (field/field-id->table-id field-id)]- (if (or (= field-table-id source-table-id)- (contains? joined-table-ids field-table-id))- (do- (log/tracef "Added filter clause for %s %s with constraint %s"- (name-for-logging :model/Table field-table-id)- (name-for-logging :model/Field field-id)- (pr-str constraint))- (add-filter query source-table-id constraint))- (do- (log/tracef "Not adding filter clause for %s %s because we did not join against its Table"- (name-for-logging :model/Table field-table-id)- (name-for-logging :model/Field field-id))- query))))- query- constraints))+ (let [id->field (u/index-by :id (lib.metadata/bulk-metadata query :metadata/column+ (into #{} (map :field-id) constraints)))]+ (reduce+ (fn [query {:keys [field-id] :as constraint}]+ ;; only add a where clause for the Field if it's part of the source Table or if we're actually joining against+ ;; the Table it belongs to. This Field might not even be part of the same Database in which case we can ignore+ ;; it.+ (let [field-table-id (:table-id (id->field field-id))]+ (if (or (= field-table-id source-table-id)+ (contains? joined-table-ids field-table-id))+ (do+ (log/tracef "Added filter clause for %s %s with constraint %s"+ (name-for-logging :model/Table field-table-id)+ (name-for-logging :model/Field field-id)+ (pr-str constraint))+ (add-filter query source-table-id id->field constraint))+ (do+ (log/tracef "Not adding filter clause for %s %s because we did not join against its Table"+ (name-for-logging :model/Table field-table-id)+ (name-for-logging :model/Field field-id))+ query))))+ query+ constraints)))(def ^:private find-joins-cache-duration-ms"Amount of time to cache results of `find-joins`. Since FK relationships in Tables are unlikely to change very@@ -339,13 +334,12 @@(f database-id source-table-id other-table-id enable-reverse-joins?)))(meta f))))-(def ^:private ^{:arglists '([source-table field-ids other-table-ids enable-reverse-joins?])} find-all-joins*+(def ^:private ^{:arglists '([db-id source-table field-ids other-table-ids enable-reverse-joins?])} find-all-joins*(memoize/ttl- ^{::memoize/args-fn (fn [[source-table-id field-ids other-table-ids enable-reverse-joins?]]+ ^{::memoize/args-fn (fn [[_db-id source-table-id field-ids other-table-ids enable-reverse-joins?]][(mdb/unique-identifier) source-table-id field-ids other-table-ids enable-reverse-joins?])}- (fn [source-table-id field-ids other-table-ids enable-reverse-joins?]- (let [db-id (database/table-id->database-id source-table-id)- all-joins (mapcat #(find-joins db-id source-table-id % enable-reverse-joins?)+ (fn [db-id source-table-id field-ids other-table-ids enable-reverse-joins?]+ (let [all-joins (mapcat #(find-joins db-id source-table-id % enable-reverse-joins?)other-table-ids)](when (seq all-joins)(log/tracef "Deduplicating for source %s; Tables to keep: %s\n%s"@@ -360,11 +354,15 @@(mu/defn- find-all-joins"Find the complete set of joins we need to do for `source-table-id` to join against Fields in `field-ids`."- [source-table-id :- ::lib.schema.id/table- field-ids :- [:set ::lib.schema.id/field]]- (when-let [other-table-ids (not-empty (disj (set (map field/field-id->table-id (set field-ids)))+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable+ database-id :- ::lib.schema.id/database+ source-table-id :- ::lib.schema.id/table+ field-ids :- [:set ::lib.schema.id/field]]+ (when-let [other-table-ids (not-empty (disj (into #{}+ (map :table-id)+ (lib.metadata/bulk-metadata metadata-providerable :metadata/column field-ids))source-table-id))]- (find-all-joins* source-table-id field-ids other-table-ids *enable-reverse-joins*)))+ (find-all-joins* database-id source-table-id field-ids other-table-ids *enable-reverse-joins*)))(mu/defn- add-joins :- ::lib.schema/query"Add joins to the MBQL `query` we're generating. The Field for which we are returning values is the \"source Field\",@@ -385,24 +383,30 @@[query :- ::lib.schema/querysource-table-id :- ::lib.schema.id/tablejoins]- (reduce- (fn [query {{lhs-table-id :table, lhs-field-id :field} :lhs, {rhs-table-id :table, rhs-field-id :field} :rhs}]- (let [lhs-field (lib.metadata/field query lhs-field-id)- rhs-field (lib.metadata/field query rhs-field-id)- rhs-table (lib.metadata/table query rhs-table-id)- join (-> (lib/join-clause rhs-table)- (lib/with-join-alias (joined-table-alias rhs-table-id))- (lib/with-join-conditions [(lib/=- (cond-> lhs-field- (not= lhs-table-id source-table-id)- (lib/with-join-alias (joined-table-alias lhs-table-id)))- (-> rhs-field- (lib/with-join-alias (joined-table-alias rhs-table-id))))]))]- (log/tracef "Adding join against %s\n%s"- (name-for-logging :model/Table rhs-table-id) (u/cprint-to-str join))- (lib/join query join)))- query- joins))+ (let [id->field (u/index-by :id (lib.metadata/bulk-metadata query :metadata/column+ (into #{} (mapcat (juxt #(get-in % [:lhs :field])+ #(get-in % [:rhs :field])))+ joins)))+ id->table (u/index-by :id (lib.metadata/bulk-metadata query :metadata/table+ (into #{} (map #(get-in % [:rhs :table])) joins)))]+ (reduce+ (fn [query {{lhs-table-id :table, lhs-field-id :field} :lhs, {rhs-table-id :table, rhs-field-id :field} :rhs}]+ (let [lhs-field (id->field lhs-field-id)+ rhs-field (id->field rhs-field-id)+ rhs-table (id->table rhs-table-id)+ join (-> (lib/join-clause rhs-table)+ (lib/with-join-alias (joined-table-alias rhs-table-id))+ (lib/with-join-conditions [(lib/=+ (cond-> lhs-field+ (not= lhs-table-id source-table-id)+ (lib/with-join-alias (joined-table-alias lhs-table-id)))+ (-> rhs-field+ (lib/with-join-alias (joined-table-alias rhs-table-id))))]))]+ (log/tracef "Adding join against %s\n%s"+ (name-for-logging :model/Table rhs-table-id) (u/cprint-to-str join))+ (lib/join query join)))+ query+ joins)))(mu/defn- tighten-join-projections :- ::lib.schema/query"Narrow each join's inner-stage `:fields` to exactly the field-ids `query` references whose `:table-id` matches the@@ -445,17 +449,19 @@constraints :- [:maybe ::constraints]{:keys [original-field-id limit]} :- [:maybe ::options]](log/tracef "Chain filter %s with constraints %s" (name-for-logging :model/Field field-id) (u/cprint-to-str constraints))+ ;; `field-id->database-id` is the one place we still bootstrap from a raw field-id: we need the Database before we+ ;; can build a (per-Database) metadata provider. Every other table-id/db-id below comes from `mp`.(let [database-id (field/field-id->database-id field-id)mp (lib-be/application-database-metadata-provider database-id)- field-table-id (field/field-id->table-id field-id)- original-table-id (when original-field-id (field/field-id->table-id original-field-id))+ field-table-id (:table-id (lib.metadata/field mp field-id))+ original-table-id (when original-field-id (:table-id (lib.metadata/field mp original-field-id)));; When the original (FK) field lives on a different table, reverse the join direction:;; make the original field's table the source and join the label table. The original table;; is typically the large fact table; putting it on the right side of a JOIN can OOM on;; engines like ClickHouse that materialize the right side in memory.reversed? (and original-table-id (not= original-table-id field-table-id))source-table-id (if reversed? original-table-id field-table-id)- joins (find-all-joins source-table-id+ joins (find-all-joins mp database-id source-table-id(cond-> (set (map :field-id constraints))reversed? (conj field-id)(not reversed?) (cond-> original-field-id (conj original-field-id))))@@ -813,12 +819,14 @@(assert (even? (count options)))(let [{:as options} optionsv->human-readable (delay (schema.metadata-queries/human-readable-remapping-map field-id))- the-remapped-field-id (delay (let [{:keys [base_type effective_type]} (memoized-field-types-by-id field-id)]+ the-remapped-field-id (delay (let [metadata-provider (lib-be/application-database-metadata-provider+ (field/field-id->database-id field-id))+ {:keys [base-type effective-type]} (lib.metadata/field metadata-provider field-id)](binding [*allow-implicit-uuid-field-remapping*;; For the details on following condition see the dynamic var's;; docstring.(or *allow-implicit-uuid-field-remapping*- (not (isa? (or effective_type base_type) :type/UUID)))]+ (not (isa? (or effective-type base-type) :type/UUID)))](remapped-field-id field-id))))](cond(str/blank? query-string)diff --git a/src/metabase/parameters/custom_values.clj b/src/metabase/parameters/custom_values.cljindex 82ef6a72773..7c35b873636 100644--- a/src/metabase/parameters/custom_values.clj+++ b/src/metabase/parameters/custom_values.clj@@ -8,6 +8,7 @@(:require[clojure.string :as str][medley.core :as m]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.schema :as lib.schema]@@ -74,48 +75,55 @@;; remapped label for a single selected value)[:exact-value {:optional true} :any]])-(defn- card-query- [{query :dataset_query, :keys [id], :as _card}]+(mu/defn- card-query :- [:maybe ::lib.schema/query]+ "Build the lib query for the value-source Card identified by `card-id`. `query` is the Card's own `:dataset_query`,+ which is metadata-providerable, so we resolve the Card as a Lib `:metadata/card` through it. Bulk-loads the metadata+ it references up front so the column resolution that follows ([[lib/visible-columns]] etc.) hits the provider cache+ instead of fetching one entity at a time."+ [card-id :- ::lib.schema.id/card+ query :- [:maybe ::lib.schema/query]](when (seq query)- (lib/query query (lib.metadata/card query id))))+ (let [value-source-query (lib/query query (lib.metadata/card query card-id))]+ (lib-be/bulk-load-query-metadata! query+ (lib/all-referenced-entity-ids [value-source-query]+ {:include-implicitly-joinable? true}))+ value-source-query)))(mu/defn- values-from-card-query :- [:maybe ::lib.schema/query]- [{:keys [id], :as card} :- [:and- :metabase.queries.schema/card- [:map- [:id ::lib.schema.id/card]]]- field-ref :- [:or :mbql.clause/field :mbql.clause/expression]+ [query :- [:maybe ::lib.schema/query]+ field-ref :- [:or :mbql.clause/field :mbql.clause/expression]{:keys [query-string label-field exact-value] :as _opts} :- [:maybe ::values-from-card-query.options]]- (when-let [query (card-query card)]- (when-let [visible-columns (or (not-empty (lib/visible-columns query))- (log/warnf "Cannot get values from Card %d: Card query has no visible columns"- id))]- (when-let [value-column (or (lib/find-matching-column query -1 field-ref visible-columns)- (log/warnf "Cannot get values from Card %d: failed to find column for ref %s\nFound: %s"- id- (pr-str field-ref)- (pr-str (map (some-fn :lib/source-column-alias :name) visible-columns))))]- (let [label-column (when label-field- (or (lib/find-matching-column query -1 label-field visible-columns)- (log/warnf "Cannot get labels from Card %d: failed to find column for ref %s"- id- (pr-str label-field))))- search-column (or label-column value-column)- value-textual? (lib.types.isa/string? value-column)- search-textual? (lib.types.isa/string? search-column)- nonempty ((if value-textual? lib/not-empty lib/not-null) value-column)- query-filter (cond- (some? exact-value) (lib/= value-column exact-value)- query-string (if search-textual?- (lib/ignore-case (lib/contains search-column query-string))- (lib/= search-column query-string)))]- (-> query- (lib/limit *max-rows*)- (lib/filter nonempty)- (cond-> query-filter (lib/filter query-filter))- (lib/breakout value-column)- ;; add the label as a second breakout so each row is a [value label] pair- (cond-> label-column (lib/breakout label-column))))))))+ (when query+ (let [card-id (lib/primary-source-card-id query)]+ (when-let [visible-columns (or (not-empty (lib/visible-columns query))+ (log/warnf "Cannot get values from Card %d: Card query has no visible columns"+ card-id))]+ (when-let [value-column (or (lib/find-matching-column query -1 field-ref visible-columns)+ (log/warnf "Cannot get values from Card %d: failed to find column for ref %s\nFound: %s"+ card-id+ (pr-str field-ref)+ (pr-str (map (some-fn :lib/source-column-alias :name) visible-columns))))]+ (let [label-column (when label-field+ (or (lib/find-matching-column query -1 label-field visible-columns)+ (log/warnf "Cannot get labels from Card %d: failed to find column for ref %s"+ card-id+ (pr-str label-field))))+ search-column (or label-column value-column)+ value-textual? (lib.types.isa/string? value-column)+ search-textual? (lib.types.isa/string? search-column)+ nonempty ((if value-textual? lib/not-empty lib/not-null) value-column)+ query-filter (cond+ (some? exact-value) (lib/= value-column exact-value)+ query-string (if search-textual?+ (lib/ignore-case (lib/contains search-column query-string))+ (lib/= search-column query-string)))]+ (-> query+ (lib/limit *max-rows*)+ (lib/filter nonempty)+ (cond-> query-filter (lib/filter query-filter))+ (lib/breakout value-column)+ ;; add the label as a second breakout so each row is a [value label] pair+ (cond-> label-column (lib/breakout label-column)))))))))(defn- result->rows"Extract rows from a QP result, dropping values for any display columns the QP injected for@@ -130,6 +138,19 @@(let [keep-idxs (into [] (keep-indexed (fn [i c] (when-not (drop-names (:name c)) i))) cols)](perf/mapv (fn [row] (perf/mapv #(nth row %) keep-idxs)) rows)))))+(mu/defn- values-from-card* :- ms/FieldValuesResult+ "Core of [[values-from-card]], working off a prebuilt value-source `query`."+ [query :- [:maybe ::lib.schema/query]+ field-ref :- [:or :mbql.clause/field :mbql.clause/expression]+ opts :- [:maybe ::values-from-card-query.options]]+ (let [mbql-query (values-from-card-query query field-ref opts)+ result (some-> mbql-query qp/process-query)+ values (some-> result result->rows)]+ {:values (or values [])+ ;; If the row_count returned = the limit we specified, then it's probably has more than that.+ ;; If the query has its own limit smaller than *max-rows*, then there's no more values.+ :has_more_values (= (:row_count result) *max-rows*)}))+(mu/defn values-from-card"Get distinct values of a field from a card.@@ -150,32 +171,25 @@([card :- :metabase.queries.schema/cardfield-ref :- [:or :mbql.clause/field :mbql.clause/expression]opts :- [:maybe ::values-from-card-query.options]]- (let [mbql-query (values-from-card-query card field-ref opts)- result (some-> mbql-query qp/process-query)- values (some-> result result->rows)]- {:values (or values [])- ;; If the row_count returned = the limit we specified, then it's probably has more than that.- ;; If the query has its own limit smaller than *max-rows*, then there's no more values.- :has_more_values (= (:row_count result) *max-rows*)})))--(mu/defn card-values- "Given a param and query returns the values."- [{config :values_source_config :as _param} :- ::parameters.schema/parameter- query-string :- [:maybe ms/NonBlankString]]- (let [card-id (:card_id config)- card (t2/select-one :model/Card :id card-id)]- (values-from-card card- (lib/->mbql5 (:value_field config))- (cond-> {:query-string query-string}- (:label_field config) (assoc :label-field (lib/->mbql5 (:label_field config)))))))+ (values-from-card* (card-query (:id card) (not-empty (:dataset_query card))) field-ref opts)))(defn- can-get-card-values?- [card value-field]+ "Whether the prebuilt value-source `query` exposes the `value-field` column."+ [query value-field](boolean- (and (not (:archived card))- (when-let [query (card-query card)]- (some? (lib/find-matching-column query -1 (lib/->mbql5 value-field)- (lib/visible-columns query)))))))+ (and query+ (some? (lib/find-matching-column query -1 (lib/->mbql5 value-field)+ (lib/visible-columns query))))))++(mu/defn- card-values :- ms/FieldValuesResult+ "Given a prebuilt value-source `query`, the parameter's `config`, and a `query-string`, return the values."+ [query :- [:maybe ::lib.schema/query]+ config :- ::parameters.schema/values-source-config+ query-string :- [:maybe ms/NonBlankString]]+ (values-from-card* query+ (lib/->mbql5 (:value_field config))+ (cond-> {:query-string query-string}+ (:label_field config) (assoc :label-field (lib/->mbql5 (:label_field config))))));;; --------------------------------------------- Putting it together ----------------------------------------------@@ -190,12 +204,15 @@default-case-thunk :- [:=> [:cat :any] ms/FieldValuesResult]](case (:values_source_type parameter):static-list (static-list-values parameter query-string)- :card (let [card (t2/select-one :model/Card :id (get-in parameter [:values_source_config :card_id]))]+ :card (let [config (:values_source_config parameter)+ card (t2/select-one :model/Card :id (:card_id config))](when-not (mi/can-read? card)(throw (ex-info "You don't have permissions to do that." {:status-code 403})))- (if (can-get-card-values? card (get-in parameter [:values_source_config :value_field]))- (card-values parameter query-string)- (default-case-thunk)))+ (or (when-not (:archived card)+ (when-let [query (card-query (:id card) (not-empty (:dataset_query card)))]+ (when (can-get-card-values? query (:value_field config))+ (card-values query config query-string))))+ (default-case-thunk)))nil (default-case-thunk)(throw (ex-info (tru "Invalid parameter source {0}" (:values_source_type parameter)){:status-code 400@@ -239,13 +256,13 @@[{config :values_source_config :as _param} value](when-let [label-field (:label_field config)](when-let [card (t2/select-one :model/Card :id (:card_id config))]- (when (and (not (:archived card))- (mi/can-read? card)- (can-get-card-values? card (:value_field config)))- (first (:values (values-from-card card- (lib/->mbql5 (:value_field config))- {:exact-value value- :label-field (lib/->mbql5 label-field)})))))))+ (when (and (not (:archived card)) (mi/can-read? card))+ (when-let [query (card-query (:id card) (not-empty (:dataset_query card)))]+ (when (can-get-card-values? query (:value_field config))+ (first (:values (values-from-card* query+ (lib/->mbql5 (:value_field config))+ {:exact-value value+ :label-field (lib/->mbql5 label-field)})))))))))(mu/defn parameter-remapped-value"Fetch the remapped value for the given `value` of parameter `param` with default values provided bydiff --git a/src/metabase/parameters/params.clj b/src/metabase/parameters/params.cljindex a7438e671fe..35bfb49aeb2 100644--- a/src/metabase/parameters/params.clj+++ b/src/metabase/parameters/params.clj@@ -17,7 +17,9 @@[metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.schema.parameter :as lib.schema.parameter][metabase.lib.schema.template-tag :as lib.schema.template-tag][metabase.parameters.schema :as parameters.schema]@@ -172,17 +174,34 @@;;; | DASHBOARD-SPECIFIC |;;; +----------------------------------------------------------------------------------------------------------------+-(defn- filterable-columns-for-query- "Get filterable columns for query."- [database-id card stage-number]- (let [metadata-provider (lib-be/application-database-metadata-provider database-id)- ;; Regular questions are used directly. If a model or metric has been used directly in this card, wrap it into- ;; a query against that model or metric.- query (lib/query metadata-provider (if (= :question (:type card))- (:dataset_query card)- (lib.metadata/card metadata-provider (:id card))))- ;; for backward compatibility, append a filter stage only with explicit stage numbers- query (cond-> query (>= stage-number 0) lib/ensure-filter-stage)]+(mr/def ::param-dashcard-info+ "Internal bookkeeping for one parameter `mapping` on a hydrated `dashcard`: the dashcard, the mapping, and the+ parameter target's Field ID when the target is already field-id-based (nil when it must be resolved from filterable+ columns)."+ [:map+ [:dashcard :map]+ [:param-mapping ::parameters.schema/parameter-mapping]+ [:param-target-field-id [:maybe ::lib.schema.id/field]]])++(mu/defn- card->filterable-columns-query :- [:maybe ::lib.schema/query]+ "Build the lib query whose filterable columns we want for `card` at `stage-number`, or nil when `card` has no query."+ [card :- :metabase.queries.schema/card+ stage-number :- :int]+ (when (and (seq (:dataset_query card)) (pos-int? (:database_id card)))+ (let [metadata-provider (lib-be/application-database-metadata-provider (:database_id card))+ ;; Regular questions are used directly. If a model or metric has been used directly in this card, wrap it+ ;; into a query against that model or metric.+ query (lib/query metadata-provider (if (= :question (:type card))+ (:dataset_query card)+ (lib.metadata/card metadata-provider (:id card))))]+ ;; for backward compatibility, append a filter stage only with explicit stage numbers+ (cond-> query (>= stage-number 0) lib/ensure-filter-stage))))++(mu/defn- filterable-columns-for-query :- [:maybe [:sequential ::lib.schema.metadata/column]]+ "Get the filterable columns of `card`'s query at `stage-number`."+ [card :- :metabase.queries.schema/card+ stage-number :- :int]+ (when-let [query (card->filterable-columns-query card stage-number)](when (and (>= stage-number -1) (< stage-number (lib/stage-count query)))(lib/filterable-columns query stage-number))))@@ -198,7 +217,7 @@(seq dataset-query)(pos-int? database-id))(assoc-in [:card-id->filterable-columns card-id stage-number]- (or (filterable-columns-for-query database-id card stage-number) []))))+ (or (filterable-columns-for-query card stage-number) []))))(defn- field-id-from-dashcards-filterable-columns"Update the `ctx` with `field-id`. This function is supposed to be used on params where target is a name field, in@@ -234,6 +253,21 @@{:card-id->filterable-columns {}:param-id->field-ids {}})+(mu/defn- param-dashcard-info->card :- [:maybe :metabase.queries.schema/card]+ "The Card on `param-dashcard-info`'s dashcard that its parameter mapping targets: matched by the mapping's `:card_id`+ against the dashcard's main Card and series Cards, falling back to the main Card when the mapping has no `:card_id`."+ [{:keys [param-mapping dashcard] :as _param-dashcard-info} :- ::param-dashcard-info]+ (if-let [card-id (:card_id param-mapping)]+ (m/find-first #(= (:id %) card-id)+ (cons (:card dashcard) (:series dashcard)))+ (:card dashcard)))++(mu/defn- param-dashcard-info->stage-number :- :int+ "The query stage of `param-dashcard-info`'s parameter target whose filterable columns we need, or -1 when the target+ doesn't pin a stage."+ [{:keys [param-mapping] :as _param-dashcard-info} :- ::param-dashcard-info]+ (get-in param-mapping [:target 2 :stage-number] -1))+(mu/defn- field-id-into-context-rf"Reducing function that generates _field id_ corresponding to `:parameter` of `param-dashcard-info` if possible,and returns new _context_ (`ctx`) with the _field id_ added.@@ -255,14 +289,7 @@merge (:card-id->filterable-columns ctx)))(:param-id->field-ids ctx))([ctx {:keys [param-mapping param-target-field-id] :as param-dashcard-info}]- (let [card-id (:card_id param-mapping)- card (if card-id- (m/find-first #(= (:id %) card-id)- (cons (get-in param-dashcard-info [:dashcard :card])- (get-in param-dashcard-info [:dashcard :series])))- (get-in param-dashcard-info [:dashcard :card]))- param-id (:parameter_id param-mapping)- stage-number (get-in param-mapping [:target 2 :stage-number] -1)]+ (let [param-id (:parameter_id param-mapping)];; Get the field id from the field-clause if it contains it. This is the common case;; for mbql queries.(if param-target-field-id@@ -270,9 +297,11 @@;; In case the card doesn't have the same result_metadata columns as filterable columns (a question that;; aggregates a native query model with a field that was mapped to a db field), we need to load metadata in;; [[ensure-filterable-columns-for-card]] to find the originating field. (#42829)- (-> ctx- (ensure-filterable-columns-for-card card stage-number)- (field-id-from-dashcards-filterable-columns param-dashcard-info stage-number))))))+ (let [card (param-dashcard-info->card param-dashcard-info)+ stage-number (param-dashcard-info->stage-number param-dashcard-info)]+ (-> ctx+ (ensure-filterable-columns-for-card card stage-number)+ (field-id-from-dashcards-filterable-columns param-dashcard-info stage-number)))))))(defn- find-card-for-mapping"Find the card that a parameter mapping refers to. Looks up the card by `:card_id` from both@@ -287,19 +316,48 @@(string? (:type card)) (update :type keyword)(seq (:dataset_query card)) (update :dataset_query lib-be/normalize-query))))+(mu/defn- mapping->param-dashcard-info :- ::param-dashcard-info+ "Build the `param-dashcard-info` for a parameter `mapping` on `dashcard`, resolving `:param-target-field-id` when the+ target is already field-id-based."+ [dashcard :- :map+ mapping :- ::parameters.schema/parameter-mapping]+ (let [card (find-card-for-mapping dashcard mapping)]+ {:dashcard dashcard+ :param-mapping mapping+ :param-target-field-id (when (:target mapping)+ (param-target->field-id (:target mapping) card))}))++(mu/defn- preload-param-filterable-columns! :- :nil+ "Bulk-load up front the metadata that the per-Card [[filterable-columns-for-query]] computations (run lazily by+ [[field-id-into-context-rf]]) will need, so those computations hit the metadata-provider cache instead of fetching+ objects one at a time. Only Cards whose parameter target is not field-id-based need filterable columns; we build+ their queries here (the same way [[ensure-filterable-columns-for-card]] does, via [[card->filterable-columns-query]]),+ gather the entities they reference, and bulk-load them per Database. Requires a metadata-provider cache scope (asserted+ by [[card->filterable-columns-query]]), since there'd otherwise be no shared cache to warm."+ [param-dashcard-infos :- [:sequential ::param-dashcard-info]]+ (doseq [[database-id infos] (->> param-dashcard-infos+ ;; field-id-based targets are resolved without filterable columns, so nothing to preload+ (remove :param-target-field-id)+ (group-by (comp :database_id param-dashcard-info->card))+ (filter (fn [[database-id _infos]] (pos-int? database-id))))]+ (when-let [queries (not-empty (into [] (keep (fn [info]+ (card->filterable-columns-query+ (param-dashcard-info->card info)+ (param-dashcard-info->stage-number info))))+ infos))]+ (lib-be/bulk-load-query-metadata! (lib-be/application-database-metadata-provider database-id)+ (lib/all-referenced-entity-ids queries {:include-implicitly-joinable? true})))))+(mu/defn dashcards->param-id->field-ids* :- [:map-of ::lib.schema.parameter/id [:set ::lib.schema.id/field]]"Return map of parameter ids to mapped field ids."[dashcards]- (letfn [(dashcard->param-dashcard-info [dashcard]- (for [mapping (:parameter_mappings dashcard)]- (let [card (find-card-for-mapping dashcard mapping)]- {:dashcard dashcard- :param-mapping mapping- :param-target-field-id (when (:target mapping)- (param-target->field-id (:target mapping) card))})))]- (transduce (mapcat dashcard->param-dashcard-info)- field-id-into-context-rf- dashcards)))+ (let [param-dashcard-infos (into []+ (mapcat (fn [dashcard]+ (for [mapping (:parameter_mappings dashcard)]+ (mapping->param-dashcard-info dashcard mapping))))+ dashcards)]+ (preload-param-filterable-columns! param-dashcard-infos)+ (transduce identity field-id-into-context-rf param-dashcard-infos)))(declare card->template-tag-id->field-ids)@@ -322,16 +380,11 @@(mu/defn dashboard-param->field-ids :- [:set ::lib.schema.id/field]"Return field ids mapped to the parameter. `dashcard` and `card` must be present for each mapping."[{:keys [mappings]} :- ::parameters.schema/parameter]- (let [param-id->field-ids (transduce (map (fn [mapping]- (let [card (find-card-for-mapping (:dashcard mapping) mapping)]- {:dashcard (:dashcard mapping)- :param-mapping mapping- :param-target-field-id (param-target->field-id- (:target mapping)- card)})))- field-id-into-context-rf- mappings)]- (into #{} cat (vals param-id->field-ids))))+ (let [param-dashcard-infos (mapv (fn [mapping]+ (mapping->param-dashcard-info (:dashcard mapping) mapping))+ mappings)]+ (preload-param-filterable-columns! param-dashcard-infos)+ (into #{} cat (vals (transduce identity field-id-into-context-rf param-dashcard-infos)))))(defn get-linked-field-ids"Retrieve a map relating parameter ids to field ids."diff --git a/src/metabase/public_sharing_rest/api.clj b/src/metabase/public_sharing_rest/api.cljindex f9ebf8ef133..1ccb78d69bd 100644--- a/src/metabase/public_sharing_rest/api.clj+++ b/src/metabase/public_sharing_rest/api.clj@@ -263,6 +263,14 @@(defn- dashboard-with-uuid [uuid] (public-dashboard :public_uuid uuid))+(defn- dashboard-with-uuid-for-param-values+ "Light Dashboard fetch for the public param-value endpoints. Unlike [[dashboard-with-uuid]]/[[public-dashboard]] it+ skips the full hydration (dashcards/cards/series/tabs/param_fields/query-average-durations) -- `param-values` and+ `dashboard-param-remapped-value` only need `:resolved-params`, which they hydrate themselves. Matches how the embed+ param-value endpoints fetch the Dashboard."+ [uuid]+ (api/check-404 (t2/select-one :model/Dashboard :public_uuid uuid, :archived false)))+;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to;; use our API + we will need it when we make auto-TypeScript-signature generation happen;;@@ -532,7 +540,7 @@[:param-key ms/NonBlankString]]constraint-param-key->value :- [:map-of string? any?]](public-sharing.validation/check-public-sharing-enabled)- (let [dashboard (dashboard-with-uuid uuid)]+ (let [dashboard (dashboard-with-uuid-for-param-values uuid)](request/as-admin(binding [qp.perms/*param-values-query* true](parameters.dashboard/param-values dashboard param-key constraint-param-key->value)))))@@ -549,7 +557,7 @@[:query ms/NonBlankString]]constraint-param-key->value](public-sharing.validation/check-public-sharing-enabled)- (let [dashboard (dashboard-with-uuid uuid)]+ (let [dashboard (dashboard-with-uuid-for-param-values uuid)](request/as-admin(binding [qp.perms/*param-values-query* true](parameters.dashboard/param-values dashboard param-key constraint-param-key->value query)))))@@ -565,7 +573,7 @@[:param-key ms/NonBlankString]]{:keys [value]} :- [:map [:value :any]]](public-sharing.validation/check-public-sharing-enabled)- (let [dashboard (dashboard-with-uuid uuid)]+ (let [dashboard (dashboard-with-uuid-for-param-values uuid)](request/as-admin(binding [qp.perms/*param-values-query* true](parameters.dashboard/dashboard-param-remapped-value dashboard param-key (codec/url-decode value))))))diff --git a/src/metabase/queries_rest/api/card.clj b/src/metabase/queries_rest/api/card.cljindex edda84f8d54..160eb4801d7 100644--- a/src/metabase/queries_rest/api/card.clj+++ b/src/metabase/queries_rest/api/card.clj@@ -370,12 +370,18 @@(def ^:private supported-series-display-type (set (keys (methods series-are-compatible?))))-(defn- fetch-compatible-series*+(mu/defn- fetch-compatible-series* :- [:sequential :map]"Implementation of `fetch-compatible-series`.Provide `page-size` to limit the number of cards returned, it does not guaranteed to return exactly `page-size` cards.Use `fetch-compatible-series` for that."- [card database-id->metadata-provider {:keys [query last-cursor page-size exclude-ids] :as _options}]+ [card :- :map+ {:keys [query last-cursor page-size exclude-ids] :as _options}+ :- [:map {:closed true}+ [:query {:optional true} [:maybe ms/NonBlankString]]+ [:last-cursor {:optional true} [:maybe ms/PositiveInt]]+ [:page-size {:optional true} [:maybe ms/PositiveInt]]+ [:exclude-ids {:optional true} [:maybe [:sequential ms/PositiveInt]]]]](let [matching-cards (t2/select :model/Card:archived false:display [:in supported-series-display-type]@@ -396,15 +402,6 @@;; this is just a heuristic, but it should be good enoughpage-size(assoc :limit (+ 10 page-size))))- database-ids (set (keys database-id->metadata-provider))- database-id->metadata-provider (->> matching-cards- (filter #(or (nil? (get-in % [:visualization_settings :graph.metrics]))- (nil? (get-in % [:visualization_settings :graph.dimensions]))))- (keep :database_id)- (set)- (remove #(contains? database-ids %))- (into database-id->metadata-provider- (map (juxt identity lib-be/application-database-metadata-provider))))compatible-cards (->> matching-cards(filter mi/can-read?)(filter #(or@@ -414,8 +411,8 @@(= (:query_type %) :native)(series-are-compatible? card %))))](if page-size- [database-id->metadata-provider (take page-size compatible-cards)]- [database-id->metadata-provider compatible-cards])))+ (take page-size compatible-cards)+ compatible-cards)))(defn- fetch-compatible-series"Fetch a list of compatible series for `card`.@@ -426,14 +423,10 @@- last-cursor: the id of the last card from the previous page- page-size: is nullable, it'll try to fetches exactly `page-size` cards if there are enough cards."([card options]- (fetch-compatible-series- card- options- {(:database_id card) (lib-be/application-database-metadata-provider (:database_id card))}- []))-- ([card {:keys [page-size] :as options} database-id->metadata-provider current-cards]- (let [[database-id->metadata-provider cards] (fetch-compatible-series* card database-id->metadata-provider options)+ (fetch-compatible-series card options []))++ ([card {:keys [page-size] :as options} current-cards]+ (let [cards (fetch-compatible-series* card options)new-cards (concat current-cards cards)];; if the total card fetches is less than page-size and there are still more, continue fetching(if (and (some? page-size)@@ -443,7 +436,6 @@(merge options{:page-size (- page-size (count cards)):last-cursor (:id (last cards))})- database-id->metadata-providernew-cards)new-cards))))@@ -784,9 +776,8 @@"Get all of the required query metadata for a card."[{:keys [id]} :- [:map[:id [:or ms/PositiveInt ms/NanoIdString]]]]- (lib-be/with-metadata-provider-cache- (let [resolved-id (eid-translation/->id-or-404 :card id)]- (queries/batch-fetch-card-metadata [(get-card resolved-id)]))))+ (let [resolved-id (eid-translation/->id-or-404 :card id)]+ (queries/batch-fetch-card-metadata [(get-card resolved-id)])));;; ------------------------------------------------- Deleting Cards -------------------------------------------------diff --git a/src/metabase/query_processor/api.clj b/src/metabase/query_processor/api.cljindex 966650473d0..7071f550a94 100644--- a/src/metabase/query_processor/api.clj+++ b/src/metabase/query_processor/api.clj@@ -186,11 +186,10 @@[:database ms/PositiveInt][:settings {:optional true} [:maybe [:map[:include_sensitive_fields {:optional true} :boolean]]]]]]- (lib-be/with-metadata-provider-cache- (queries/batch-fetch-query-metadata- [query]- (when-some [include-sensitive-fields (get-in query [:settings :include_sensitive_fields])]- {:include-sensitive-fields? include-sensitive-fields}))))+ (queries/batch-fetch-query-metadata+ [query]+ (when-some [include-sensitive-fields (get-in query [:settings :include_sensitive_fields])]+ {:include-sensitive-fields? include-sensitive-fields})));; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to;; use our API + we will need it when we make auto-TypeScript-signature generation happendiff --git a/test/metabase/lib/walk/util_test.cljc b/test/metabase/lib/walk/util_test.cljcindex 90d4a5386d3..1fea88c95b7 100644--- a/test/metabase/lib/walk/util_test.cljc+++ b/test/metabase/lib/walk/util_test.cljc@@ -345,3 +345,51 @@:segment #{}:snippet #{snippet-id}}(lib/all-referenced-entity-ids [query])))))++(deftest ^:parallel all-referenced-entity-ids-implicitly-joinable-table-test+ (testing ":include-implicitly-joinable? adds the source Table's columns' FK-target Tables to :table"+ (let [query (lib/query meta/metadata-provider (meta/table-metadata :orders))]+ (is (= #{(meta/id :orders)}+ (:table (lib/all-referenced-entity-ids [query])))+ "without the option, only the source Table is referenced")+ (is (= #{(meta/id :orders) (meta/id :people) (meta/id :products)}+ (:table (lib/all-referenced-entity-ids [query] {:include-implicitly-joinable? true})))+ "with the option, the FK-target Tables of the source columns are included too"))))++(deftest ^:parallel all-referenced-entity-ids-implicitly-joinable-card-test+ (testing ":include-implicitly-joinable? adds a source Card's result-metadata columns' FK-target Tables to :table"+ (let [card-id 1+ mp (lib.tu/metadata-provider-with-card-from-query+ meta/metadata-provider card-id+ (lib/query meta/metadata-provider (meta/table-metadata :orders)))+ query (lib/query mp (lib.metadata/card mp card-id))]+ (is (= #{}+ (:table (lib/all-referenced-entity-ids [query])))+ "without the option, the Card source references no Tables")+ (is (= #{(meta/id :people) (meta/id :products)}+ (:table (lib/all-referenced-entity-ids [query] {:include-implicitly-joinable? true})))+ "with the option, the Card columns' FK-target Tables are included"))))++(deftest ^:parallel all-referenced-entity-ids-implicitly-joinable-result-metadata-fk-override-test+ (testing ":include-implicitly-joinable? follows an FK target set on a Card's result metadata even when the raw Field+ has no such FK"+ (let [products-query (lib/query meta/metadata-provider (meta/table-metadata :products))+ returned (lib/returned-columns products-query)+ plain-mp (lib.tu/metadata-provider-with-card-from-query meta/metadata-provider 1 products-query)+ override-mp (lib.tu/metadata-provider-with-card-from-query+ meta/metadata-provider 2 products-query+ {:result-metadata (mapv (fn [col]+ (cond-> col+ (= (:id col) (meta/id :products :price))+ (assoc :fk-target-field-id (meta/id :people :id))))+ returned)})]+ (is (= #{}+ (:table (lib/all-referenced-entity-ids+ [(lib/query plain-mp (lib.metadata/card plain-mp 1))]+ {:include-implicitly-joinable? true})))+ "PRODUCTS columns have no raw FKs, so nothing is implicitly joinable")+ (is (= #{(meta/id :people)}+ (:table (lib/all-referenced-entity-ids+ [(lib/query override-mp (lib.metadata/card override-mp 2))]+ {:include-implicitly-joinable? true})))+ "the result-metadata FK-target override pulls in PEOPLE, which the raw PRODUCTS.PRICE Field does not reference"))))diff --git a/test/metabase/parameters/chain_filter_test.clj b/test/metabase/parameters/chain_filter_test.cljindex 159c7be7a23..aa6ebac9646 100644--- a/test/metabase/parameters/chain_filter_test.clj+++ b/test/metabase/parameters/chain_filter_test.clj@@ -2,6 +2,7 @@(:require[clojure.set :as set][clojure.test :refer :all]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.util :as lib.util]@@ -63,6 +64,7 @@(are [op] (some? (#'chain-filter/add-filter(lib/query mp (lib.metadata/table mp (mt/id :venues)))(mt/id :venues)+ {(mt/id :venues :name) (lib.metadata/field mp (mt/id :venues :name))}{:field-id (mt/id :venues :name):op op:value value@@ -294,7 +296,7 @@(binding [chain-filter/*enable-reverse-joins* false](mt/$ids nil(is (= [{:lhs {:table $$venues, :field %venues.category_id}, :rhs {:table $$categories, :field %categories.id}}]- (#'chain-filter/find-all-joins $$venues #{%categories.name %users.id})))))))+ (#'chain-filter/find-all-joins (mt/metadata-provider) (mt/id) $$venues #{%categories.name %users.id})))))))(deftest ^:parallel find-all-joins-test-2(mt/dataset airports@@ -305,7 +307,7 @@:rhs {:table $$municipality, :field %municipality.id}}{:lhs {:table $$municipality, :field %municipality.region_id}:rhs {:table $$region, :field %region.id}}]- (#'chain-filter/find-all-joins $$airport #{%region.name %municipality.name %region.id}))))))))+ (#'chain-filter/find-all-joins (mt/metadata-provider) (mt/id) $$airport #{%region.name %municipality.name %region.id}))))))))(deftest ^:parallel multi-hop-test(mt/dataset airports@@ -881,13 +883,14 @@[:field {} (mt/id :checkins :date)]-32:week]]}]}- (#'chain-filter/add-filter- (let [mp (mt/metadata-provider)]- (lib/query mp (lib.metadata/table mp (mt/id :checkins))))- $$checkins- {:field-id %checkins.date- :op :=- :value "past32weeks"}))))))+ (let [mp (mt/metadata-provider)]+ (#'chain-filter/add-filter+ (lib/query mp (lib.metadata/table mp (mt/id :checkins)))+ $$checkins+ {%checkins.date (lib.metadata/field mp %checkins.date)}+ {:field-id %checkins.date+ :op :=+ :value "past32weeks"})))))))(mt/defdataset nil-values-dataset[["tbl"@@ -1045,3 +1048,93 @@(t2/update! :model/Field {:id %users.id} {:active false})(testing "there are no connections when PK is inactive"(is (nil? (#'chain-filter/find-joins (mt/id) $$messages $$users)))))))))++;;; --------------------- chain-filter-mbql-query metadata app-DB call counts (no metadata N+1) ---------------------+;;;+;;; `chain-filter-mbql-query` bulk-loads all the Field/Table metadata it needs while building the query (see+;;; `find-all-joins`, `add-joins`, `add-filters`), so the number of app-DB calls is bounded: it does NOT grow with the+;;; number of constraints, nor with the number/depth of joins. A second build reuses the request-scoped+;;; metadata-provider cache and makes no app-DB calls at all. If a metadata fetch regresses to+;;; one-object-at-a-time, these exact counts change.++(defn- chain-filter-mbql-query-call-counts+ "Warm the process-global memoized caches (FK graph, `field-id->database-id`) for `field-id`/`constraints`, then+ return the app-DB call counts of a first build (fresh provider cache) and a second build (warm cache)."+ [field-id constraints]+ (#'chain-filter/chain-filter-mbql-query field-id constraints nil)+ (lib-be/with-metadata-provider-cache+ {:first (t2/with-call-count [call-count] (#'chain-filter/chain-filter-mbql-query field-id constraints nil) (call-count))+ :second (t2/with-call-count [call-count] (#'chain-filter/chain-filter-mbql-query field-id constraints nil) (call-count))}))++(deftest ^:parallel chain-filter-mbql-query-no-constraints-test+ (mt/dataset test-data+ (mt/$ids+ (let [{:keys [first second]} (chain-filter-mbql-query-call-counts %venues.name [])]+ ;; - fetch the source Field (the one we return values of)+ ;; - fetch the source Table+ ;; - fetch the Database+ (is (= 3 first))+ (testing "a second build reuses the cached metadata and makes no app-DB calls"+ (is (= 0 second)))))))++(deftest ^:parallel chain-filter-mbql-query-one-constraint-test+ (mt/dataset test-data+ (mt/$ids+ ;; a constraint on the source Table does not add a join+ (let [{:keys [first second]} (chain-filter-mbql-query-call-counts %venues.name [(shorthand->constraint %venues.price 3)])]+ ;; - fetch the source Field+ ;; - fetch the source Table+ ;; - fetch the Database+ ;; - bulk-fetch the constraint Field(s)+ (is (= 4 first))+ (testing "a second build reuses the cached metadata and makes no app-DB calls"+ (is (= 0 second)))))))++(deftest ^:parallel chain-filter-mbql-query-three-constraints-test+ (mt/dataset test-data+ (mt/$ids+ (let [{:keys [first second]} (chain-filter-mbql-query-call-counts+ %venues.name+ [(shorthand->constraint %venues.price 3)+ (shorthand->constraint %venues.id 1)+ (shorthand->constraint %venues.latitude 1)])]+ ;; same as the one-constraint case -- the constraint Fields are bulk-loaded together, not one at a time:+ ;; - fetch the source Field+ ;; - fetch the source Table+ ;; - fetch the Database+ ;; - bulk-fetch the (three) constraint Fields+ (is (= 4 first))+ (testing "a second build reuses the cached metadata and makes no app-DB calls"+ (is (= 0 second)))))))++(deftest ^:parallel chain-filter-mbql-query-one-join-test+ (mt/dataset test-data+ (mt/$ids+ ;; a constraint on another Table (categories) forces a venues -> categories join+ (let [{:keys [first second]} (chain-filter-mbql-query-call-counts %venues.name [(shorthand->constraint %categories.name "BBQ")])]+ ;; - fetch the source Field+ ;; - fetch the source Table+ ;; - fetch the Database+ ;; - bulk-fetch the constraint Field(s)+ ;; - bulk-fetch the join Fields+ ;; - bulk-fetch the join Table(s)+ (is (= 6 first))+ (testing "a second build reuses the cached metadata and makes no app-DB calls"+ (is (= 0 second)))))))++(deftest ^:parallel chain-filter-mbql-query-multi-hop-joins-test+ (mt/dataset airports+ (mt/$ids+ ;; airport -> municipality -> region is two joins+ (let [{:keys [first second]} (chain-filter-mbql-query-call-counts %airport.name [(shorthand->constraint %region.name "x")])]+ ;; same count as the single-join case -- the join Fields and Tables are each bulk-loaded in one call,+ ;; regardless of how many joins there are:+ ;; - fetch the source Field+ ;; - fetch the source Table+ ;; - fetch the Database+ ;; - bulk-fetch the constraint Field(s)+ ;; - bulk-fetch the join Fields (across both joins)+ ;; - bulk-fetch the join Tables (both of them)+ (is (= 6 first))+ (testing "a second build reuses the cached metadata and makes no app-DB calls"+ (is (= 0 second)))))))diff --git a/test/metabase/parameters/custom_values_test.clj b/test/metabase/parameters/custom_values_test.cljindex f70363f77e0..ce78c83bed2 100644--- a/test/metabase/parameters/custom_values_test.clj+++ b/test/metabase/parameters/custom_values_test.clj@@ -2,11 +2,13 @@{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.parameters.custom-values-test]}}}}}}(:require[clojure.test :refer :all]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.parameters.custom-values :as custom-values][metabase.test :as mt]- [metabase.test.fixtures :as fixtures]))+ [metabase.test.fixtures :as fixtures]+ [toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -629,3 +631,29 @@(deftest ^:parallel pk-of-fk-pk-field-ids-test-6(testing "just a non-key"(is (nil? (custom-values/pk-of-fk-pk-field-ids [(mt/id :people :name)])))))++(deftest ^:parallel card-query-bulk-loads-metadata-test+ (testing "card-query bulk-loads the value-source Card's metadata up front, so resolving the value field's column hits+ the provider cache instead of fetching one entity at a time (no N+1)"+ (let [mp (mt/metadata-provider)+ orders-query (lib/query mp (lib.metadata/table mp (mt/id :orders)))]+ (mt/with-temp [:model/Card card {:database_id (mt/id)+ :table_id (mt/id :orders)+ :type :question+ :dataset_query orders-query+ :result_metadata (lib/returned-columns orders-query)}]+ (let [card (t2/select-one :model/Card :id (:id card))+ value-field [:field "TOTAL" {:base-type :type/Float}]]+ ;; The 6 batched loads are:+ ;; - the source Card+ ;; - the source Database+ ;; - the Card's result-metadata Fields+ ;; - those Fields' FK targets+ ;; - the FK-target Tables+ ;; - those Tables' columns+ ;; This count is constant -- it must NOT grow with the number of columns/Tables (that would be the N+1 this+ ;; bulk-loading exists to prevent).+ (is (= 6 (lib-be/with-metadata-provider-cache+ (t2/with-call-count [call-count]+ (#'custom-values/can-get-card-values? (#'custom-values/card-query (:id card) (:dataset_query card)) value-field)+ (call-count))))))))))diff --git a/test/metabase/parameters/params_test.clj b/test/metabase/parameters/params_test.cljindex 884d751c7fd..d134b2cf0d5 100644--- a/test/metabase/parameters/params_test.clj+++ b/test/metabase/parameters/params_test.clj@@ -3,6 +3,9 @@{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.parameters.params-test]}}}}}}(:require[clojure.test :refer :all]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata][metabase.parameters.params :as params][metabase.public-sharing-rest.api-test :as public-test][metabase.test :as mt]@@ -169,7 +172,8 @@:target [:dimension[:field "CATEGORY" {:base-type :type/Text}]{:stage-number 2}]}]}]- (let [param-fields (-> (t2/hydrate dashboard :param_fields) :param_fields)]+ (let [param-fields (lib-be/with-metadata-provider-cache+ (-> (t2/hydrate dashboard :param_fields) :param_fields))](is (=? {"p1" [{:id (mt/id :products :id)}]"p2" [{:id (mt/id :products :id)}]"p3" [{:id (mt/id :products :id)}]@@ -266,3 +270,44 @@(is (= #{(mt/id :orders :id)(mt/id :products :id)}(params/card->template-tag-field-ids card))))))))++(deftest ^:parallel dashcards->param-field-ids-bulk-loads-metadata-test+ (testing "name-based parameter targets are resolved via filterable-columns, whose metadata is bulk-loaded across all+ the dashboard's cards up front instead of one card at a time (no N+1)"+ (mt/dataset test-data+ (let [mp (mt/metadata-provider)+ orders-query (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ products-query (lib/query mp (lib.metadata/table mp (mt/id :products)))]+ (mt/with-temp+ [:model/Card src1 {:database_id (mt/id) :table_id (mt/id :orders)+ :dataset_query orders-query+ :result_metadata (lib/returned-columns orders-query)}+ :model/Card agg1 {:database_id (mt/id)+ :dataset_query {:database (mt/id) :type :query+ :query {:source-table (str "card__" (:id src1)) :aggregation [[:count]]}}}+ :model/Card src2 {:database_id (mt/id) :table_id (mt/id :products)+ :dataset_query products-query+ :result_metadata (lib/returned-columns products-query)}+ :model/Card agg2 {:database_id (mt/id)+ :dataset_query {:database (mt/id) :type :query+ :query {:source-table (str "card__" (:id src2)) :aggregation [[:count]]}}}+ :model/Dashboard {dash-id :id} {}+ :model/DashboardCard dc1 {:dashboard_id dash-id :card_id (:id agg1)+ :parameter_mappings [{:parameter_id "p1" :card_id (:id agg1)+ :target [:dimension [:field "PRODUCT_ID" {:base-type :type/Integer}]]}]}+ :model/DashboardCard dc2 {:dashboard_id dash-id :card_id (:id agg2)+ :parameter_mappings [{:parameter_id "p2" :card_id (:id agg2)+ :target [:dimension [:field "CATEGORY" {:base-type :type/Text}]]}]}]+ (let [dashcards (-> (t2/select :model/DashboardCard :id [:in [(:id dc1) (:id dc2)]])+ (t2/hydrate :card :series))]+ ;; Each is a single set load, so this count is constant in the number of cards -- it must NOT grow with more+ ;; dashcards/Cards/Tables (that would be the N+1 this preloading exists to prevent).+ ;; - one bulk fetch of the source Cards+ ;; - one bulk fetch of those Cards' result-metadata Fields+ ;; - one bulk fetch of those Fields' FK targets+ ;; - one bulk fetch of the FK-target Tables+ ;; - one bulk fetch of those Tables' columns+ (is (= 5 (lib-be/with-metadata-provider-cache+ (t2/with-call-count [call-count]+ (params/dashcards->param-field-ids dashcards)+ (call-count)))))))))))