Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Improve app db connection pool usage with public dashboards (#75770)

Alexander Polyankin5d1dd5c03218
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 0c17f844756..3d1f36b7ef6 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -894,6 +894,7 @@
metabase.eid-translation.core
metabase.eid-translation.init}
:uses #{api
+ batch-processing
settings
util}
:model-imports #{:model/Action
diff --git a/enterprise/backend/src/metabase_enterprise/checker/provider.clj b/enterprise/backend/src/metabase_enterprise/checker/provider.clj
index 9812cffbd0f..11fc71263fb 100644
--- a/enterprise/backend/src/metabase_enterprise/checker/provider.clj
+++ b/enterprise/backend/src/metabase_enterprise/checker/provider.clj
@@ -299,7 +299,7 @@
(when-let [data (store/load-database! store db-name)]
(->database-metadata data))))
- (metadatas [_this {:keys [lib/type id table-id name]}]
+ (metadatas [_this {:keys [lib/type id table-ids name]}]
(case type
:metadata/table
(vec
@@ -329,13 +329,14 @@
:when data]
(->field-metadata (make-import-resolver store nil) data))
- table-id
- (let [table-path (store/id->ref store :table table-id)]
- (when table-path
- (for [fpath (store/fields-for-table store table-path)
- :let [data (store/load-field! store fpath)]
- :when data]
- (->field-metadata (make-import-resolver store nil) data))))
+ table-ids
+ (for [table-id table-ids
+ :let [table-path (store/id->ref store :table table-id)]
+ :when table-path
+ fpath (store/fields-for-table store table-path)
+ :let [data (store/load-field! store fpath)]
+ :when data]
+ (->field-metadata (make-import-resolver store nil) data))
:else
(for [path (store/all-field-paths store)
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj b/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj
index e01ae56e446..33343dc604a 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj
@@ -23,7 +23,7 @@
(defn- metadatas [delegate overrides {metadata-type :lib/type
search-name :name
- :keys [id card-id table-id]
+ :keys [id card-ids table-ids]
:as metadata-spec}]
(let [type-overrides (get overrides metadata-type)
metadata-keys (set (or id search-name))]
@@ -56,13 +56,23 @@
(into entities (lib.metadata.protocols/metadatas delegate metadata-spec))
(m/distinct-by :id entities))
- ;; For columns, return overrides if they're present.
+ ;; For columns, return overrides if they're present. `:card-ids` and `:table-ids` are sets of ids; entities
+ ;; with overrides use the overridden columns, the rest are delegated.
(= metadata-type :metadata/column)
- (if-let [overridden-columns (cond
- card-id (get-in overrides [::card-columns card-id])
- table-id (get-in overrides [::table-columns table-id]))]
- @overridden-columns
- (lib.metadata.protocols/metadatas delegate metadata-spec))
+ (let [[override-k spec-k ids] (cond
+ card-ids [::card-columns :card-ids card-ids]
+ table-ids [::table-columns :table-ids table-ids])]
+ (if-not override-k
+ (lib.metadata.protocols/metadatas delegate metadata-spec)
+ (let [id->overridden (into {}
+ (keep (fn [id]
+ (when-let [columns (get-in overrides [override-k id])]
+ [id columns])))
+ ids)
+ missing-ids (into #{} (remove id->overridden) ids)]
+ (concat (mapcat (comp deref val) id->overridden)
+ (when (seq missing-ids)
+ (lib.metadata.protocols/metadatas delegate (assoc metadata-spec spec-k missing-ids)))))))
:else (throw (ex-info "Unknown :lib/metadata type for OverridingMetadataProvider" metadata-spec)))))
@@ -124,7 +134,7 @@
;; should be ignored.
{[:metadata/card id] (delay (let [temp (merge (when id
(-> (inner-mp mp)
- (lib.metadata/card id)))
+ (lib.metadata.protocols/card id)))
updates)
result-metadata (or (:result-metadata updates)
(and (.returned-columns-fn mp)
diff --git a/src/metabase/activity_feed/events/recent_views.clj b/src/metabase/activity_feed/events/recent_views.clj
index 58038ff81de..7e4af0d85cc 100644
--- a/src/metabase/activity_feed/events/recent_views.clj
+++ b/src/metabase/activity_feed/events/recent_views.clj
@@ -48,10 +48,11 @@
"Handle processing for a single card query event."
[topic {:keys [card-id user-id context] :as _event}]
(try
- (let [user-id (or user-id api/*current-user-id*)]
+ (let [user-id (or user-id api/*current-user-id*)]
;; we don't want to count pinned card views or document cards
- (when-not (or (#{:collection :dashboard :dashboard-subscription} context)
- (some? (t2/select-one-fn :document_id :model/Card :id card-id)))
+ (when (and user-id
+ (not (#{:collection :dashboard :dashboard-subscription} context))
+ (not (t2/select-one-fn :document_id :model/Card :id card-id)))
(recent-views/update-users-recent-views! user-id :model/Card card-id :view)))
(catch Throwable e
(log/warnf e "Failed to process recent_views event: %s" topic))))
@@ -74,15 +75,14 @@
(m/defmethod events/publish-event! ::legacy-card-event
"Handle recent-view processing for card reads"
[topic {:keys [object-id user-id context]}]
- ;; Cards can be read indirectly, either through a pinned collection, or in a dashboard.
- ;; We only want to count direct views of cards, so we skip processing for indirect views here.
- ;; Also skip document cards.
- (when (and (= context :question)
- (nil? (t2/select-one-fn :document_id :model/Card :id object-id)))
- (try
- (recent-views/update-users-recent-views! (or user-id api/*current-user-id*) :model/Card object-id :view)
- (catch Throwable e
- (log/warnf e "Failed to process recent_views event: %s" topic)))))
+ (let [user-id (or user-id api/*current-user-id*)]
+ (when (and user-id
+ (= context :question)
+ (nil? (t2/select-one-fn :document_id :model/Card :id object-id)))
+ (try
+ (recent-views/update-users-recent-views! user-id :model/Card object-id :view)
+ (catch Throwable e
+ (log/warnf e "Failed to process recent_views event: %s" topic))))))
(derive ::collection-touch-event :metabase/event)
(derive :event/collection-touch ::collection-touch-event)
diff --git a/src/metabase/dashboards/models/dashboard.clj b/src/metabase/dashboards/models/dashboard.clj
index 598779b5f13..d61f51cc05f 100644
--- a/src/metabase/dashboards/models/dashboard.clj
+++ b/src/metabase/dashboards/models/dashboard.clj
@@ -370,8 +370,14 @@
[:name ms/NonBlankString]
[:mappings [:maybe [:set ::parameters.schema/parameter-mapping]]]])
-(mu/defn- dashboard->resolved-params :- [:map-of ms/NonBlankString ParamWithMapping]
- [dashboard :- [:map [:parameters [:maybe [:sequential :map]]]]]
+(mu/defn dashboard->resolved-params :- [:map-of ms/NonBlankString ParamWithMapping]
+ "Return map of Dashboard parameter key -> param with resolved `:mappings` (see the `:resolved-params` hydration
+ below for an example). Callers that only need the mappings (e.g. the QP) can pass slim dashcards instead of paying
+ for the full hydration."
+ [dashboard :- [:map
+ [:parameters [:maybe [:sequential :map]]]
+ [:dashcards [:maybe [:sequential [:map
+ [:parameter_mappings [:maybe [:sequential :map]]]]]]]]]
(let [param-key->mappings (apply
merge-with set/union
(for [dashcard (:dashcards dashboard)
diff --git a/src/metabase/eid_translation/util.clj b/src/metabase/eid_translation/util.clj
index 8862dc1fdec..2f11dc5c68d 100644
--- a/src/metabase/eid_translation/util.clj
+++ b/src/metabase/eid_translation/util.clj
@@ -2,6 +2,7 @@
(:require
[malli.core :as mc]
[malli.error :as me]
+ [metabase.batch-processing.core :as grouper]
[metabase.eid-translation.impl :as eid-translation]
[metabase.eid-translation.settings :as eid-translation.settings]
[metabase.util :as u]
@@ -71,12 +72,29 @@
;; -------------------- Entity Id Translation Analytics --------------------
+(def ^:private update-translation-count-interval-seconds (* 5 60))
+
+(def ^:private update-translation-count-queue-capacity 1000)
+
+(defn- update-translation-counts!*
+ "Update the entity-id translation counter with the statuses of a batch of entity-id translations."
+ [statuses]
+ (eid-translation.settings/entity-id-translation-counter!
+ (merge-with + (frequencies statuses) (eid-translation.settings/entity-id-translation-counter))))
+
+(defonce ^:private update-translation-count-queue
+ (delay (grouper/start!
+ #'update-translation-counts!*
+ :capacity update-translation-count-queue-capacity
+ :interval (* update-translation-count-interval-seconds 1000))))
+
(mu/defn- update-translation-count!
- "Update the entity-id translation counter with the results of a batch of entity-id translations."
+ "Count the results of a batch of entity-id translations. The counter is only used for the daily anonymous usage
+ stats, so the updates are batched and written asynchronously instead of paying for a (cluster-wide, cache-invalidating)
+ Setting update on every translation."
[results :- [:sequential eid-translation/Status]]
- (let [processed-result (frequencies results)]
- (eid-translation.settings/entity-id-translation-counter!
- (merge-with + processed-result (eid-translation.settings/entity-id-translation-counter)))))
+ (doseq [status results]
+ (grouper/submit! @update-translation-count-queue status)))
(mu/defn- entity-ids->id-for-model :- [:sequential [:tuple
;; We want to pass incorrectly formatted entity-ids through here,
diff --git a/src/metabase/lib/field/resolution.cljc b/src/metabase/lib/field/resolution.cljc
index 55f12f83ba3..0d7b1e6ee16 100644
--- a/src/metabase/lib/field/resolution.cljc
+++ b/src/metabase/lib/field/resolution.cljc
@@ -73,7 +73,7 @@
(pos-int? table-id))
(first (lib.metadata.protocols/metadatas
(lib.metadata/->metadata-provider metadata-providerable)
- {:lib/type :metadata/column, :table-id table-id, :name #{id-or-name}})))]
+ {:lib/type :metadata/column, :table-ids #{table-id}, :name #{id-or-name}})))]
(-> col
(assoc :lib/source :source/table-defaults
:lib/source-column-alias (:name col)
diff --git a/src/metabase/lib/metadata/cached_provider.cljc b/src/metabase/lib/metadata/cached_provider.cljc
index d3a70fdb3e3..fc8e72e2213 100644
--- a/src/metabase/lib/metadata/cached_provider.cljc
+++ b/src/metabase/lib/metadata/cached_provider.cljc
@@ -104,10 +104,37 @@
(lib.metadata.protocols/default-spec-filter-xform metadata-spec))
key-set)))
+;; cache keys used by this function have the shape [::spec <spec-with-a-single-table-id>], so fetching metadata for
+;; several Tables at once (e.g. to pre-warm the cache) and fetching it for a single Table can share cache entries
+(defn- metadatas-for-table-ids
+ [cache uncached-provider {metadata-type :lib/type, table-ids :table-ids, :as metadata-spec}]
+ (let [cache-key (fn [table-id]
+ [::spec (assoc metadata-spec :table-ids #{table-id})])
+ cached-ids (let [cache* @cache]
+ ;; [[get-in]] instead of [[get-in-cache]] because we don't want to filter out `::nil` tombstones
+ (into #{} (filter #(get-in cache* (cache-key %))) table-ids))
+ missing-ids (set/difference table-ids cached-ids)]
+ (when (seq missing-ids)
+ (let [newly-fetched (lib.metadata.protocols/metadatas uncached-provider (assoc metadata-spec :table-ids missing-ids))
+ by-table-id (group-by :table-id newly-fetched)]
+ (doseq [table-id missing-ids
+ :let [metadatas (vec (get by-table-id table-id))]]
+ (store-in-cache! cache (cache-key table-id) metadatas)
+ (doseq [metadata metadatas
+ k [:id :name]]
+ (store-in-cache! cache [metadata-type k (k metadata)] metadata)))))
+ (into [] (mapcat #(get-in-cache cache (cache-key %))) (sort table-ids))))
+
(mu/defn- metadatas
- [cache uncached-provider {metadata-type :lib/type, id-set :id, name-set :name, :as metadata-spec} :- ::lib.metadata.protocols/metadata-spec]
- (if (or id-set name-set)
+ [cache uncached-provider {metadata-type :lib/type, id-set :id, name-set :name, table-ids :table-ids, :as metadata-spec} :- ::lib.metadata.protocols/metadata-spec]
+ (cond
+ (or id-set name-set)
(metadatas-by-id-or-name cache uncached-provider metadata-spec)
+
+ table-ids
+ (metadatas-for-table-ids cache uncached-provider metadata-spec)
+
+ :else
(get-in-cache-or-fetch cache
[::spec metadata-spec]
(fn []
diff --git a/src/metabase/lib/metadata/invocation_tracker.cljc b/src/metabase/lib/metadata/invocation_tracker.cljc
index 33c1cf83dfb..52ac1f7e27b 100644
--- a/src/metabase/lib/metadata/invocation_tracker.cljc
+++ b/src/metabase/lib/metadata/invocation_tracker.cljc
@@ -22,23 +22,23 @@
;;; TODO (Cam 9/10/25) -- we're missing stuff here like handling when we fetch by name. When we added support for
;;; fetching metadata by name in #62283 we never added it.
-(defn- track-spec! [tracker {metadata-type :lib/type, id-set :id, :keys [table-id card-id], :as _metadata-spec}]
+(defn- track-spec! [tracker {metadata-type :lib/type, id-set :id, :keys [table-ids card-ids], :as _metadata-spec}]
(cond
id-set
(track-ids! tracker metadata-type id-set)
- table-id
+ table-ids
(let [tracking-type (case metadata-type
:metadata/column ::table-fields
:metadata/measure ::table-measures
:metadata/metric ::table-metrics
:metadata/segment ::table-segments)]
- (track-ids! tracker tracking-type [table-id]))
+ (track-ids! tracker tracking-type table-ids))
- card-id
+ card-ids
(let [tracking-type (case metadata-type
:metadata/metric ::card-metrics)]
- (track-ids! tracker tracking-type [card-id]))))
+ (track-ids! tracker tracking-type card-ids))))
(defn- metadatas
[tracker metadata-provider metadata-spec]
diff --git a/src/metabase/lib/metadata/protocols.cljc b/src/metabase/lib/metadata/protocols.cljc
index fe008dfad92..52fe690e96a 100644
--- a/src/metabase/lib/metadata/protocols.cljc
+++ b/src/metabase/lib/metadata/protocols.cljc
@@ -28,25 +28,25 @@
[:lib/type [:ref ::metadata-type-excluding-database]]
[:id {:optional true} [:set {:min 1} pos-int?]]
[:name {:optional true} [:set {:min 1} :string]]
- [:table-id {:optional true} ::lib.schema.id/table]
- [:card-id {:optional true} ::lib.schema.id/card]
+ [:table-ids {:optional true} [:set {:min 1} ::lib.schema.id/table]]
+ [:card-ids {:optional true} [:set {:min 1} ::lib.schema.id/card]]
[:include-sensitive? {:optional true} :boolean]]
[:fn
{:error/message ":id and :name cannot be used at the same time."}
(complement (every-pred :id :name))]
[:fn
- {:error/message ":table-id is currently only supported for Fields, Measures, Metrics, and Segments."}
+ {:error/message ":table-ids is currently only supported for Fields, Measures, Metrics, and Segments."}
(fn [spec]
- (or (not (:table-id spec))
+ (or (not (:table-ids spec))
(#{:metadata/column :metadata/measure :metadata/metric :metadata/segment} (:lib/type spec))))]
[:fn
- {:error/message ":card-id is currently only supported for Metrics."}
+ {:error/message ":card-ids is currently only supported for Metrics."}
(fn [spec]
- (or (not (:card-id spec))
+ (or (not (:card-ids spec))
(#{:metadata/metric} (:lib/type spec))))]
[:fn
{:error/message "All metadata types except for :metadata/table and :metadata/transform must include at least one filter"}
- (some-fn :id :name :table-id :card-id #(= (:lib/type %) :metadata/table) #(= (:lib/type %) :metadata/transform))]])
+ (some-fn :id :name :table-ids :card-ids #(= (:lib/type %) :metadata/table) #(= (:lib/type %) :metadata/transform))]])
(defn active-column-pred
"Returns a predicate that filters out columns that should not be visible based on `:active` status and
@@ -69,21 +69,21 @@
This should match [[metabase.lib-be.metadata.jvm/metadata-spec->honey-sql]] as closely as
possible."
- [{metadata-type :lib/type, id-set :id, name-set :name, :keys [table-id card-id include-sensitive?], :as _metadata-spec} :- ::metadata-spec]
+ [{metadata-type :lib/type, id-set :id, name-set :name, :keys [table-ids card-ids include-sensitive?], :as _metadata-spec} :- ::metadata-spec]
(let [active-only? (not (or id-set name-set))
metric? (= metadata-type :metadata/metric)
preds [(when id-set
#(contains? id-set (:id %)))
(when name-set
#(contains? name-set (:name %)))
- (when table-id
- #(= (:table-id %) table-id))
- (when (and table-id metric?)
+ (when table-ids
+ #(contains? table-ids (:table-id %)))
+ (when (and table-ids metric?)
#(nil? (:source-card-id %)))
- (when card-id
+ (when card-ids
(if metric?
- #(= (:source-card-id %) card-id)
- #(= (:card-id %) card-id)))
+ #(contains? card-ids (:source-card-id %))
+ #(contains? card-ids (:card-id %))))
(when active-only?
(case metadata-type
:metadata/table
@@ -223,7 +223,7 @@
[metadata-provider :- ::metadata-provider
metadata-type :- [:enum :metadata/column :metadata/measure :metadata/metric :metadata/segment]
table-id :- ::lib.schema.id/table]
- (metadatas metadata-provider {:lib/type metadata-type, :table-id table-id}))
+ (metadatas metadata-provider {:lib/type metadata-type, :table-ids #{table-id}}))
(mu/defn metadatas-for-card :- [:maybe [:sequential ::metadata]]
"Return active (non-archived) metadatas associated with a particular Card, currently only Metrics, so
@@ -231,7 +231,7 @@
[metadata-provider :- ::metadata-provider
metadata-type :- [:enum :metadata/column :metadata/metric :metadata/segment]
card-id :- ::lib.schema.id/card]
- (metadatas metadata-provider {:lib/type metadata-type, :card-id card-id}))
+ (metadatas metadata-provider {:lib/type metadata-type, :card-ids #{card-id}}))
(mu/defn table :- [:maybe ::lib.schema.metadata/table]
"Return metadata for a specific Table. Metadata should satisfy `:metabase.lib.schema.metadata/table`."
@@ -303,23 +303,24 @@
([metadata-provider :- ::metadata-provider
table-id :- ::lib.schema.id/table
opts :- [:maybe [:map [:include-sensitive? {:optional true} :boolean]]]]
- (metadatas metadata-provider {:lib/type :metadata/column
- :table-id table-id
- :include-sensitive? (boolean (:include-sensitive? opts))})))
+ ;; only include `:include-sensitive?` when truthy, so that this shares cached-provider cache entries with other
+ ;; ways of fetching the fields for a table, e.g. [[metadatas-for-tables]]
+ (metadatas metadata-provider (cond-> {:lib/type :metadata/column, :table-ids #{table-id}}
+ (:include-sensitive? opts) (assoc :include-sensitive? true)))))
(mu/defn segments :- [:maybe [:sequential ::lib.schema.metadata/segment]]
"Return a sequence of legacy Segments associated with a Table with the given `table-id`. Segments should satisfy
the `:metabase.lib.schema.metadata/segment` schema. If no Table with ID `table-id` exists, this should error."
[metadata-provider :- ::metadata-provider
table-id :- ::lib.schema.id/table]
- (metadatas metadata-provider {:lib/type :metadata/segment, :table-id table-id}))
+ (metadatas metadata-provider {:lib/type :metadata/segment, :table-ids #{table-id}}))
(mu/defn measures :- [:maybe [:sequential ::lib.schema.metadata/measure]]
"Return a sequence of Measures associated with a Table with the given `table-id`. Measures should satisfy
the `:metabase.lib.schema.metadata/measure` schema. If no Table with ID `table-id` exists, this should error."
[metadata-provider :- ::metadata-provider
table-id :- ::lib.schema.id/table]
- (metadatas metadata-provider {:lib/type :metadata/measure, :table-id table-id}))
+ (metadatas metadata-provider {:lib/type :metadata/measure, :table-ids #{table-id}}))
(#?(:clj p/defprotocol+ :cljs defprotocol) CachedMetadataProvider
"Optional. A protocol for a MetadataProvider that some sort of internal cache. This is mostly useful for
diff --git a/src/metabase/lib_be/metadata/jvm.clj b/src/metabase/lib_be/metadata/jvm.clj
index 75d9595ed11..0698d844ec0 100644
--- a/src/metabase/lib_be/metadata/jvm.clj
+++ b/src/metabase/lib_be/metadata/jvm.clj
@@ -513,20 +513,20 @@
{:closed true}
[:where {:optional true} vector?]]
"This should match [[metabase.lib.metadata.protocols/default-spec-filter-xform]] as closely as possible."
- [database-id :- ::lib.schema.id/database
- {metadata-type :lib/type, id-set :id, name-set :name, :keys [table-id card-id include-sensitive?], :as _metadata-spec} :- ::lib.metadata.protocols/metadata-spec]
+ [database-id :- ::lib.schema.id/database
+ {metadata-type :lib/type, id-set :id, name-set :name, :keys [table-ids card-ids include-sensitive?], :as _metadata-spec} :- ::lib.metadata.protocols/metadata-spec]
(let [database-id-key (db-id-key metadata-type)
active-only? (not (or id-set name-set))
metric? (= metadata-type :metadata/metric)
where-clauses (cond-> []
- database-id-key (conj [:= database-id-key database-id])
- id-set (conj [:in (id-key metadata-type) id-set])
- name-set (conj [:in (name-key metadata-type) name-set])
- table-id (conj [:= (table-id-key metadata-type) table-id])
- card-id (conj [:= (card-id-key metadata-type) card-id])
- active-only? (conj (active-only-honeysql-filter metadata-type {:include-sensitive? include-sensitive?}))
- metric? (conj [:= :type [:inline "metric"]])
- (and metric? table-id) (conj [:= :source_card_id nil]))]
+ database-id-key (conj [:= database-id-key database-id])
+ id-set (conj [:in (id-key metadata-type) id-set])
+ name-set (conj [:in (name-key metadata-type) name-set])
+ table-ids (conj [:in (table-id-key metadata-type) table-ids])
+ card-ids (conj [:in (card-id-key metadata-type) card-ids])
+ active-only? (conj (active-only-honeysql-filter metadata-type {:include-sensitive? include-sensitive?}))
+ metric? (conj [:= :type [:inline "metric"]])
+ (and metric? table-ids) (conj [:= :source_card_id nil]))]
(reduce
sql.helpers/where
{}
diff --git a/src/metabase/lib_be/query.clj b/src/metabase/lib_be/query.clj
index ece3fc94893..6d005300cdb 100644
--- a/src/metabase/lib_be/query.clj
+++ b/src/metabase/lib_be/query.clj
@@ -1,6 +1,8 @@
(ns metabase.lib-be.query
(:require
[metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.metadata.protocols :as lib.metadata.protocols]
+ [metabase.lib.metadata.util :as lib.metadata.util]
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.lib.walk.util :as lib.walk.util]
[metabase.util.malli :as mu]))
@@ -9,10 +11,18 @@
"Bulk-load all metadata referenced by `referenced-entity-ids` into `metadata-providerable`'s cache."
[metadata-providerable :- ::lib.schema.metadata/metadata-providerable
{:keys [table card metric measure segment snippet]} :- ::lib.walk.util/referenced-entity-ids]
- (lib.metadata/bulk-metadata metadata-providerable :metadata/table table)
- (lib.metadata/bulk-metadata metadata-providerable :metadata/card card)
- (lib.metadata/bulk-metadata metadata-providerable :metadata/metric metric)
- (lib.metadata/bulk-metadata metadata-providerable :metadata/measure measure)
- (lib.metadata/bulk-metadata metadata-providerable :metadata/segment segment)
- (lib.metadata/bulk-metadata metadata-providerable :metadata/native-query-snippet snippet)
+ (when (seq table)
+ (lib.metadata/bulk-metadata metadata-providerable :metadata/table table)
+ (lib.metadata.protocols/metadatas (lib.metadata.util/->metadata-provider metadata-providerable)
+ {:lib/type :metadata/column, :table-ids (set table)}))
+ (when (seq card)
+ (lib.metadata/bulk-metadata metadata-providerable :metadata/card card))
+ (when (seq metric)
+ (lib.metadata/bulk-metadata metadata-providerable :metadata/metric metric))
+ (when (seq measure)
+ (lib.metadata/bulk-metadata metadata-providerable :metadata/measure measure))
+ (when (seq segment)
+ (lib.metadata/bulk-metadata metadata-providerable :metadata/segment segment))
+ (when (seq snippet)
+ (lib.metadata/bulk-metadata metadata-providerable :metadata/native-query-snippet snippet))
nil)
diff --git a/src/metabase/lib_metric/dimension.cljc b/src/metabase/lib_metric/dimension.cljc
index 111682bae77..82cc939c85e 100644
--- a/src/metabase/lib_metric/dimension.cljc
+++ b/src/metabase/lib_metric/dimension.cljc
@@ -213,7 +213,7 @@
table-id :- pos-int?]
(lib.metadata.protocols/metadatas
metadata-provider
- {:lib/type :metadata/dimension :table-id table-id}))
+ {:lib/type :metadata/dimension :table-ids #{table-id}}))
;;; ------------------------------------------------- Dimension Resolution -------------------------------------------------
;;; Functions for resolving dimension UUIDs to field IDs through dimension mappings.
diff --git a/src/metabase/lib_metric/metadata/jvm.clj b/src/metabase/lib_metric/metadata/jvm.clj
index 94f9b13dd5e..6034dd4dbf1 100644
--- a/src/metabase/lib_metric/metadata/jvm.clj
+++ b/src/metabase/lib_metric/metadata/jvm.clj
@@ -30,14 +30,14 @@
(defn- metric-spec->honey-sql
"Build HoneySQL WHERE clause for fetching metrics.
Metrics are Cards with type='metric' - not scoped to any database."
- [{id-set :id, name-set :name, :keys [table-id card-id], :as _metadata-spec}]
+ [{id-set :id, name-set :name, :keys [table-ids card-ids], :as _metadata-spec}]
(let [active-only? (not (or id-set name-set))
where-clauses (cond-> [[:= :type [:inline "metric"]]]
id-set (conj [:in :id id-set])
name-set (conj [:in :name name-set])
- table-id (conj [:= :table_id table-id])
- table-id (conj [:= :source_card_id nil])
- card-id (conj [:= :source_card_id card-id])
+ table-ids (conj [:in :table_id table-ids])
+ table-ids (conj [:= :source_card_id nil])
+ card-ids (conj [:in :source_card_id card-ids])
active-only? (conj [:= :archived false]))]
(reduce sql.helpers/where {} where-clauses)))
@@ -76,12 +76,12 @@
(defn- measure-spec->honey-sql
"Build HoneySQL WHERE clause for fetching measures."
- [{id-set :id, name-set :name, :keys [table-id], :as _metadata-spec}]
+ [{id-set :id, name-set :name, :keys [table-ids], :as _metadata-spec}]
(let [active-only? (not (or id-set name-set))
where-clauses (cond-> []
id-set (conj [:in :measure/id id-set])
name-set (conj [:in :measure/name name-set])
- table-id (conj [:= :measure/table_id table-id])
+ table-ids (conj [:in :measure/table_id table-ids])
active-only? (conj [:= :measure/archived false]))]
(reduce sql.helpers/where {} where-clauses)))
@@ -99,19 +99,19 @@
(defn- fetch-dimensions
"Fetch dimensions by aggregating from metrics and measures.
Dimensions are extracted from their parent entities and annotated with source info."
- [{id-set :id, :keys [metric-id measure-id table-id]}]
+ [{id-set :id, :keys [metric-id measure-id table-ids]}]
(let [;; Fetch from metrics if not a measure-specific query
metric-dims (when-not measure-id
(let [metric-spec (cond-> {:lib/type :metadata/metric}
metric-id (assoc :id #{metric-id})
- table-id (assoc :table-id table-id))
+ table-ids (assoc :table-ids table-ids))
metrics (fetch-metrics metric-spec)]
(mapcat #(extract-dimensions-from-entity % :metric) metrics)))
;; Fetch from measures if not a metric-specific query
measure-dims (when-not metric-id
(let [measure-spec (cond-> {:lib/type :metadata/measure}
measure-id (assoc :id #{measure-id})
- table-id (assoc :table-id table-id))
+ table-ids (assoc :table-ids table-ids))
measures (fetch-measures-for-dimensions measure-spec)]
(mapcat #(extract-dimensions-from-entity % :measure) measures)))
all-dims (concat metric-dims measure-dims)]
@@ -144,7 +144,7 @@
(lib.metadata.protocols/metadatas mp {:lib/type :metadata/dimension :metric-id 1})
;; Routes to correct database provider for table 1
- (lib.metadata.protocols/metadatas mp {:lib/type :metadata/column :table-id 1})
+ (lib.metadata.protocols/metadatas mp {:lib/type :metadata/column :table-ids #{1}})
```"
[]
(let [table->db (table->database-id)
diff --git a/src/metabase/lib_metric/metadata/provider.cljc b/src/metabase/lib_metric/metadata/provider.cljc
index d619ae0c765..66706b3b481 100644
--- a/src/metabase/lib_metric/metadata/provider.cljc
+++ b/src/metabase/lib_metric/metadata/provider.cljc
@@ -38,12 +38,15 @@
tables-by-db))))
(defn- route-metadata-by-table
- "Route a metadata request to the appropriate database provider based on table-id."
- [table->db-fn db-provider-fn {:keys [table-id], :as metadata-spec}]
- (when table-id
- (when-let [db-id (table->db-fn table-id)]
- (when-let [provider (db-provider-fn db-id)]
- (lib.metadata.protocols/metadatas provider metadata-spec)))))
+ "Route a metadata request to the appropriate database provider(s) based on its `:table-ids` set."
+ [table->db-fn db-provider-fn {table-ids :table-ids, :as metadata-spec}]
+ (when (seq table-ids)
+ (into []
+ (mapcat (fn [[db-id table-ids]]
+ (when db-id
+ (when-let [provider (db-provider-fn db-id)]
+ (lib.metadata.protocols/metadatas provider (assoc metadata-spec :table-ids (set table-ids)))))))
+ (group-by table->db-fn table-ids))))
(defn- route-card-metadata
"Route card metadata request. Cards can span databases, so we need to handle this specially."
diff --git a/src/metabase/queries/core.clj b/src/metabase/queries/core.clj
index 5b2a6d480e7..21338dbebf9 100644
--- a/src/metabase/queries/core.clj
+++ b/src/metabase/queries/core.clj
@@ -44,7 +44,7 @@
[metabase.queries.models.query
average-execution-time-ms
query->database-and-table-ids
- save-query-and-update-average-execution-time!])
+ save-queries-and-update-average-execution-times!])
#_{:clj-kondo/ignore [:missing-docstring]}
(p/import-def metabase.queries.models.card/populate-query-fields populate-card-query-fields)
diff --git a/src/metabase/queries/models/query.clj b/src/metabase/queries/models/query.clj
index 9eac4389b5e..ffb118ef735 100644
--- a/src/metabase/queries/models/query.clj
+++ b/src/metabase/queries/models/query.clj
@@ -9,11 +9,14 @@
[metabase.queries.schema :as queries.schema]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.json :as json]
+ [metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[methodical.core :as methodical]
[toucan2.core :as t2]
- [toucan2.model :as t2.model]))
+ [toucan2.model :as t2.model])
+ (:import
+ (java.nio ByteBuffer)))
(set! *warn-on-reflection* true)
@@ -43,12 +46,22 @@
:unsigned
:integer))
+(def ^:private smoothing-factor
+ "The weight of the latest execution time in the exponential rolling average formula:
+
+ new-average = decay-factor * average + smoothing-factor * execution-time"
+ 0.1)
+
+(def ^:private decay-factor
+ "The weight of the previous average in the exponential rolling average formula, see [[smoothing-factor]]."
+ (- 1.0 smoothing-factor))
+
(defn- update-rolling-average-execution-time!
"Update the rolling average execution time for query with `query-hash`. Returns `true` if a record was updated,
or `false` if no matching records were found."
^Boolean [query ^bytes query-hash ^Integer execution-time-ms]
- (let [avg-execution-time (h2x/cast (int-casting-type) (h2x/round (h2x/+ (h2x/* [:inline 0.9] :average_execution_time)
- [:inline (* 0.1 execution-time-ms)])
+ (let [avg-execution-time (h2x/cast (int-casting-type) (h2x/round (h2x/+ (h2x/* [:inline decay-factor] :average_execution_time)
+ [:inline (* smoothing-factor execution-time-ms)])
[:inline 0]))]
(or
;; if it DOES NOT have a query (yet) set that. In 0.31.0 we added the query.query column, and it gets set for all
@@ -63,29 +76,113 @@
{:query_hash query-hash}
{:average_execution_time avg-execution-time})))))
-(defn- record-new-query-entry!
- "Record a query and its execution time for a `query` with `query-hash` that's not already present in the DB.
- `execution-time-ms` is used as a starting point."
- [query ^bytes query-hash ^Integer execution-time-ms]
- (first (t2/insert-returning-instances! :model/Query
- :query query
- :query_hash query-hash
- :average_execution_time execution-time-ms)))
-
-(defn save-query-and-update-average-execution-time!
- "Update the recorded average execution time (or insert a new record if needed) for `query` with `query-hash`."
- [query ^bytes query-hash ^Integer execution-time-ms]
- {:pre [(bytes? query-hash)]}
- (or
- ;; if there's already a matching Query update the rolling average
- (update-rolling-average-execution-time! query query-hash execution-time-ms)
- ;; otherwise try adding a new entry. If for some reason there was a race condition and a Query entry was added in
- ;; the meantime we'll try updating that existing record
- (try (record-new-query-entry! query query-hash execution-time-ms)
- (catch Throwable e
- (or (update-rolling-average-execution-time! query query-hash execution-time-ms)
- ;; rethrow e if updating an existing average execution time failed
- (throw e))))))
+(defn- rolling-average-coefficients
+ "Collapse applying the rolling-average formula (see [[smoothing-factor]]) once per execution time into a single
+ linear equation `new-average = c0 * average + c1`."
+ [execution-times-ms]
+ (reduce (fn [[c0 c1] t]
+ [(* decay-factor c0) (+ (* decay-factor c1) (* smoothing-factor t))])
+ [1.0 0.0]
+ execution-times-ms))
+
+(defn- initial-average-execution-time
+ "Average execution time for a new Query entry: the first execution time, with any subsequent ones folded in the same
+ way consecutive [[update-rolling-average-execution-time!]] calls would."
+ ^long [execution-times-ms]
+ (reduce (fn [avg t]
+ (Math/round (+ (* decay-factor (double avg)) (* smoothing-factor (double t)))))
+ (long (first execution-times-ms))
+ (rest execution-times-ms)))
+
+(defn- hash-key
+ "Wrap a query-hash byte array so it can be used as a map key: unlike byte arrays themselves, `ByteBuffer`s have
+ value-based equality and hash code."
+ [hash-bytes]
+ (when hash-bytes
+ (ByteBuffer/wrap ^bytes hash-bytes)))
+
+(defn- rolling-average-update-expr
+ "HoneySQL expression that updates `:average_execution_time` as if the rolling-average formula were applied once per
+ execution time, in order."
+ [execution-times-ms]
+ (let [[c0 c1] (rolling-average-coefficients execution-times-ms)]
+ (h2x/cast (int-casting-type)
+ (h2x/round (h2x/+ (h2x/* [:inline c0] :average_execution_time)
+ [:inline c1])
+ [:inline 0]))))
+
+(defn- update-rolling-average-execution-times!
+ "Update the rolling average execution times for `groups` (each a sequence of entries sharing a query hash) with a
+ single atomic UPDATE."
+ [groups]
+ (when (seq groups)
+ (let [hash+exprs (vec (for [group groups]
+ [(:query-hash (first group))
+ (rolling-average-update-expr (map :execution-time-ms group))]))]
+ (t2/query {:update (t2/table-name :model/Query)
+ :set {:average_execution_time (into [:case]
+ (mapcat (fn [[query-hash expr]]
+ [[:= :query_hash query-hash] expr]))
+ hash+exprs)}
+ :where [:in :query_hash (mapv first hash+exprs)]}))))
+
+(defn- insert-query-entries!
+ "Insert new Query rows for `groups` (each a sequence of entries sharing a query hash) with a single INSERT."
+ [groups]
+ (t2/insert! :model/Query (for [group groups]
+ {:query (:query (first group))
+ :query_hash (:query-hash (first group))
+ :average_execution_time (initial-average-execution-time
+ (map :execution-time-ms group))})))
+
+(defn- query-hash->row-status
+ "Fetch which of `query-hashes` already have Query rows. Returns a map of [[hash-key]] -> `:up-to-date` or
+ `:needs-query-backfill` (for rows predating 0.31.0 whose `query` column was never set); hashes without a row are
+ absent from the map."
+ [query-hashes]
+ (if (empty? query-hashes)
+ {}
+ (into {}
+ (map (fn [{:keys [query_hash missing_query]}]
+ [(hash-key query_hash)
+ ;; mysql returns 0/1 instead of booleans
+ (if (contains? #{true 1} missing_query)
+ :needs-query-backfill
+ :up-to-date)]))
+ (t2/reducible-query {:select [:query_hash [[:= :query nil] :missing_query]]
+ :from [(t2/table-name :model/Query)]
+ :where [:in :query_hash query-hashes]}))))
+
+(defn save-queries-and-update-average-execution-times!
+ "Update the recorded average execution times (or insert new records as needed) for `entries`, maps with `:query`,
+ `:query-hash`, and `:execution-time-ms` keys, using a fixed number of statements rather than several per entry.
+ Multiple entries for the same hash are folded into a single atomic update equivalent to applying the rolling-average
+ formula once per entry, in order.
+
+ The INSERT of new Query rows runs in a nested transaction (a savepoint), so that if it conflicts inside a caller's
+ transaction (e.g. when batch updates run synchronously inside a transaction in tests), the constraint violation
+ doesn't abort the outer transaction -- Postgres refuses to run any further statements in a transaction with a failed
+ statement otherwise. A conflict means someone else just inserted one of these hashes with its own initial average;
+ dropping this batch's samples for a brand-new row doesn't meaningfully change the rolling average, so there is no
+ need to retry."
+ [entries]
+ (when (seq entries)
+ (let [groups (vals (group-by (fn [entry] (hash-key (:query-hash entry))) entries))
+ hash->status (query-hash->row-status (map (fn [group] (:query-hash (first group))) groups))
+ {:keys [up-to-date needs-query-backfill new-entry]}
+ (group-by (fn [group]
+ (get hash->status (hash-key (:query-hash (first group))) :new-entry))
+ groups)]
+ (update-rolling-average-execution-times! up-to-date)
+ ;; rare legacy path: rows whose `query` still needs backfilling are handled individually
+ (doseq [{:keys [query query-hash execution-time-ms]} (apply concat needs-query-backfill)]
+ (update-rolling-average-execution-time! query query-hash execution-time-ms))
+ (when (seq new-entry)
+ (try
+ (t2/with-transaction [_conn]
+ (insert-query-entries! new-entry))
+ (catch Throwable e
+ (log/tracef e "Error inserting concurrently created Query entries: %s" (ex-message e))))))))
(mr/def ::database-and-table-ids
[:map
diff --git a/src/metabase/query_processor/dashboard.clj b/src/metabase/query_processor/dashboard.clj
index f491a8033a1..de94dd6677d 100644
--- a/src/metabase/query_processor/dashboard.clj
+++ b/src/metabase/query_processor/dashboard.clj
@@ -5,6 +5,7 @@
[clojure.string :as str]
[medley.core :as m]
[metabase.api.common :as api]
+ [metabase.dashboards.models.dashboard :as dashboard]
[metabase.dashboards.schema :as dashboards.schema]
[metabase.events.core :as events]
[metabase.lib.core :as lib]
@@ -127,9 +128,11 @@
request-params :- [:maybe [:sequential :map]]]
(log/tracef "Resolving Dashboard %d Card %d query request parameters" dashboard-id card-id)
(let [request-params (some-> request-params not-empty (->> (lib/normalize ::dashboards.schema/parameters)))
- dashboard (-> (t2/select-one :model/Dashboard :id dashboard-id)
- (t2/hydrate :resolved-params)
- (api/check-404))
+ dashboard (api/check-404 (t2/select-one :model/Dashboard :id dashboard-id))
+ dashcard (api/check-404 (t2/select-one [:model/DashboardCard :id :card_id :dashboard_id :parameter_mappings]
+ :id dashcard-id
+ :dashboard_id dashboard-id))
+ resolved-params (dashboard/dashboard->resolved-params (assoc dashboard :dashcards [dashcard]))
dashboard-param-id->param (into {}
;; remove the `:default` values from Dashboard params. We don't ACTUALLY want to
;; use these values ourselves -- the expectation is that the frontend will pass
@@ -139,7 +142,7 @@
;; more information.
(map (fn [[param-id param]]
[param-id (dissoc param :default)]))
- (:resolved-params dashboard))
+ resolved-params)
;; ignore default values in request params as well. (#20516)
request-param-id->param (into {} (map (juxt :id #(dissoc % :default))) request-params)
merged-parameters (vals (merge (dashboard-param-defaults dashboard-param-id->param card-id)
diff --git a/src/metabase/query_processor/middleware/cache.clj b/src/metabase/query_processor/middleware/cache.clj
index 0589f663aff..9e5f1e168a2 100644
--- a/src/metabase/query_processor/middleware/cache.clj
+++ b/src/metabase/query_processor/middleware/cache.clj
@@ -12,6 +12,7 @@
[clojure.string :as str]
[java-time.api :as t]
[medley.core :as m]
+ [metabase.batch-processing.core :as grouper]
[metabase.cache.core :as cache]
[metabase.config.core :as config]
[metabase.lib.core :as lib]
@@ -45,14 +46,31 @@
;;; ------------------------------------------------------ Save ------------------------------------------------------
-(defn- purge! [backend]
- (try
- (log/tracef "Purging cache entries older than %s" (u/format-seconds (cache/query-caching-max-ttl)))
- (i/purge-old-entries! backend (cache/query-caching-max-ttl))
- (log/trace "Successfully purged old cache entries.")
- :done
- (catch Throwable e
- (log/errorf e "Error purging old cache entries: %s" (ex-message e)))))
+(def ^:private purge-interval-seconds
+ "How often, at most, to purge cache entries older than [[cache/query-caching-max-ttl]]. Purging on every save is
+ wasteful: the max TTL is measured in days, and on instances with a high cache-miss rate the repeated DELETEs are
+ expensive and contend with concurrent cache writes."
+ (* 5 60))
+
+(def ^:private purge-queue-capacity 1000)
+
+(defn- purge!* [backends]
+ (doseq [backend (distinct backends)]
+ (try
+ (log/tracef "Purging cache entries older than %s" (u/format-seconds (cache/query-caching-max-ttl)))
+ (i/purge-old-entries! backend (cache/query-caching-max-ttl))
+ (log/trace "Successfully purged old cache entries.")
+ (catch Throwable e
+ (log/errorf e "Error purging old cache entries: %s" (ex-message e))))))
+
+(defonce ^:private purge-queue
+ (delay (grouper/start!
+ #'purge!*
+ :capacity purge-queue-capacity
+ :interval (* purge-interval-seconds 1000))))
+
+(defn- schedule-purge! [backend]
+ (grouper/submit! @purge-queue backend))
(def ^:private ^:dynamic *in-fn*
"The `in-fn` provided by [[impl/do-with-serialization]]."
@@ -87,7 +105,7 @@
(log/trace "Got serialized bytes; saving to cache backend")
(i/save-results! *backend* query-hash bytez)
(log/debug "Successfully cached results for query.")
- (purge! *backend*))))
+ (schedule-purge! *backend*))))
:done
(catch Throwable e
(if (= (:type (ex-data e)) ::impl/max-bytes)
diff --git a/src/metabase/query_processor/middleware/prefetch_metadata.clj b/src/metabase/query_processor/middleware/prefetch_metadata.clj
new file mode 100644
index 00000000000..d33266ecb42
--- /dev/null
+++ b/src/metabase/query_processor/middleware/prefetch_metadata.clj
@@ -0,0 +1,46 @@
+(ns metabase.query-processor.middleware.prefetch-metadata
+ "Middleware that bulk-loads all the metadata referenced by a query into the metadata provider's cache up front, so
+ that later middleware and SQL compilation hit the cache instead of fetching objects from the app DB one at a time."
+ (:require
+ [clojure.set :as set]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.metadata.protocols :as lib.metadata.protocols]
+ [metabase.util.log :as log]))
+
+(defn- card->referenced-entity-ids
+ [metadata-provider card]
+ (try
+ (when (seq (:dataset-query card))
+ (lib/all-referenced-entity-ids [(lib/card->underlying-query metadata-provider card)]))
+ (catch Throwable e
+ ;; prefetching is an optimization: a Card we cannot make sense of here will be dealt with when it is resolved
+ (log/debugf e "Error getting referenced entity ids for Card %s" (:id card)))))
+
+(defn- all-referenced-entity-ids
+ "Collect the ids of all the entities referenced by `query`, following Card references (source cards and
+ `{{#card-id}}` template tags) recursively: the referenced Cards are fetched level by level, one bulk call per level
+ of nesting, and the entities their queries reference are folded into the result."
+ [metadata-provider query]
+ (loop [referenced-ids (lib/all-referenced-entity-ids [query])
+ seen-card-ids #{}]
+ (let [new-card-ids (set/difference (:card referenced-ids) seen-card-ids)]
+ (if (seq new-card-ids)
+ (recur (transduce (keep #(card->referenced-entity-ids metadata-provider %))
+ (partial merge-with set/union)
+ referenced-ids
+ (lib.metadata/bulk-metadata metadata-provider :metadata/card new-card-ids))
+ (into seen-card-ids new-card-ids))
+ referenced-ids))))
+
+(defn prefetch-metadata
+ "Bulk-load the metadata for everything referenced by `query` -- tables, cards (recursively), metrics, measures,
+ segments, snippets, and the fields referenced by template tags or implicit joins -- into the metadata provider's
+ cache using a constant number of app-DB calls per level of Card nesting. Runs before source cards are resolved, so
+ that resolving them hits the cache."
+ [query]
+ (let [metadata-provider (lib.metadata/->metadata-provider query)]
+ (when (lib.metadata.protocols/cached-metadata-provider-with-cache? metadata-provider)
+ (lib-be/bulk-load-query-metadata! metadata-provider (all-referenced-entity-ids metadata-provider query))))
+ query)
diff --git a/src/metabase/query_processor/middleware/process_userland_query.clj b/src/metabase/query_processor/middleware/process_userland_query.clj
index 387ed0e094d..379fa350da7 100644
--- a/src/metabase/query_processor/middleware/process_userland_query.clj
+++ b/src/metabase/query_processor/middleware/process_userland_query.clj
@@ -5,13 +5,14 @@
ViewLog recording is triggered indirectly by the call to [[events/publish-event!]] with the `:event/card-query`
event -- see [[metabase.view-log.events.view-log]]."
- (:refer-clojure :exclude [every? empty? get-in])
+ (:refer-clojure :exclude [every? empty? get-in not-empty])
(:require
[java-time.api :as t]
[metabase.analytics-interface.core :as analytics]
[metabase.analytics.core :as analytics.core]
[metabase.analytics.settings :as analytics.settings]
[metabase.api.common :as api]
+ [metabase.batch-processing.core :as grouper]
[metabase.events.core :as events]
[metabase.lib.computed :as lib.computed]
[metabase.queries.models.query :as query]
@@ -23,7 +24,7 @@
[metabase.util.json :as json]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
- [metabase.util.performance :refer [every? empty? get-in]]
+ [metabase.util.performance :refer [every? empty? get-in not-empty]]
^{:clj-kondo/ignore [:discouraged-namespace]}
[toucan2.core :as t2]))
@@ -40,43 +41,52 @@
;;; | Save Query Execution |
;;; +----------------------------------------------------------------------------------------------------------------+
-;; TODO - I'm not sure whether this should happen async as is currently the case, or should happen synchronously e.g.
-;; in the completing arity of the rf
-;;
-;; Async seems like it makes sense from a performance standpoint, but should we have some sort of shared threadpool
-;; for other places where we would want to do async saves (such as results-metadata for Cards?)
+(def ^:private save-execution-metadata-interval-seconds 20)
+
+(def ^:private save-execution-metadata-queue-capacity 500)
+
(defn- save-execution-metadata!*
- "Save a `QueryExecution` and update the average execution time for the corresponding `Query`."
- [{query :json_query, query-hash :hash, running-time :running_time, context :context :as query-execution}]
+ "Save a batch of `QueryExecution`s and update the average execution times for the corresponding `Query`s."
+ [query-executions]
(tracing/with-span :db-app "db-app.save-query-execution" {}
- (when-not (:cache_hit query-execution)
- (query/save-query-and-update-average-execution-time! query query-hash running-time))
- (if-not context
- (log/warn "Cannot save QueryExecution, missing :context")
- (t2/insert-returning-pk! :model/QueryExecution (dissoc query-execution :json_query)))))
+ (log/tracef "Saving %d QueryExecutions" (count query-executions))
+ (when-let [entries (not-empty
+ (for [{query :json_query, query-hash :hash, running-time :running_time, :as query-execution} query-executions
+ :when (and (not (:cache_hit query-execution)) query-hash running-time)]
+ {:query query, :query-hash query-hash, :execution-time-ms running-time}))]
+ (try
+ (query/save-queries-and-update-average-execution-times! entries)
+ (catch Throwable e
+ (log/error e "Error updating query average execution times"))))
+ (try
+ (let [{with-context true, no-context false} (group-by (comp some? :context) query-executions)]
+ (when (seq no-context)
+ (log/warnf "Cannot save %d QueryExecution(s), missing :context" (count no-context)))
+ (when (seq with-context)
+ (t2/insert! :model/QueryExecution (map #(dissoc % :json_query) with-context))))
+ (catch Throwable e
+ (log/error e "Error saving query execution info")))))
+
+(defonce ^:private save-execution-metadata-queue
+ (delay (grouper/start!
+ #'save-execution-metadata!*
+ :capacity save-execution-metadata-queue-capacity
+ :interval (* save-execution-metadata-interval-seconds 1000))))
(defn- save-execution-metadata!
- "Save a `QueryExecution` row containing `execution-info`. Done asynchronously when a query is finished."
+ "Save a `QueryExecution` row containing `execution-info`. Saves are batched and written asynchronously, so they may
+ not be visible for up to [[save-execution-metadata-interval-seconds]] (and may be lost on non-graceful shutdown);
+ this is analytics data, so we consider that an acceptable trade for not paying for an INSERT per query execution.
+ Tests can bind [[qp.util/*execute-async?*]] to `false` to save synchronously on the current thread."
[execution-info]
- (let [;; Capture SDK info now, on the request thread, because `with-execute-async` below intentionally
- ;; does not convey dynamic var bindings to the background thread (to avoid holding DB connections).
- ;; `include-sdk-info` also runs in the `before-insert` hook as a safety net for any code path
- ;; that inserts QueryExecution directly (where dynamic vars would still be bound).
- execution-info' (analytics.core/include-sdk-info execution-info)]
- (qp.util/with-execute-async
- ;; 1. Asynchronously save QueryExecution, update query average execution time etc. using the Agent/pooledExecutor
- ;; pool, which is a fixed pool of size `nthreads + 2`. This way we don't spin up a ton of threads doing unimportant
- ;; background query execution saving (as `future` would do, which uses an unbounded thread pool by default)
- ;;
- ;; 2. This is on purpose! By *not* using `bound-fn` or `future`, any dynamic variables in play when the task is
- ;; submitted, such as `db/*connection*`, won't be in play when the task is actually executed. That way we won't
- ;; attempt to use closed DB connections
- (fn []
- (log/trace "Saving QueryExecution info")
- (try
- (save-execution-metadata!* (add-running-time execution-info'))
- (catch Throwable e
- (log/error e "Error saving query execution info")))))))
+ (let [;; Capture SDK info and compute the running time now, on the query execution thread: by the time the batch is
+ ;; processed the request (and its dynamic bindings and clock) is long gone. `include-sdk-info` also runs in
+ ;; the `before-insert` hook as a safety net for any code path that inserts QueryExecution directly (where
+ ;; dynamic vars would still be bound).
+ execution-info' (add-running-time (analytics.core/include-sdk-info execution-info))]
+ (if qp.util/*execute-async?*
+ (grouper/submit! @save-execution-metadata-queue execution-info')
+ (save-execution-metadata!* [execution-info']))))
(defn- save-successful-execution-metadata! [cache-details is-sandboxed? query-execution result-rows]
(let [qe-map (assoc query-execution
diff --git a/src/metabase/query_processor/preprocess.clj b/src/metabase/query_processor/preprocess.clj
index bb3a215637b..4268f3d1223 100644
--- a/src/metabase/query_processor/preprocess.clj
+++ b/src/metabase/query_processor/preprocess.clj
@@ -31,6 +31,7 @@
[metabase.query-processor.middleware.parameters :as parameters]
[metabase.query-processor.middleware.permissions :as qp.perms]
[metabase.query-processor.middleware.persistence :as qp.persistence]
+ [metabase.query-processor.middleware.prefetch-metadata :as prefetch-metadata]
[metabase.query-processor.middleware.reconcile-breakout-and-order-by-bucketing :as reconcile-bucketing]
[metabase.query-processor.middleware.remove-inactive-field-refs :as qp.remove-inactive-field-refs]
[metabase.query-processor.middleware.resolve-fields :as qp.resolve-fields]
@@ -64,6 +65,7 @@
#'qp.perms/remove-persisted-info-native-keys
#'qp.constraints/maybe-add-default-userland-constraints
#'validate/validate-query
+ #'prefetch-metadata/prefetch-metadata
#'fetch-source-query/resolve-source-cards
#'drop-fields-in-summaries/drop-fields-in-summaries
#'expand-aggregations/expand-aggregations
diff --git a/src/metabase/query_processor/util.clj b/src/metabase/query_processor/util.clj
index 41fc37ec065..558723f0710 100644
--- a/src/metabase/query_processor/util.clj
+++ b/src/metabase/query_processor/util.clj
@@ -98,23 +98,11 @@
col))))
(def ^:dynamic *execute-async?*
- "Used to control `with-execute-async` to whether or not execute its body asynchronously."
+ "Whether to save QueryExecutions (and other post-execution side effects) asynchronously via the batch-processing
+ queue. Bind (or redef, when the query runs on another thread) to `false` in tests to save synchronously on the
+ calling thread."
true)
-(defn do-with-execute-async
- "Impl of `with-execute-async`"
- [thunk]
- (if *execute-async?*
- (.submit clojure.lang.Agent/pooledExecutor ^Runnable thunk)
- (thunk)))
-
-(defmacro with-execute-async
- "Execute body asynchronously in a pooled executor.
-
- Used for side effects during query execution like saving query execution info."
- [thunk]
- `(do-with-execute-async ~thunk))
-
(mu/defn userland-query? :- :boolean
"Returns true if the query is an userland query, else false."
[query :- ::qp.schema/any-query]
diff --git a/src/metabase/view_log/events/view_log.clj b/src/metabase/view_log/events/view_log.clj
index 5df06d7bda9..bb999527165 100644
--- a/src/metabase/view_log/events/view_log.clj
+++ b/src/metabase/view_log/events/view_log.clj
@@ -2,6 +2,7 @@
"This namespace is responsible for subscribing to events which should update the view log and view counts."
(:require
[java-time.api :as t]
+ [metabase.analytics.core :as analytics]
[metabase.api.common :as api]
[metabase.app-db.cluster-lock :as cluster-lock]
[metabase.audit-app.core :as audit]
@@ -64,11 +65,13 @@
(def ^:private increment-view-count-interval-seconds 20)
+(def ^:private increment-view-count-queue-capacity 500)
+
(defonce ^:private
increase-view-count-queue
(delay (grouper/start!
#'increment-view-counts!*
- :capacity 500
+ :capacity increment-view-count-queue-capacity
:interval (* increment-view-count-interval-seconds 1000))))
(defn increment-view-counts!
@@ -76,13 +79,36 @@
[model model-id]
(grouper/submit! @increase-view-count-queue {:model model :id model-id}))
+(def ^:private record-view-interval-seconds 20)
+
+(def ^:private record-view-queue-capacity 500)
+
+(defn- record-views!* [views]
+ (log/debugf "Recording %d views" (count views))
+ (try
+ (t2/insert! :model/ViewLog views)
+ (catch Exception e
+ (log/error e "Failed to record views"))))
+
+(defonce ^:private record-view-queue
+ (delay (grouper/start!
+ #'record-views!*
+ :capacity record-view-queue-capacity
+ :interval (* record-view-interval-seconds 1000))))
+
(mu/defn record-views!
- "Simple base function for recording a view of a given `model` and `model-id` by a certain `user`."
+ "Simple base function for recording a view of a given `model` and `model-id` by a certain `user`. Views are batched
+ and inserted asynchronously, so they may not be visible in the view log for up
+ to [[record-view-interval-seconds]] (plus they are lost on non-graceful shutdown); we consider that an acceptable
+ trade for not paying for a synchronous INSERT on every read/query request."
[view-or-views :- [:or :map [:sequential :map]]]
(span/with-span!
{:name "record-view!"}
(when (premium-features/log-enabled?)
- (t2/insert! :model/ViewLog view-or-views))))
+ (doseq [view (u/one-or-many view-or-views)]
+ (grouper/submit! @record-view-queue (-> view
+ (assoc :timestamp (t/offset-date-time))
+ analytics/include-sdk-info))))))
(defn generate-view
"Generates a view, given an event map. The event map either has an `object` or a `model` and `object-id`."
@@ -116,6 +142,8 @@
(def ^:private update-dashboard-last-viewed-at-interval-seconds 20)
+(def ^:private update-dashboard-last-viewed-at-queue-capacity 500)
+
(defn- update-dashboard-last-viewed-at!* [dashboard-id-timestamps]
(let [dashboard-id->timestamp (update-vals (group-by :id dashboard-id-timestamps)
(fn [xs] (apply t/max (map :timestamp xs))))]
@@ -138,7 +166,7 @@
(def ^:private update-dashboard-last-viewed-at-queue
(delay (grouper/start!
#'update-dashboard-last-viewed-at!*
- :capacity 500
+ :capacity update-dashboard-last-viewed-at-queue-capacity
:interval (* update-dashboard-last-viewed-at-interval-seconds 1000))))
(defn- update-dashboard-last-viewed-at!
diff --git a/test/metabase/agent_api/api_test.clj b/test/metabase/agent_api/api_test.clj
index dad419e4af5..4227e841dd6 100644
--- a/test/metabase/agent_api/api_test.clj
+++ b/test/metabase/agent_api/api_test.clj
@@ -1118,19 +1118,20 @@
(is (string? (some-> resp :via first :error)))))
(testing "A successful query records a QueryExecution audit row tagged :agent"
;; Bypass-of-userland regression (#1): without `prepare-agent-query` no audit row was
- ;; written. Verify a row lands and carries the agent context. The QueryExecution
- ;; insert is async, so poll for the new row.
- (let [before (t2/count :model/QueryExecution)]
- (mt/user-http-request :crowberto :post 202 "agent/v1/execute-sql"
- {:database_id (mt/id)
- :sql "SELECT 1 AS audit_probe"})
- (let [latest (u/poll {:thunk (fn [] (t2/select-one :model/QueryExecution
- {:order-by [[:started_at :desc]]}))
- :done? (fn [_qe] (> (t2/count :model/QueryExecution) before))
- :timeout-ms 5000
- :interval-ms 50})]
- (is (some? latest) "QueryExecution row should be inserted within 5s")
- (is (= :agent (:context latest)))))))
+ ;; written. Verify a row lands and carries the agent context. QueryExecutions are saved in async batches by
+ ;; default, so save them synchronously to be able to assert on them right away.
+ (mt/with-temporary-setting-values [synchronous-batch-updates true]
+ (let [before (t2/count :model/QueryExecution)]
+ (mt/user-http-request :crowberto :post 202 "agent/v1/execute-sql"
+ {:database_id (mt/id)
+ :sql "SELECT 1 AS audit_probe"})
+ (let [latest (u/poll {:thunk (fn [] (t2/select-one :model/QueryExecution
+ {:order-by [[:started_at :desc]]}))
+ :done? (fn [_qe] (> (t2/count :model/QueryExecution) before))
+ :timeout-ms 5000
+ :interval-ms 50})]
+ (is (some? latest) "QueryExecution row should be inserted within 5s")
+ (is (= :agent (:context latest))))))))
;;; ------------------------------------------------- Read Resource Tests -----------------------------------------
diff --git a/test/metabase/documents/view_log_test.clj b/test/metabase/documents/view_log_test.clj
index 763ddd81900..4a63cee4211 100644
--- a/test/metabase/documents/view_log_test.clj
+++ b/test/metabase/documents/view_log_test.clj
@@ -1,14 +1,19 @@
-(ns metabase.documents.view-log-test
+(ns ^:synchronous metabase.documents.view-log-test
(:require
[clojure.test :refer :all]
[java-time.api :as t]
[metabase.documents.view-log :as documents.view-log]
[metabase.events.core :as events]
[metabase.test :as mt]
+ [methodical.core :as methodical]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
+(use-fixtures :each (fn [thunk]
+ (mt/with-temporary-setting-values [synchronous-batch-updates true]
+ (thunk))))
+
(defn latest-view
"Returns the most recent view for a given user and document ID"
[user-id document-id]
@@ -109,13 +114,20 @@
:document "{\"type\":\"doc\",\"content\":[]}"
:creator_id (:id user)}]
(let [events-published (atom [])]
- (mt/with-dynamic-fn-redefs [events/publish-event! (fn [topic event]
- (swap! events-published conj topic)
- event)]
+ (methodical/add-aux-method-with-unique-key!
+ #'events/publish-event! :before :default
+ (fn [topic _event]
+ (swap! events-published conj topic))
+ ::publish-event-spy)
+ (try
(#'documents.view-log/update-document-last-viewed-at!*
[{:id (:id document) :timestamp (t/offset-date-time)}])
(is (not (contains? (set @events-published) :event/document-update))
- "updating last_viewed_at should not publish :event/document-update"))))))
+ "updating last_viewed_at should not publish :event/document-update")
+ (finally
+ (methodical/remove-aux-method-with-unique-key!
+ #'events/publish-event! :before :default
+ ::publish-event-spy)))))))
(deftest document-event-derivation-test
(testing "Document events are properly derived from base events"
diff --git a/test/metabase/eid_translation/util_test.clj b/test/metabase/eid_translation/util_test.clj
index d81ef2e8249..7d1d92778ed 100644
--- a/test/metabase/eid_translation/util_test.clj
+++ b/test/metabase/eid_translation/util_test.clj
@@ -22,30 +22,31 @@
@#'eid-translation.util/api-name->model))))
(deftest ->id-test
- (#'stats/clear-translation-count!)
- (is (= (assoc eid-translation/default-counter :total 0)
- (#'stats/get-translation-count)))
- (mt/with-temp [:model/Card {card-id :id card-eid :entity_id} {}]
- (is (= card-id (eid-translation.util/->id :card card-id)))
- (is (= card-id (eid-translation.util/->id :model/Card card-id)))
- (is (partial= {:ok 0 :total 0} (#'stats/get-translation-count))
- "Translations are not counted when they don't occur")
+ (mt/with-temporary-setting-values [synchronous-batch-updates true]
(#'stats/clear-translation-count!)
- (is (= card-id (eid-translation.util/->id :card card-eid)))
- (is (= card-id (eid-translation.util/->id :model/Card card-eid)))
- (is (partial= {:ok 2 :total 2} (#'stats/get-translation-count))
- "Translations are counted when they do occur")
- (#'stats/clear-translation-count!))
- (let [samples (t2/select-fn->fn :id :entity_id [:model/Card :id :entity_id] {:limit 100})]
- (when (seq samples)
- (doseq [[card-id entity-id] samples]
- (testing (str "card-id: " card-id " entity-id: " entity-id)
- (is (= card-id (eid-translation.util/->id :model/Card card-id)))
- (is (= card-id (eid-translation.util/->id :card card-id)))
- (is (= card-id (eid-translation.util/->id :model/Card entity-id)))
- (is (= card-id (eid-translation.util/->id :card entity-id)))))
- (is (malli= [:map [:ok pos-int?] [:total pos-int?]]
- (#'stats/get-translation-count))))))
+ (is (= (assoc eid-translation/default-counter :total 0)
+ (#'stats/get-translation-count)))
+ (mt/with-temp [:model/Card {card-id :id card-eid :entity_id} {}]
+ (is (= card-id (eid-translation.util/->id :card card-id)))
+ (is (= card-id (eid-translation.util/->id :model/Card card-id)))
+ (is (partial= {:ok 0 :total 0} (#'stats/get-translation-count))
+ "Translations are not counted when they don't occur")
+ (#'stats/clear-translation-count!)
+ (is (= card-id (eid-translation.util/->id :card card-eid)))
+ (is (= card-id (eid-translation.util/->id :model/Card card-eid)))
+ (is (partial= {:ok 2 :total 2} (#'stats/get-translation-count))
+ "Translations are counted when they do occur")
+ (#'stats/clear-translation-count!))
+ (let [samples (t2/select-fn->fn :id :entity_id [:model/Card :id :entity_id] {:limit 100})]
+ (when (seq samples)
+ (doseq [[card-id entity-id] samples]
+ (testing (str "card-id: " card-id " entity-id: " entity-id)
+ (is (= card-id (eid-translation.util/->id :model/Card card-id)))
+ (is (= card-id (eid-translation.util/->id :card card-id)))
+ (is (= card-id (eid-translation.util/->id :model/Card entity-id)))
+ (is (= card-id (eid-translation.util/->id :card entity-id)))))
+ (is (malli= [:map [:ok pos-int?] [:total pos-int?]]
+ (#'stats/get-translation-count)))))))
(deftest ^:parallel entity-id-single-card-translations-test
(mt/with-temp
diff --git a/test/metabase/lib/metadata/composed_provider_test.cljc b/test/metabase/lib/metadata/composed_provider_test.cljc
index 0737d48cc7e..b5ecb9d64d9 100644
--- a/test/metabase/lib/metadata/composed_provider_test.cljc
+++ b/test/metabase/lib/metadata/composed_provider_test.cljc
@@ -54,4 +54,4 @@
(lib.metadata/field mp (meta/id :orders :tax)))))
(testing "Fetching by Name"
(is (=? [{:active false}]
- (lib.metadata.protocols/metadatas mp {:lib/type :metadata/column, :table-id (meta/id :orders), :name #{"TAX"}})))))))
+ (lib.metadata.protocols/metadatas mp {:lib/type :metadata/column, :table-ids #{(meta/id :orders)}, :name #{"TAX"}})))))))
diff --git a/test/metabase/lib/test_metadata/graph_provider.cljc b/test/metabase/lib/test_metadata/graph_provider.cljc
index 543b1d5d1d2..752bf9df058 100644
--- a/test/metabase/lib/test_metadata/graph_provider.cljc
+++ b/test/metabase/lib/test_metadata/graph_provider.cljc
@@ -17,17 +17,17 @@
(:tables metadata-graph)))
(defn- graph-metadatas
- [metadata-graph {metadata-type :lib/type, :keys [table-id], :as metadata-spec}]
+ [metadata-graph {metadata-type :lib/type, :keys [table-ids], :as metadata-spec}]
(let [objects (case metadata-type
:metadata/table (:tables metadata-graph)
- :metadata/column (if table-id
- (:fields (find-table metadata-graph table-id))
+ :metadata/column (if table-ids
+ (mapcat #(:fields (find-table metadata-graph %)) table-ids)
(mapcat :fields (:tables metadata-graph)))
- :metadata/metric (if table-id
- (:metrics (find-table metadata-graph table-id))
+ :metadata/metric (if table-ids
+ (mapcat #(:metrics (find-table metadata-graph %)) table-ids)
(mapcat :metrics (:tables metadata-graph)))
- :metadata/segment (if table-id
- (:segments (find-table metadata-graph table-id))
+ :metadata/segment (if table-ids
+ (mapcat #(:segments (find-table metadata-graph %)) table-ids)
(mapcat :segments (:tables metadata-graph)))
#_else
;; not implemented for the simple graph metadata provider.
diff --git a/test/metabase/lib_be/metadata/jvm_test.clj b/test/metabase/lib_be/metadata/jvm_test.clj
index 4a515671a49..e4c65b8978c 100644
--- a/test/metabase/lib_be/metadata/jvm_test.clj
+++ b/test/metabase/lib_be/metadata/jvm_test.clj
@@ -249,16 +249,16 @@
{:lib/type :metadata/table, :name #{"Table"}}
{:lib/type :metadata/column, :id #{1}}
{:lib/type :metadata/column, :name #{"Field"}}
- {:lib/type :metadata/column, :table-id 1}
+ {:lib/type :metadata/column, :table-ids #{1}}
{:lib/type :metadata/card, :id #{1}}
{:lib/type :metadata/card, :name #{"Card"}}
{:lib/type :metadata/metric, :id #{1}}
{:lib/type :metadata/metric, :name #{"Metric"}}
- {:lib/type :metadata/metric, :table-id 1}
- {:lib/type :metadata/metric, :card-id 1}
+ {:lib/type :metadata/metric, :table-ids #{1}}
+ {:lib/type :metadata/metric, :card-ids #{1}}
{:lib/type :metadata/segment, :id #{1}}
{:lib/type :metadata/segment, :name #{"Segment"}}
- {:lib/type :metadata/segment, :table-id 1}
+ {:lib/type :metadata/segment, :table-ids #{1}}
{:lib/type :metadata/native-query-snippet, :id #{1}}
{:lib/type :metadata/native-query-snippet, :name #{"Snippet"}}))))
diff --git a/test/metabase/lib_metric/filter_test.cljc b/test/metabase/lib_metric/filter_test.cljc
index 1cacea0cbc4..3e506256d11 100644
--- a/test/metabase/lib_metric/filter_test.cljc
+++ b/test/metabase/lib_metric/filter_test.cljc
@@ -823,12 +823,15 @@
(let [wanted (or id #{1 2})]
(filterv #(contains? wanted (:id %)) [seg-source-metric-1 seg-source-metric-2])))
-(defn- seg-segment-fetcher [{:keys [table-id]}]
- (case table-id
- 10 [seg-source-segment]
- 20 [seg-joined-segment]
- 30 [seg-second-source-segment]
- []))
+(defn- seg-segment-fetcher [{table-ids :table-ids}]
+ (into []
+ (mapcat (fn [table-id]
+ (case table-id
+ 10 [seg-source-segment]
+ 20 [seg-joined-segment]
+ 30 [seg-second-source-segment]
+ [])))
+ table-ids))
(def ^:private seg-mock-provider
(lib-metric.metadata.provider/metric-context-metadata-provider
diff --git a/test/metabase/lib_metric/metadata/jvm_test.clj b/test/metabase/lib_metric/metadata/jvm_test.clj
index 4ea1e1b0166..e59ba10468a 100644
--- a/test/metabase/lib_metric/metadata/jvm_test.clj
+++ b/test/metabase/lib_metric/metadata/jvm_test.clj
@@ -40,7 +40,7 @@
:query {:source-table (mt/id :orders)
:aggregation [[:count]]}}}]
(let [mp (metric-jvm/metadata-provider)
- metrics (lib.metadata.protocols/metadatas mp {:lib/type :metadata/metric :table-id (mt/id :orders)})]
+ metrics (lib.metadata.protocols/metadatas mp {:lib/type :metadata/metric :table-ids #{(mt/id :orders)}})]
(is (some #(= (:id metric) (:id %)) metrics))
(is (every? #(= (mt/id :orders) (:table-id %)) metrics))))))
@@ -66,7 +66,7 @@
:query {:source-table (mt/id :orders)
:aggregation [[:count]]}}}]
(let [mp (metric-jvm/metadata-provider)
- metrics (lib.metadata.protocols/metadatas mp {:lib/type :metadata/metric :table-id (mt/id :orders)})]
+ metrics (lib.metadata.protocols/metadatas mp {:lib/type :metadata/metric :table-ids #{(mt/id :orders)}})]
(is (some #(= (:id active-metric) (:id %)) metrics))
(is (not-any? #(= (:id archived-metric) (:id %)) metrics))))))
@@ -90,7 +90,7 @@
(deftest ^:parallel metadatas-routes-columns-to-database-provider-test
(testing "metadatas should route column requests to the appropriate database provider"
(let [mp (metric-jvm/metadata-provider)
- columns (lib.metadata.protocols/metadatas mp {:lib/type :metadata/column :table-id (mt/id :orders)})]
+ columns (lib.metadata.protocols/metadatas mp {:lib/type :metadata/column :table-ids #{(mt/id :orders)}})]
(is (seq columns))
(is (every? #(= (mt/id :orders) (:table-id %)) columns)))))
diff --git a/test/metabase/lib_metric/metadata/provider_test.cljc b/test/metabase/lib_metric/metadata/provider_test.cljc
index 1ea975f4e17..e24ddf966aa 100644
--- a/test/metabase/lib_metric/metadata/provider_test.cljc
+++ b/test/metabase/lib_metric/metadata/provider_test.cljc
@@ -37,7 +37,7 @@
(defn- mock-metric-fetcher
"Mock metric fetcher that filters mock-metrics based on spec."
- [{id-set :id, name-set :name, :keys [table-id card-id], :as _spec}]
+ [{id-set :id, name-set :name, :keys [table-ids card-ids], :as _spec}]
(let [active-only? (not (or id-set name-set))]
(into []
(comp
@@ -47,12 +47,12 @@
(if name-set
(filter #(contains? name-set (:name %)))
identity)
- (if table-id
- (filter #(and (= (:table-id %) table-id)
+ (if table-ids
+ (filter #(and (contains? table-ids (:table-id %))
(nil? (:source-card-id %))))
identity)
- (if card-id
- (filter #(= (:source-card-id %) card-id))
+ (if card-ids
+ (filter #(contains? card-ids (:source-card-id %)))
identity)
(if active-only?
(filter #(not (:archived %)))
@@ -69,7 +69,7 @@
(reify lib.metadata.protocols/MetadataProvider
(database [_this]
{:id db-id :name (str "DB-" db-id) :lib/type :metadata/database})
- (metadatas [_this {metadata-type :lib/type, id-set :id, :keys [table-id], :as _spec}]
+ (metadatas [_this {metadata-type :lib/type, id-set :id, :keys [table-ids], :as _spec}]
(case metadata-type
:metadata/table
(if id-set
@@ -81,8 +81,8 @@
(filter #(= (:db-id %) db-id))
(vals mock-tables)))
:metadata/column
- (when table-id
- (get mock-columns table-id []))
+ (when table-ids
+ (into [] (mapcat #(get mock-columns % [])) table-ids))
[]))
(setting [_this k]
(get mock-settings k))))
@@ -122,7 +122,7 @@
(deftest ^:parallel metadatas-fetches-metrics-by-table-id-test
(testing "metadatas should fetch metrics by table-id (excludes card-based metrics)"
(let [mp (create-mock-provider)
- metrics (lib.metadata.protocols/metadatas mp {:lib/type :metadata/metric :table-id 10})]
+ metrics (lib.metadata.protocols/metadatas mp {:lib/type :metadata/metric :table-ids #{10}})]
(is (= 2 (count metrics)))
(is (every? #(= 10 (:table-id %)) metrics))
(is (every? #(nil? (:source-card-id %)) metrics)))))
@@ -130,7 +130,7 @@
(deftest ^:parallel metadatas-fetches-metrics-by-card-id-test
(testing "metadatas should fetch metrics by card-id"
(let [mp (create-mock-provider)
- metrics (lib.metadata.protocols/metadatas mp {:lib/type :metadata/metric :card-id 100})]
+ metrics (lib.metadata.protocols/metadatas mp {:lib/type :metadata/metric :card-ids #{100}})]
(is (= 1 (count metrics)))
(is (= 100 (:source-card-id (first metrics)))))))
@@ -145,7 +145,7 @@
(deftest ^:parallel metadatas-routes-columns-to-db-provider-test
(testing "metadatas should route column requests to the appropriate database provider"
(let [mp (create-mock-provider)
- columns (lib.metadata.protocols/metadatas mp {:lib/type :metadata/column :table-id 10})]
+ columns (lib.metadata.protocols/metadatas mp {:lib/type :metadata/column :table-ids #{10}})]
(is (= 2 (count columns)))
(is (every? #(= 10 (:table-id %)) columns)))))
diff --git a/test/metabase/metrics/api_test.clj b/test/metabase/metrics/api_test.clj
index 55e6b2f9074..f45623566af 100644
--- a/test/metabase/metrics/api_test.clj
+++ b/test/metabase/metrics/api_test.clj
@@ -4,7 +4,6 @@
[clojure.test :refer :all]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
- [metabase.query-processor.util :as qp.util]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[toucan2.core :as t2]))
@@ -350,7 +349,7 @@
(deftest dataset-leaf-records-query-execution-test
(testing "POST /api/metric/dataset (leaf path) writes a QueryExecution row with :context :metric"
(mt/test-helpers-set-global-values!
- (binding [qp.util/*execute-async?* false]
+ (mt/with-temporary-setting-values [synchronous-batch-updates true]
(mt/with-temp [:model/Card metric {:name "QE Leaf Metric"
:type :metric
:dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}]
@@ -366,7 +365,7 @@
(deftest dataset-arithmetic-records-one-qe-per-leaf-test
(testing "POST /api/metric/dataset (arithmetic) writes one QueryExecution row per leaf"
(mt/test-helpers-set-global-values!
- (binding [qp.util/*execute-async?* false]
+ (mt/with-temporary-setting-values [synchronous-batch-updates true]
(mt/with-temp [:model/Card metric-a {:name "QE Arith A"
:type :metric
:dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}
@@ -388,7 +387,7 @@
(deftest breakout-values-records-query-execution-test
(testing "POST /api/metric/breakout-values writes a QueryExecution row with :context :metric"
(mt/test-helpers-set-global-values!
- (binding [qp.util/*execute-async?* false]
+ (mt/with-temporary-setting-values [synchronous-batch-updates true]
(mt/with-temp [:model/Card metric {:name "QE Breakout Metric"
:type :metric
:dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}]
diff --git a/test/metabase/queries_rest/api/card_test.clj b/test/metabase/queries_rest/api/card_test.clj
index 3ceabf56cec..5a2e5607e73 100644
--- a/test/metabase/queries_rest/api/card_test.clj
+++ b/test/metabase/queries_rest/api/card_test.clj
@@ -38,7 +38,6 @@
[metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.middleware.constraints :as qp.constraints]
[metabase.query-processor.pivot.test-util :as api.pivots]
- [metabase.query-processor.util :as qp.util]
[metabase.revisions.models.revision :as revision]
[metabase.test :as mt]
[metabase.test.data.users :as test.users]
@@ -308,16 +307,18 @@
version-string (str "1." (rand-int 1000) "." (rand-int 1000))]
(mt/with-temp [:model/Card card-1 {:name "Card 1"
:dataset_query (mt/mbql-query venues {:limit 1})}]
- (mt/with-premium-features #{:audit-app}
- (mt/user-http-request :crowberto :post 202 (str "card/" (u/the-id card-1) "/query")
- {:request-options {:headers {"x-metabase-client" client-string
- "x-metabase-client-version" version-string}}}))
+ (mt/with-temporary-setting-values [synchronous-batch-updates true]
+ (mt/with-premium-features #{:audit-app}
+ (mt/user-http-request :crowberto :post 202 (str "card/" (u/the-id card-1) "/query")
+ {:request-options {:headers {"x-metabase-client" client-string
+ "x-metabase-client-version" version-string}}})))
(is (= {:embedding_client client-string, :embedding_sdk_version version-string}
(t2/select-one [:model/ViewLog :embedding_client :embedding_sdk_version] :model "card" :model_id (u/the-id card-1))))))))
(deftest embedding-sdk-info-saves-query-execution-test
(testing "GET /api/card with embedding headers set"
- (binding [qp.util/*execute-async?* false]
+ ;; QueryExecutions are saved in async batches by default; save them synchronously so we can assert on them
+ (mt/with-temporary-setting-values [synchronous-batch-updates true]
(mt/with-temp [:model/Card card-1 {:name "Card 1"
;; This query is just to make sure the card actually runs a query, otherwise
;; there won't be a QueryExecution record to check!
diff --git a/test/metabase/query_processor/api_test.clj b/test/metabase/query_processor/api_test.clj
index 2741b16fca3..56dd7227422 100644
--- a/test/metabase/query_processor/api_test.clj
+++ b/test/metabase/query_processor/api_test.clj
@@ -84,115 +84,118 @@
(deftest basic-query-test
(testing "POST /api/dataset"
(testing "\nJust a basic sanity check to make sure Query Processor endpoint is still working correctly."
- (let [query (mt/mbql-query checkins
- {:aggregation [[:count]]})
- result (mt/user-http-request :crowberto :post 202 "dataset" query)]
- (testing "\nAPI Response"
- (is (=?
- {:data {:rows [[1000]]
- :cols [(mt/obj->json->obj (qp.test-util/aggregate-col :count))]
- :native_form true
- :results_timezone "UTC"}
- :row_count 1
- :status "completed"
- :context "ad-hoc"
- :json_query (-> (mt/mbql-query checkins
- {:aggregation [[:count]]})
- (assoc-in [:query :aggregation] [["count"]])
- (assoc :type "query")
- (merge query-defaults))
- :started_at true
- :running_time true
- :average_execution_time nil
- :database_id (mt/id)}
- (format-response result))))
- (testing "\nSaved QueryExecution"
- (is (= {:hash true
- :row_count 1
- :result_rows 1
- :context :ad-hoc
- :executor_id (mt/user->id :crowberto)
- :native false
- :pulse_id nil
- :card_id nil
- :is_sandboxed false
- :dashboard_id nil
- :transform_id nil
- :lens_id nil
- :lens_params nil
- :error nil
- :id true
- :action_id nil
- :cache_hit false
- :cache_hash false
- :parameterized false
- :database_id (mt/id)
- :started_at true
- :running_time true
- :embedding_client nil
- :embedding_hostname nil
- :embedding_path nil
- :embedding_route nil
- :embedding_sdk_version nil
- :metabase_version true
- :auth_method true
- :ip_address nil
- :is_db_routed false
- :is_impersonated false
- :parameters nil
- :tenant_id nil
- :user_agent nil
- :sanitized_user_agent nil}
- (format-response (most-recent-query-execution-for-query query)))))))))
+ (mt/with-temporary-setting-values [synchronous-batch-updates true]
+ (let [query (mt/mbql-query checkins
+ {:aggregation [[:count]]})
+ result (mt/user-http-request :crowberto :post 202 "dataset" query)]
+ (testing "\nAPI Response"
+ (is (=?
+ {:data {:rows [[1000]]
+ :cols [(mt/obj->json->obj (qp.test-util/aggregate-col :count))]
+ :native_form true
+ :results_timezone "UTC"}
+ :row_count 1
+ :status "completed"
+ :context "ad-hoc"
+ :json_query (-> (mt/mbql-query checkins
+ {:aggregation [[:count]]})
+ (assoc-in [:query :aggregation] [["count"]])
+ (assoc :type "query")
+ (merge query-defaults))
+ :started_at true
+ :running_time true
+ :average_execution_time nil
+ :database_id (mt/id)}
+ (format-response result))))
+ (testing "\nSaved QueryExecution"
+ (is (= {:hash true
+ :row_count 1
+ :result_rows 1
+ :context :ad-hoc
+ :executor_id (mt/user->id :crowberto)
+ :native false
+ :pulse_id nil
+ :card_id nil
+ :is_sandboxed false
+ :dashboard_id nil
+ :transform_id nil
+ :lens_id nil
+ :lens_params nil
+ :error nil
+ :id true
+ :action_id nil
+ :cache_hit false
+ :cache_hash false
+ :parameterized false
+ :database_id (mt/id)
+ :started_at true
+ :running_time true
+ :embedding_client nil
+ :embedding_hostname nil
+ :embedding_path nil
+ :embedding_route nil
+ :embedding_sdk_version nil
+ :metabase_version true
+ :auth_method true
+ :ip_address nil
+ :is_db_routed false
+ :is_impersonated false
+ :parameters nil
+ :tenant_id nil
+ :user_agent nil
+ :sanitized_user_agent nil}
+ (format-response (most-recent-query-execution-for-query query))))))))))
(deftest failure-test
;; clear out recent query executions!
(t2/delete! :model/QueryExecution)
(testing "POST /api/dataset"
(testing "\nA failed query should return a 400 response from the API"
- ;; Error message's format can differ a bit depending on DB version and the comment we prepend to it, so check
- ;; that it exists and contains the substring "Syntax error in SQL statement"
- (let [query {:database (mt/id)
- :type "native"
- :native {:query "foobar"}}
- result (mt/user-http-request :crowberto :post 400 "dataset" query)]
- (testing "\nAPI Response"
- (is (malli= [:map
- [:data [:map
- [:rows [:= []]]
- [:cols [:= []]]]]
- [:row_count [:= 0]]
- [:status [:= "failed"]]
- [:context [:= "ad-hoc"]]
- [:error #"Syntax error in SQL statement"]
- [:json_query [:map
- [:database [:= (mt/id)]]
- [:type [:= "native"]]
- [:native [:map
- [:query [:= "foobar"]]]]
- [:middleware [:map
- [:add-default-userland-constraints? [:= true]]
- [:js-int-to-string? [:= true]]]]]]
- [:database_id [:= (mt/id)]]
- [:state [:= "42000"]]
- [:class [:= "class org.h2.jdbc.JdbcSQLSyntaxErrorException"]]]
- result)))
- (testing "\nSaved QueryExecution"
- (is (malli=
- [:map
- [:hash (ms/InstanceOfClass (Class/forName "[B"))]
- [:id ms/PositiveInt]
- [:result_rows [:= 0]]
- [:row_count [:= 0]]
- [:context [:= :ad-hoc]]
- [:error #"Syntax error in SQL statement"]
- [:database_id [:= (mt/id)]]
- [:executor_id [:= (mt/user->id :crowberto)]]
- [:native [:= true]]
- [:pulse_id nil?]
- [:card_id nil?]
- [:dashboard_id nil?]]
- (most-recent-query-execution-for-query query))))))))
+ ;; QueryExecutions are saved in async batches by default; save them synchronously so we can assert on them
+ (mt/with-temporary-setting-values [synchronous-batch-updates true]
+ ;; Error message's format can differ a bit depending on DB version and the comment we prepend to it, so check
+ ;; that it exists and contains the substring "Syntax error in SQL statement"
+ (let [query {:database (mt/id)
+ :type "native"
+ :native {:query "foobar"}}
+ result (mt/user-http-request :crowberto :post 400 "dataset" query)]
+ (testing "\nAPI Response"
+ (is (malli= [:map
+ [:data [:map
+ [:rows [:= []]]
+ [:cols [:= []]]]]
+ [:row_count [:= 0]]
+ [:status [:= "failed"]]
+ [:context [:= "ad-hoc"]]
+ [:error #"Syntax error in SQL statement"]
+ [:json_query [:map
+ [:database [:= (mt/id)]]
+ [:type [:= "native"]]
+ [:native [:map
+ [:query [:= "foobar"]]]]
+ [:middleware [:map
+ [:add-default-userland-constraints? [:= true]]
+ [:js-int-to-string? [:= true]]]]]]
+ [:database_id [:= (mt/id)]]
+ [:state [:= "42000"]]
+ [:class [:= "class org.h2.jdbc.JdbcSQLSyntaxErrorException"]]]
+ result)))
+ (testing "\nSaved QueryExecution"
+ (is (malli=
+ [:map
+ [:hash (ms/InstanceOfClass (Class/forName "[B"))]
+ [:id ms/PositiveInt]
+ [:result_rows [:= 0]]
+ [:row_count [:= 0]]
+ [:context [:= :ad-hoc]]
+ [:error #"Syntax error in SQL statement"]
+ [:database_id [:= (mt/id)]]
+ [:executor_id [:= (mt/user->id :crowberto)]]
+ [:native [:= true]]
+ [:pulse_id nil?]
+ [:card_id nil?]
+ [:dashboard_id nil?]]
+ (most-recent-query-execution-for-query query)))))))))
(defn- test-download-response-headers
[url]
diff --git a/test/metabase/query_processor/middleware/cache_test.clj b/test/metabase/query_processor/middleware/cache_test.clj
index 0f2ce2c1eae..213b05a2d90 100644
--- a/test/metabase/query_processor/middleware/cache_test.clj
+++ b/test/metabase/query_processor/middleware/cache_test.clj
@@ -104,7 +104,8 @@
(defn do-with-mock-cache! [f]
(mt/with-open-channels [save-chan (a/chan 10)
purge-chan (a/chan 10)]
- (mt/with-temporary-setting-values [query-caching-max-ttl 60]
+ (mt/with-temporary-setting-values [query-caching-max-ttl 60
+ synchronous-batch-updates true]
(binding [cache/*backend* (test-backend save-chan purge-chan)
*save-chan* save-chan
*purge-chan* purge-chan]
@@ -420,17 +421,17 @@
update-avg-execution-count (atom 0)
called-promise (promise)
save-execution-metadata-original (mt/original-fn #'process-userland-query/save-execution-metadata!*)
- save-query-update-avg-time-original (mt/original-fn #'query/save-query-and-update-average-execution-time!)]
- ;; save-execution-metadata!* and save-query-and-update-average-execution-time! are invoked from
+ save-query-update-avg-time-original (mt/original-fn #'query/save-queries-and-update-average-execution-times!)]
+ ;; save-execution-metadata!* and save-queries-and-update-average-execution-times! are invoked from
;; the QP pipeline on worker threads that don't inherit *local-redefs* — use with-redefs.
#_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}
- (with-redefs [process-userland-query/save-execution-metadata!* (fn [& args]
- (swap! save-execution-metadata-count inc)
- (apply save-execution-metadata-original args)
- (deliver called-promise true))
- query/save-query-and-update-average-execution-time! (fn [& args]
- (swap! update-avg-execution-count inc)
- (apply save-query-update-avg-time-original args))]
+ (with-redefs [process-userland-query/save-execution-metadata!* (fn [& args]
+ (swap! save-execution-metadata-count inc)
+ (apply save-execution-metadata-original args)
+ (deliver called-promise true))
+ query/save-queries-and-update-average-execution-times! (fn [entries]
+ (swap! update-avg-execution-count + (count entries))
+ (save-query-update-avg-time-original entries))]
(let [query (assoc (mt/mbql-query venues {:order-by [[:asc $id]] :limit 42})
:cache-strategy (assoc (ttl-strategy) :multiplier 5000))
q-hash (qp.util/query-hash query)]
diff --git a/test/metabase/query_processor/middleware/prefetch_metadata_test.clj b/test/metabase/query_processor/middleware/prefetch_metadata_test.clj
new file mode 100644
index 00000000000..2839e906ca5
--- /dev/null
+++ b/test/metabase/query_processor/middleware/prefetch_metadata_test.clj
@@ -0,0 +1,128 @@
+(ns metabase.query-processor.middleware.prefetch-metadata-test
+ ;; these tests count real app-DB calls, so they have to use the real application-database metadata provider (and
+ ;; `with-temp` for source cards) rather than a mock metadata provider
+ {:clj-kondo/config '{:linters {:discouraged-var {metabase.test/with-temp {:level :off}
+ toucan2.tools.with-temp/with-temp {:level :off}}}}}
+ (:require
+ [clojure.test :refer :all]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.query-processor.preprocess :as qp.preprocess]
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(defn- preprocess-app-db-call-count
+ "Preprocess `query` and return the number of app-DB calls made. Binds a fresh metadata-provider cache, like the REST
+ API does for each HTTP request."
+ [query]
+ (lib-be/with-metadata-provider-cache
+ (t2/with-call-count [call-count]
+ (qp.preprocess/preprocess (dissoc query :lib/metadata))
+ (call-count))))
+
+(deftest prefetch-metadata-mbql-one-table-call-count-test
+ (mt/dataset test-data
+ (let [mp (mt/metadata-provider)
+ query (-> (lib/query mp (lib.metadata/table mp (mt/id :venues)))
+ (lib/limit 1))]
+ ;; the baseline:
+ ;; - fetch the Database
+ ;; - fetch the Tables in bulk
+ ;; - fetch the Tables' columns in bulk
+ ;; - the permissions check (EE only)
+ ;; - the database-routing check (EE only)
+ ;; - the table-remapping check (EE only)
+ (is (<= (preprocess-app-db-call-count query) 6)))))
+
+(deftest prefetch-metadata-mbql-three-tables-call-count-test
+ (mt/dataset test-data
+ (let [mp (mt/metadata-provider)
+ query (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))
+ (lib/join (lib/join-clause (lib.metadata/table mp (mt/id :products))
+ [(lib/= (lib.metadata/field mp (mt/id :orders :product_id))
+ (lib.metadata/field mp (mt/id :products :id)))]))
+ (lib/join (lib/join-clause (lib.metadata/table mp (mt/id :people))
+ [(lib/= (lib.metadata/field mp (mt/id :orders :user_id))
+ (lib.metadata/field mp (mt/id :people :id)))]))
+ (lib/limit 1))]
+ ;; the same number of calls as for a single-table query: all the referenced tables and their columns are
+ ;; bulk-loaded, so the count does not depend on the number of tables
+ (is (<= (preprocess-app-db-call-count query) 6)))))
+
+(deftest prefetch-metadata-native-one-field-filter-call-count-test
+ (mt/dataset test-data
+ (let [mp (mt/metadata-provider)
+ query (-> (lib/native-query mp "SELECT * FROM VENUES WHERE {{ff1}}")
+ (lib/with-template-tags
+ {"ff1" {:type :dimension
+ :name "ff1"
+ :display-name "ff1"
+ :id "id1"
+ :dimension (lib/ref (lib.metadata/field mp (mt/id :venues :price)))
+ :widget-type :id}})
+ (assoc :parameters [{:type :id
+ :target [:dimension [:template-tag "ff1"]]
+ :value [1]}]))]
+ ;; one more call than for MBQL queries: the fields referenced by the template tags are bulk-loaded by ID, which
+ ;; is also how their tables are discovered
+ (is (<= (preprocess-app-db-call-count query) 7)))))
+
+(deftest prefetch-metadata-native-three-field-filters-call-count-test
+ (mt/dataset test-data
+ (let [mp (mt/metadata-provider)
+ tag (fn [tag-name field-id]
+ {:type :dimension
+ :name tag-name
+ :display-name tag-name
+ :id tag-name
+ :dimension (lib/ref (lib.metadata/field mp field-id))
+ :widget-type :id})
+ query (-> (lib/native-query mp "SELECT * FROM VENUES WHERE {{ff1}} AND {{ff2}} AND {{ff3}}")
+ (lib/with-template-tags
+ {"ff1" (tag "ff1" (mt/id :venues :price))
+ "ff2" (tag "ff2" (mt/id :venues :category_id))
+ "ff3" (tag "ff3" (mt/id :venues :name))})
+ (assoc :parameters [{:type :id
+ :target [:dimension [:template-tag "ff1"]]
+ :value [1]}]))]
+ ;; the same calls as with a single field filter: all the template-tag Fields are loaded with a single bulk
+ ;; call, while without prefetching parameter substitution would resolve them one at a time
+ (is (<= (preprocess-app-db-call-count query) 7)))))
+
+(deftest prefetch-metadata-card-source-call-count-test
+ (mt/dataset test-data
+ (let [mp (mt/metadata-provider)]
+ (mt/with-temp [:model/Card card {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :venues)))}]
+ (let [query (-> (lib/query mp (lib.metadata/card mp (:id card)))
+ (lib/limit 1))]
+ ;; the baseline calls, plus:
+ ;; - one bulk fetch of the referenced Cards; resolving the source card afterwards hits the cache, and the
+ ;; card's table and columns are folded into the baseline bulk loads
+ (is (<= (preprocess-app-db-call-count query) 7)))))))
+
+(deftest prefetch-metadata-nested-card-source-call-count-test
+ (mt/dataset test-data
+ (let [mp (mt/metadata-provider)]
+ (mt/with-temp [:model/Card inner-card {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :venues)))}]
+ (mt/with-temp [:model/Card outer-card {:dataset_query (lib/query mp (lib.metadata/card mp (:id inner-card)))}]
+ (let [query (-> (lib/query mp (lib.metadata/card mp (:id outer-card)))
+ (lib/limit 1))]
+ ;; one more call than for a single source card: referenced Cards are prefetched with one bulk fetch per
+ ;; level of nesting, and both cards' tables and columns are folded into the same bulk loads
+ (is (<= (preprocess-app-db-call-count query) 8))))))))
+
+(deftest prefetch-metadata-card-source-with-join-call-count-test
+ (mt/dataset test-data
+ (let [mp (mt/metadata-provider)]
+ (mt/with-temp [:model/Card card {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :venues)))}]
+ (let [query (-> (lib/query mp (lib.metadata/card mp (:id card)))
+ (lib/join (lib/join-clause (lib.metadata/table mp (mt/id :categories))
+ [(lib/= (lib.metadata/field mp (mt/id :venues :category_id))
+ (lib.metadata/field mp (mt/id :categories :id)))]))
+ (lib/limit 1))]
+ ;; the same calls as for a plain source card: the joined Table and its columns are folded into the same
+ ;; bulk loads
+ (is (<= (preprocess-app-db-call-count query) 7)))))))
diff --git a/test/metabase/query_processor/middleware/process_userland_query_test.clj b/test/metabase/query_processor/middleware/process_userland_query_test.clj
index e6bac3abdbb..b63b03dcb83 100644
--- a/test/metabase/query_processor/middleware/process_userland_query_test.clj
+++ b/test/metabase/query_processor/middleware/process_userland_query_test.clj
@@ -28,13 +28,17 @@
result (promise)]
;; save-execution-metadata!* is invoked from the QP pipeline transducer, which runs on a thread
;; that doesn't inherit *local-redefs* — use with-redefs so worker threads see the replacement.
+ ;; `*execute-async?*` is redef'd (not bound) for the same reason: it makes the save happen synchronously
+ ;; on whatever thread runs the query instead of going through the (batched, async) grouper queue.
#_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}
- (with-redefs [process-userland-query/save-execution-metadata!*
- (fn [query-execution]
- (when-let [^bytes qe-hash (:hash query-execution)]
+ (with-redefs [qp.util/*execute-async?* false
+ process-userland-query/save-execution-metadata!*
+ (fn [query-executions]
+ (doseq [{qe-hash :hash, :as query-execution} query-executions
+ :when qe-hash]
(deliver
result
- (if (java.util.Arrays/equals qe-hash original-hash)
+ (if (java.util.Arrays/equals ^bytes qe-hash original-hash)
query-execution
;; if you're seeing this there is probably some
;; bug that is causing query hashes to get
@@ -44,7 +48,7 @@
{:query query
:original-hash (some-> original-hash codecs/bytes->hex)
:query-execution query-execution
- :query-execution-hash (some-> qe-hash codecs/bytes->hex)
+ :query-execution-hash (some-> ^bytes qe-hash codecs/bytes->hex)
:query-execution-query (:json_query query-execution)})))))]
(run
(fn qe-result* []
diff --git a/test/metabase/view_log/events/view_log_test.clj b/test/metabase/view_log/events/view_log_test.clj
index 87dd520abfe..3d38461c21f 100644
--- a/test/metabase/view_log/events/view_log_test.clj
+++ b/test/metabase/view_log/events/view_log_test.clj
@@ -1,4 +1,4 @@
-(ns metabase.view-log.events.view-log-test
+(ns ^:synchronous metabase.view-log.events.view-log-test
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.view-log.events.view-log-test]}}}}}}
(:require
[clojure.test :refer :all]
@@ -22,6 +22,10 @@
(set! *warn-on-reflection* true)
+(use-fixtures :each (fn [thunk]
+ (mt/with-temporary-setting-values [synchronous-batch-updates true]
+ (thunk))))
+
(defn latest-view
"Returns the most recent view for a given user and model ID"
[user-id model-id]