The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Improve app db connection pool usage with public dashboards (#75770)
Alexander Polyankin5d1dd5c03218
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 0c17f844756..3d1f36b7ef6 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -894,6 +894,7 @@metabase.eid-translation.coremetabase.eid-translation.init}:uses #{api+ batch-processingsettingsutil}:model-imports #{:model/Actiondiff --git a/enterprise/backend/src/metabase_enterprise/checker/provider.clj b/enterprise/backend/src/metabase_enterprise/checker/provider.cljindex 9812cffbd0f..11fc71263fb 100644--- a/enterprise/backend/src/metabase_enterprise/checker/provider.clj+++ b/enterprise/backend/src/metabase_enterprise/checker/provider.clj@@ -299,7 +299,7 @@(when-let [data (store/load-database! store db-name)](->database-metadata data))))- (metadatas [_this {:keys [lib/type id table-id name]}]+ (metadatas [_this {:keys [lib/type id table-ids name]}](case type:metadata/table(vec@@ -329,13 +329,14 @@:when data](->field-metadata (make-import-resolver store nil) data))- table-id- (let [table-path (store/id->ref store :table table-id)]- (when table-path- (for [fpath (store/fields-for-table store table-path)- :let [data (store/load-field! store fpath)]- :when data]- (->field-metadata (make-import-resolver store nil) data))))+ table-ids+ (for [table-id table-ids+ :let [table-path (store/id->ref store :table table-id)]+ :when table-path+ fpath (store/fields-for-table store table-path)+ :let [data (store/load-field! store fpath)]+ :when data]+ (->field-metadata (make-import-resolver store nil) data)):else(for [path (store/all-field-paths store)diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj b/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.cljindex e01ae56e446..33343dc604a 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj@@ -23,7 +23,7 @@(defn- metadatas [delegate overrides {metadata-type :lib/typesearch-name :name- :keys [id card-id table-id]+ :keys [id card-ids table-ids]:as metadata-spec}](let [type-overrides (get overrides metadata-type)metadata-keys (set (or id search-name))]@@ -56,13 +56,23 @@(into entities (lib.metadata.protocols/metadatas delegate metadata-spec))(m/distinct-by :id entities))- ;; For columns, return overrides if they're present.+ ;; For columns, return overrides if they're present. `:card-ids` and `:table-ids` are sets of ids; entities+ ;; with overrides use the overridden columns, the rest are delegated.(= metadata-type :metadata/column)- (if-let [overridden-columns (cond- card-id (get-in overrides [::card-columns card-id])- table-id (get-in overrides [::table-columns table-id]))]- @overridden-columns- (lib.metadata.protocols/metadatas delegate metadata-spec))+ (let [[override-k spec-k ids] (cond+ card-ids [::card-columns :card-ids card-ids]+ table-ids [::table-columns :table-ids table-ids])]+ (if-not override-k+ (lib.metadata.protocols/metadatas delegate metadata-spec)+ (let [id->overridden (into {}+ (keep (fn [id]+ (when-let [columns (get-in overrides [override-k id])]+ [id columns])))+ ids)+ missing-ids (into #{} (remove id->overridden) ids)]+ (concat (mapcat (comp deref val) id->overridden)+ (when (seq missing-ids)+ (lib.metadata.protocols/metadatas delegate (assoc metadata-spec spec-k missing-ids))))))):else (throw (ex-info "Unknown :lib/metadata type for OverridingMetadataProvider" metadata-spec)))))@@ -124,7 +134,7 @@;; should be ignored.{[:metadata/card id] (delay (let [temp (merge (when id(-> (inner-mp mp)- (lib.metadata/card id)))+ (lib.metadata.protocols/card id)))updates)result-metadata (or (:result-metadata updates)(and (.returned-columns-fn mp)diff --git a/src/metabase/activity_feed/events/recent_views.clj b/src/metabase/activity_feed/events/recent_views.cljindex 58038ff81de..7e4af0d85cc 100644--- a/src/metabase/activity_feed/events/recent_views.clj+++ b/src/metabase/activity_feed/events/recent_views.clj@@ -48,10 +48,11 @@"Handle processing for a single card query event."[topic {:keys [card-id user-id context] :as _event}](try- (let [user-id (or user-id api/*current-user-id*)]+ (let [user-id (or user-id api/*current-user-id*)];; we don't want to count pinned card views or document cards- (when-not (or (#{:collection :dashboard :dashboard-subscription} context)- (some? (t2/select-one-fn :document_id :model/Card :id card-id)))+ (when (and user-id+ (not (#{:collection :dashboard :dashboard-subscription} context))+ (not (t2/select-one-fn :document_id :model/Card :id card-id)))(recent-views/update-users-recent-views! user-id :model/Card card-id :view)))(catch Throwable e(log/warnf e "Failed to process recent_views event: %s" topic))))@@ -74,15 +75,14 @@(m/defmethod events/publish-event! ::legacy-card-event"Handle recent-view processing for card reads"[topic {:keys [object-id user-id context]}]- ;; Cards can be read indirectly, either through a pinned collection, or in a dashboard.- ;; We only want to count direct views of cards, so we skip processing for indirect views here.- ;; Also skip document cards.- (when (and (= context :question)- (nil? (t2/select-one-fn :document_id :model/Card :id object-id)))- (try- (recent-views/update-users-recent-views! (or user-id api/*current-user-id*) :model/Card object-id :view)- (catch Throwable e- (log/warnf e "Failed to process recent_views event: %s" topic)))))+ (let [user-id (or user-id api/*current-user-id*)]+ (when (and user-id+ (= context :question)+ (nil? (t2/select-one-fn :document_id :model/Card :id object-id)))+ (try+ (recent-views/update-users-recent-views! user-id :model/Card object-id :view)+ (catch Throwable e+ (log/warnf e "Failed to process recent_views event: %s" topic))))))(derive ::collection-touch-event :metabase/event)(derive :event/collection-touch ::collection-touch-event)diff --git a/src/metabase/dashboards/models/dashboard.clj b/src/metabase/dashboards/models/dashboard.cljindex 598779b5f13..d61f51cc05f 100644--- a/src/metabase/dashboards/models/dashboard.clj+++ b/src/metabase/dashboards/models/dashboard.clj@@ -370,8 +370,14 @@[:name ms/NonBlankString][:mappings [:maybe [:set ::parameters.schema/parameter-mapping]]]])-(mu/defn- dashboard->resolved-params :- [:map-of ms/NonBlankString ParamWithMapping]- [dashboard :- [:map [:parameters [:maybe [:sequential :map]]]]]+(mu/defn dashboard->resolved-params :- [:map-of ms/NonBlankString ParamWithMapping]+ "Return map of Dashboard parameter key -> param with resolved `:mappings` (see the `:resolved-params` hydration+ below for an example). Callers that only need the mappings (e.g. the QP) can pass slim dashcards instead of paying+ for the full hydration."+ [dashboard :- [:map+ [:parameters [:maybe [:sequential :map]]]+ [:dashcards [:maybe [:sequential [:map+ [:parameter_mappings [:maybe [:sequential :map]]]]]]]]](let [param-key->mappings (applymerge-with set/union(for [dashcard (:dashcards dashboard)diff --git a/src/metabase/eid_translation/util.clj b/src/metabase/eid_translation/util.cljindex 8862dc1fdec..2f11dc5c68d 100644--- a/src/metabase/eid_translation/util.clj+++ b/src/metabase/eid_translation/util.clj@@ -2,6 +2,7 @@(:require[malli.core :as mc][malli.error :as me]+ [metabase.batch-processing.core :as grouper][metabase.eid-translation.impl :as eid-translation][metabase.eid-translation.settings :as eid-translation.settings][metabase.util :as u]@@ -71,12 +72,29 @@;; -------------------- Entity Id Translation Analytics --------------------+(def ^:private update-translation-count-interval-seconds (* 5 60))++(def ^:private update-translation-count-queue-capacity 1000)++(defn- update-translation-counts!*+ "Update the entity-id translation counter with the statuses of a batch of entity-id translations."+ [statuses]+ (eid-translation.settings/entity-id-translation-counter!+ (merge-with + (frequencies statuses) (eid-translation.settings/entity-id-translation-counter))))++(defonce ^:private update-translation-count-queue+ (delay (grouper/start!+ #'update-translation-counts!*+ :capacity update-translation-count-queue-capacity+ :interval (* update-translation-count-interval-seconds 1000))))+(mu/defn- update-translation-count!- "Update the entity-id translation counter with the results of a batch of entity-id translations."+ "Count the results of a batch of entity-id translations. The counter is only used for the daily anonymous usage+ stats, so the updates are batched and written asynchronously instead of paying for a (cluster-wide, cache-invalidating)+ Setting update on every translation."[results :- [:sequential eid-translation/Status]]- (let [processed-result (frequencies results)]- (eid-translation.settings/entity-id-translation-counter!- (merge-with + processed-result (eid-translation.settings/entity-id-translation-counter)))))+ (doseq [status results]+ (grouper/submit! @update-translation-count-queue status)))(mu/defn- entity-ids->id-for-model :- [:sequential [:tuple;; We want to pass incorrectly formatted entity-ids through here,diff --git a/src/metabase/lib/field/resolution.cljc b/src/metabase/lib/field/resolution.cljcindex 55f12f83ba3..0d7b1e6ee16 100644--- a/src/metabase/lib/field/resolution.cljc+++ b/src/metabase/lib/field/resolution.cljc@@ -73,7 +73,7 @@(pos-int? table-id))(first (lib.metadata.protocols/metadatas(lib.metadata/->metadata-provider metadata-providerable)- {:lib/type :metadata/column, :table-id table-id, :name #{id-or-name}})))]+ {:lib/type :metadata/column, :table-ids #{table-id}, :name #{id-or-name}})))](-> col(assoc :lib/source :source/table-defaults:lib/source-column-alias (:name col)diff --git a/src/metabase/lib/metadata/cached_provider.cljc b/src/metabase/lib/metadata/cached_provider.cljcindex d3a70fdb3e3..fc8e72e2213 100644--- a/src/metabase/lib/metadata/cached_provider.cljc+++ b/src/metabase/lib/metadata/cached_provider.cljc@@ -104,10 +104,37 @@(lib.metadata.protocols/default-spec-filter-xform metadata-spec))key-set)))+;; cache keys used by this function have the shape [::spec <spec-with-a-single-table-id>], so fetching metadata for+;; several Tables at once (e.g. to pre-warm the cache) and fetching it for a single Table can share cache entries+(defn- metadatas-for-table-ids+ [cache uncached-provider {metadata-type :lib/type, table-ids :table-ids, :as metadata-spec}]+ (let [cache-key (fn [table-id]+ [::spec (assoc metadata-spec :table-ids #{table-id})])+ cached-ids (let [cache* @cache]+ ;; [[get-in]] instead of [[get-in-cache]] because we don't want to filter out `::nil` tombstones+ (into #{} (filter #(get-in cache* (cache-key %))) table-ids))+ missing-ids (set/difference table-ids cached-ids)]+ (when (seq missing-ids)+ (let [newly-fetched (lib.metadata.protocols/metadatas uncached-provider (assoc metadata-spec :table-ids missing-ids))+ by-table-id (group-by :table-id newly-fetched)]+ (doseq [table-id missing-ids+ :let [metadatas (vec (get by-table-id table-id))]]+ (store-in-cache! cache (cache-key table-id) metadatas)+ (doseq [metadata metadatas+ k [:id :name]]+ (store-in-cache! cache [metadata-type k (k metadata)] metadata)))))+ (into [] (mapcat #(get-in-cache cache (cache-key %))) (sort table-ids))))+(mu/defn- metadatas- [cache uncached-provider {metadata-type :lib/type, id-set :id, name-set :name, :as metadata-spec} :- ::lib.metadata.protocols/metadata-spec]- (if (or id-set name-set)+ [cache uncached-provider {metadata-type :lib/type, id-set :id, name-set :name, table-ids :table-ids, :as metadata-spec} :- ::lib.metadata.protocols/metadata-spec]+ (cond+ (or id-set name-set)(metadatas-by-id-or-name cache uncached-provider metadata-spec)++ table-ids+ (metadatas-for-table-ids cache uncached-provider metadata-spec)++ :else(get-in-cache-or-fetch cache[::spec metadata-spec](fn []diff --git a/src/metabase/lib/metadata/invocation_tracker.cljc b/src/metabase/lib/metadata/invocation_tracker.cljcindex 33c1cf83dfb..52ac1f7e27b 100644--- a/src/metabase/lib/metadata/invocation_tracker.cljc+++ b/src/metabase/lib/metadata/invocation_tracker.cljc@@ -22,23 +22,23 @@;;; TODO (Cam 9/10/25) -- we're missing stuff here like handling when we fetch by name. When we added support for;;; fetching metadata by name in #62283 we never added it.-(defn- track-spec! [tracker {metadata-type :lib/type, id-set :id, :keys [table-id card-id], :as _metadata-spec}]+(defn- track-spec! [tracker {metadata-type :lib/type, id-set :id, :keys [table-ids card-ids], :as _metadata-spec}](condid-set(track-ids! tracker metadata-type id-set)- table-id+ table-ids(let [tracking-type (case metadata-type:metadata/column ::table-fields:metadata/measure ::table-measures:metadata/metric ::table-metrics:metadata/segment ::table-segments)]- (track-ids! tracker tracking-type [table-id]))+ (track-ids! tracker tracking-type table-ids))- card-id+ card-ids(let [tracking-type (case metadata-type:metadata/metric ::card-metrics)]- (track-ids! tracker tracking-type [card-id]))))+ (track-ids! tracker tracking-type card-ids))))(defn- metadatas[tracker metadata-provider metadata-spec]diff --git a/src/metabase/lib/metadata/protocols.cljc b/src/metabase/lib/metadata/protocols.cljcindex fe008dfad92..52fe690e96a 100644--- a/src/metabase/lib/metadata/protocols.cljc+++ b/src/metabase/lib/metadata/protocols.cljc@@ -28,25 +28,25 @@[:lib/type [:ref ::metadata-type-excluding-database]][:id {:optional true} [:set {:min 1} pos-int?]][:name {:optional true} [:set {:min 1} :string]]- [:table-id {:optional true} ::lib.schema.id/table]- [:card-id {:optional true} ::lib.schema.id/card]+ [:table-ids {:optional true} [:set {:min 1} ::lib.schema.id/table]]+ [:card-ids {:optional true} [:set {:min 1} ::lib.schema.id/card]][:include-sensitive? {:optional true} :boolean]][:fn{:error/message ":id and :name cannot be used at the same time."}(complement (every-pred :id :name))][:fn- {:error/message ":table-id is currently only supported for Fields, Measures, Metrics, and Segments."}+ {:error/message ":table-ids is currently only supported for Fields, Measures, Metrics, and Segments."}(fn [spec]- (or (not (:table-id spec))+ (or (not (:table-ids spec))(#{:metadata/column :metadata/measure :metadata/metric :metadata/segment} (:lib/type spec))))][:fn- {:error/message ":card-id is currently only supported for Metrics."}+ {:error/message ":card-ids is currently only supported for Metrics."}(fn [spec]- (or (not (:card-id spec))+ (or (not (:card-ids spec))(#{:metadata/metric} (:lib/type spec))))][:fn{:error/message "All metadata types except for :metadata/table and :metadata/transform must include at least one filter"}- (some-fn :id :name :table-id :card-id #(= (:lib/type %) :metadata/table) #(= (:lib/type %) :metadata/transform))]])+ (some-fn :id :name :table-ids :card-ids #(= (:lib/type %) :metadata/table) #(= (:lib/type %) :metadata/transform))]])(defn active-column-pred"Returns a predicate that filters out columns that should not be visible based on `:active` status and@@ -69,21 +69,21 @@This should match [[metabase.lib-be.metadata.jvm/metadata-spec->honey-sql]] as closely aspossible."- [{metadata-type :lib/type, id-set :id, name-set :name, :keys [table-id card-id include-sensitive?], :as _metadata-spec} :- ::metadata-spec]+ [{metadata-type :lib/type, id-set :id, name-set :name, :keys [table-ids card-ids include-sensitive?], :as _metadata-spec} :- ::metadata-spec](let [active-only? (not (or id-set name-set))metric? (= metadata-type :metadata/metric)preds [(when id-set#(contains? id-set (:id %)))(when name-set#(contains? name-set (:name %)))- (when table-id- #(= (:table-id %) table-id))- (when (and table-id metric?)+ (when table-ids+ #(contains? table-ids (:table-id %)))+ (when (and table-ids metric?)#(nil? (:source-card-id %)))- (when card-id+ (when card-ids(if metric?- #(= (:source-card-id %) card-id)- #(= (:card-id %) card-id)))+ #(contains? card-ids (:source-card-id %))+ #(contains? card-ids (:card-id %))))(when active-only?(case metadata-type:metadata/table@@ -223,7 +223,7 @@[metadata-provider :- ::metadata-providermetadata-type :- [:enum :metadata/column :metadata/measure :metadata/metric :metadata/segment]table-id :- ::lib.schema.id/table]- (metadatas metadata-provider {:lib/type metadata-type, :table-id table-id}))+ (metadatas metadata-provider {:lib/type metadata-type, :table-ids #{table-id}}))(mu/defn metadatas-for-card :- [:maybe [:sequential ::metadata]]"Return active (non-archived) metadatas associated with a particular Card, currently only Metrics, so@@ -231,7 +231,7 @@[metadata-provider :- ::metadata-providermetadata-type :- [:enum :metadata/column :metadata/metric :metadata/segment]card-id :- ::lib.schema.id/card]- (metadatas metadata-provider {:lib/type metadata-type, :card-id card-id}))+ (metadatas metadata-provider {:lib/type metadata-type, :card-ids #{card-id}}))(mu/defn table :- [:maybe ::lib.schema.metadata/table]"Return metadata for a specific Table. Metadata should satisfy `:metabase.lib.schema.metadata/table`."@@ -303,23 +303,24 @@([metadata-provider :- ::metadata-providertable-id :- ::lib.schema.id/tableopts :- [:maybe [:map [:include-sensitive? {:optional true} :boolean]]]]- (metadatas metadata-provider {:lib/type :metadata/column- :table-id table-id- :include-sensitive? (boolean (:include-sensitive? opts))})))+ ;; only include `:include-sensitive?` when truthy, so that this shares cached-provider cache entries with other+ ;; ways of fetching the fields for a table, e.g. [[metadatas-for-tables]]+ (metadatas metadata-provider (cond-> {:lib/type :metadata/column, :table-ids #{table-id}}+ (:include-sensitive? opts) (assoc :include-sensitive? true)))))(mu/defn segments :- [:maybe [:sequential ::lib.schema.metadata/segment]]"Return a sequence of legacy Segments associated with a Table with the given `table-id`. Segments should satisfythe `:metabase.lib.schema.metadata/segment` schema. If no Table with ID `table-id` exists, this should error."[metadata-provider :- ::metadata-providertable-id :- ::lib.schema.id/table]- (metadatas metadata-provider {:lib/type :metadata/segment, :table-id table-id}))+ (metadatas metadata-provider {:lib/type :metadata/segment, :table-ids #{table-id}}))(mu/defn measures :- [:maybe [:sequential ::lib.schema.metadata/measure]]"Return a sequence of Measures associated with a Table with the given `table-id`. Measures should satisfythe `:metabase.lib.schema.metadata/measure` schema. If no Table with ID `table-id` exists, this should error."[metadata-provider :- ::metadata-providertable-id :- ::lib.schema.id/table]- (metadatas metadata-provider {:lib/type :metadata/measure, :table-id table-id}))+ (metadatas metadata-provider {:lib/type :metadata/measure, :table-ids #{table-id}}))(#?(:clj p/defprotocol+ :cljs defprotocol) CachedMetadataProvider"Optional. A protocol for a MetadataProvider that some sort of internal cache. This is mostly useful fordiff --git a/src/metabase/lib_be/metadata/jvm.clj b/src/metabase/lib_be/metadata/jvm.cljindex 75d9595ed11..0698d844ec0 100644--- a/src/metabase/lib_be/metadata/jvm.clj+++ b/src/metabase/lib_be/metadata/jvm.clj@@ -513,20 +513,20 @@{:closed true}[:where {:optional true} vector?]]"This should match [[metabase.lib.metadata.protocols/default-spec-filter-xform]] as closely as possible."- [database-id :- ::lib.schema.id/database- {metadata-type :lib/type, id-set :id, name-set :name, :keys [table-id card-id include-sensitive?], :as _metadata-spec} :- ::lib.metadata.protocols/metadata-spec]+ [database-id :- ::lib.schema.id/database+ {metadata-type :lib/type, id-set :id, name-set :name, :keys [table-ids card-ids include-sensitive?], :as _metadata-spec} :- ::lib.metadata.protocols/metadata-spec](let [database-id-key (db-id-key metadata-type)active-only? (not (or id-set name-set))metric? (= metadata-type :metadata/metric)where-clauses (cond-> []- database-id-key (conj [:= database-id-key database-id])- id-set (conj [:in (id-key metadata-type) id-set])- name-set (conj [:in (name-key metadata-type) name-set])- table-id (conj [:= (table-id-key metadata-type) table-id])- card-id (conj [:= (card-id-key metadata-type) card-id])- active-only? (conj (active-only-honeysql-filter metadata-type {:include-sensitive? include-sensitive?}))- metric? (conj [:= :type [:inline "metric"]])- (and metric? table-id) (conj [:= :source_card_id nil]))]+ database-id-key (conj [:= database-id-key database-id])+ id-set (conj [:in (id-key metadata-type) id-set])+ name-set (conj [:in (name-key metadata-type) name-set])+ table-ids (conj [:in (table-id-key metadata-type) table-ids])+ card-ids (conj [:in (card-id-key metadata-type) card-ids])+ active-only? (conj (active-only-honeysql-filter metadata-type {:include-sensitive? include-sensitive?}))+ metric? (conj [:= :type [:inline "metric"]])+ (and metric? table-ids) (conj [:= :source_card_id nil]))](reducesql.helpers/where{}diff --git a/src/metabase/lib_be/query.clj b/src/metabase/lib_be/query.cljindex ece3fc94893..6d005300cdb 100644--- a/src/metabase/lib_be/query.clj+++ b/src/metabase/lib_be/query.clj@@ -1,6 +1,8 @@(ns metabase.lib-be.query(:require[metabase.lib.metadata :as lib.metadata]+ [metabase.lib.metadata.protocols :as lib.metadata.protocols]+ [metabase.lib.metadata.util :as lib.metadata.util][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.walk.util :as lib.walk.util][metabase.util.malli :as mu]))@@ -9,10 +11,18 @@"Bulk-load all metadata referenced by `referenced-entity-ids` into `metadata-providerable`'s cache."[metadata-providerable :- ::lib.schema.metadata/metadata-providerable{:keys [table card metric measure segment snippet]} :- ::lib.walk.util/referenced-entity-ids]- (lib.metadata/bulk-metadata metadata-providerable :metadata/table table)- (lib.metadata/bulk-metadata metadata-providerable :metadata/card card)- (lib.metadata/bulk-metadata metadata-providerable :metadata/metric metric)- (lib.metadata/bulk-metadata metadata-providerable :metadata/measure measure)- (lib.metadata/bulk-metadata metadata-providerable :metadata/segment segment)- (lib.metadata/bulk-metadata metadata-providerable :metadata/native-query-snippet snippet)+ (when (seq table)+ (lib.metadata/bulk-metadata metadata-providerable :metadata/table table)+ (lib.metadata.protocols/metadatas (lib.metadata.util/->metadata-provider metadata-providerable)+ {:lib/type :metadata/column, :table-ids (set table)}))+ (when (seq card)+ (lib.metadata/bulk-metadata metadata-providerable :metadata/card card))+ (when (seq metric)+ (lib.metadata/bulk-metadata metadata-providerable :metadata/metric metric))+ (when (seq measure)+ (lib.metadata/bulk-metadata metadata-providerable :metadata/measure measure))+ (when (seq segment)+ (lib.metadata/bulk-metadata metadata-providerable :metadata/segment segment))+ (when (seq snippet)+ (lib.metadata/bulk-metadata metadata-providerable :metadata/native-query-snippet snippet))nil)diff --git a/src/metabase/lib_metric/dimension.cljc b/src/metabase/lib_metric/dimension.cljcindex 111682bae77..82cc939c85e 100644--- a/src/metabase/lib_metric/dimension.cljc+++ b/src/metabase/lib_metric/dimension.cljc@@ -213,7 +213,7 @@table-id :- pos-int?](lib.metadata.protocols/metadatasmetadata-provider- {:lib/type :metadata/dimension :table-id table-id}))+ {:lib/type :metadata/dimension :table-ids #{table-id}}));;; ------------------------------------------------- Dimension Resolution -------------------------------------------------;;; Functions for resolving dimension UUIDs to field IDs through dimension mappings.diff --git a/src/metabase/lib_metric/metadata/jvm.clj b/src/metabase/lib_metric/metadata/jvm.cljindex 94f9b13dd5e..6034dd4dbf1 100644--- a/src/metabase/lib_metric/metadata/jvm.clj+++ b/src/metabase/lib_metric/metadata/jvm.clj@@ -30,14 +30,14 @@(defn- metric-spec->honey-sql"Build HoneySQL WHERE clause for fetching metrics.Metrics are Cards with type='metric' - not scoped to any database."- [{id-set :id, name-set :name, :keys [table-id card-id], :as _metadata-spec}]+ [{id-set :id, name-set :name, :keys [table-ids card-ids], :as _metadata-spec}](let [active-only? (not (or id-set name-set))where-clauses (cond-> [[:= :type [:inline "metric"]]]id-set (conj [:in :id id-set])name-set (conj [:in :name name-set])- table-id (conj [:= :table_id table-id])- table-id (conj [:= :source_card_id nil])- card-id (conj [:= :source_card_id card-id])+ table-ids (conj [:in :table_id table-ids])+ table-ids (conj [:= :source_card_id nil])+ card-ids (conj [:in :source_card_id card-ids])active-only? (conj [:= :archived false]))](reduce sql.helpers/where {} where-clauses)))@@ -76,12 +76,12 @@(defn- measure-spec->honey-sql"Build HoneySQL WHERE clause for fetching measures."- [{id-set :id, name-set :name, :keys [table-id], :as _metadata-spec}]+ [{id-set :id, name-set :name, :keys [table-ids], :as _metadata-spec}](let [active-only? (not (or id-set name-set))where-clauses (cond-> []id-set (conj [:in :measure/id id-set])name-set (conj [:in :measure/name name-set])- table-id (conj [:= :measure/table_id table-id])+ table-ids (conj [:in :measure/table_id table-ids])active-only? (conj [:= :measure/archived false]))](reduce sql.helpers/where {} where-clauses)))@@ -99,19 +99,19 @@(defn- fetch-dimensions"Fetch dimensions by aggregating from metrics and measures.Dimensions are extracted from their parent entities and annotated with source info."- [{id-set :id, :keys [metric-id measure-id table-id]}]+ [{id-set :id, :keys [metric-id measure-id table-ids]}](let [;; Fetch from metrics if not a measure-specific querymetric-dims (when-not measure-id(let [metric-spec (cond-> {:lib/type :metadata/metric}metric-id (assoc :id #{metric-id})- table-id (assoc :table-id table-id))+ table-ids (assoc :table-ids table-ids))metrics (fetch-metrics metric-spec)](mapcat #(extract-dimensions-from-entity % :metric) metrics)));; Fetch from measures if not a metric-specific querymeasure-dims (when-not metric-id(let [measure-spec (cond-> {:lib/type :metadata/measure}measure-id (assoc :id #{measure-id})- table-id (assoc :table-id table-id))+ table-ids (assoc :table-ids table-ids))measures (fetch-measures-for-dimensions measure-spec)](mapcat #(extract-dimensions-from-entity % :measure) measures)))all-dims (concat metric-dims measure-dims)]@@ -144,7 +144,7 @@(lib.metadata.protocols/metadatas mp {:lib/type :metadata/dimension :metric-id 1});; Routes to correct database provider for table 1- (lib.metadata.protocols/metadatas mp {:lib/type :metadata/column :table-id 1})+ (lib.metadata.protocols/metadatas mp {:lib/type :metadata/column :table-ids #{1}})```"[](let [table->db (table->database-id)diff --git a/src/metabase/lib_metric/metadata/provider.cljc b/src/metabase/lib_metric/metadata/provider.cljcindex d619ae0c765..66706b3b481 100644--- a/src/metabase/lib_metric/metadata/provider.cljc+++ b/src/metabase/lib_metric/metadata/provider.cljc@@ -38,12 +38,15 @@tables-by-db))))(defn- route-metadata-by-table- "Route a metadata request to the appropriate database provider based on table-id."- [table->db-fn db-provider-fn {:keys [table-id], :as metadata-spec}]- (when table-id- (when-let [db-id (table->db-fn table-id)]- (when-let [provider (db-provider-fn db-id)]- (lib.metadata.protocols/metadatas provider metadata-spec)))))+ "Route a metadata request to the appropriate database provider(s) based on its `:table-ids` set."+ [table->db-fn db-provider-fn {table-ids :table-ids, :as metadata-spec}]+ (when (seq table-ids)+ (into []+ (mapcat (fn [[db-id table-ids]]+ (when db-id+ (when-let [provider (db-provider-fn db-id)]+ (lib.metadata.protocols/metadatas provider (assoc metadata-spec :table-ids (set table-ids)))))))+ (group-by table->db-fn table-ids))))(defn- route-card-metadata"Route card metadata request. Cards can span databases, so we need to handle this specially."diff --git a/src/metabase/queries/core.clj b/src/metabase/queries/core.cljindex 5b2a6d480e7..21338dbebf9 100644--- a/src/metabase/queries/core.clj+++ b/src/metabase/queries/core.clj@@ -44,7 +44,7 @@[metabase.queries.models.queryaverage-execution-time-msquery->database-and-table-ids- save-query-and-update-average-execution-time!])+ save-queries-and-update-average-execution-times!])#_{:clj-kondo/ignore [:missing-docstring]}(p/import-def metabase.queries.models.card/populate-query-fields populate-card-query-fields)diff --git a/src/metabase/queries/models/query.clj b/src/metabase/queries/models/query.cljindex 9eac4389b5e..ffb118ef735 100644--- a/src/metabase/queries/models/query.clj+++ b/src/metabase/queries/models/query.clj@@ -9,11 +9,14 @@[metabase.queries.schema :as queries.schema][metabase.util.honey-sql-2 :as h2x][metabase.util.json :as json]+ [metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.registry :as mr][methodical.core :as methodical][toucan2.core :as t2]- [toucan2.model :as t2.model]))+ [toucan2.model :as t2.model])+ (:import+ (java.nio ByteBuffer)))(set! *warn-on-reflection* true)@@ -43,12 +46,22 @@:unsigned:integer))+(def ^:private smoothing-factor+ "The weight of the latest execution time in the exponential rolling average formula:++ new-average = decay-factor * average + smoothing-factor * execution-time"+ 0.1)++(def ^:private decay-factor+ "The weight of the previous average in the exponential rolling average formula, see [[smoothing-factor]]."+ (- 1.0 smoothing-factor))+(defn- update-rolling-average-execution-time!"Update the rolling average execution time for query with `query-hash`. Returns `true` if a record was updated,or `false` if no matching records were found."^Boolean [query ^bytes query-hash ^Integer execution-time-ms]- (let [avg-execution-time (h2x/cast (int-casting-type) (h2x/round (h2x/+ (h2x/* [:inline 0.9] :average_execution_time)- [:inline (* 0.1 execution-time-ms)])+ (let [avg-execution-time (h2x/cast (int-casting-type) (h2x/round (h2x/+ (h2x/* [:inline decay-factor] :average_execution_time)+ [:inline (* smoothing-factor execution-time-ms)])[:inline 0]))](or;; if it DOES NOT have a query (yet) set that. In 0.31.0 we added the query.query column, and it gets set for all@@ -63,29 +76,113 @@{:query_hash query-hash}{:average_execution_time avg-execution-time})))))-(defn- record-new-query-entry!- "Record a query and its execution time for a `query` with `query-hash` that's not already present in the DB.- `execution-time-ms` is used as a starting point."- [query ^bytes query-hash ^Integer execution-time-ms]- (first (t2/insert-returning-instances! :model/Query- :query query- :query_hash query-hash- :average_execution_time execution-time-ms)))--(defn save-query-and-update-average-execution-time!- "Update the recorded average execution time (or insert a new record if needed) for `query` with `query-hash`."- [query ^bytes query-hash ^Integer execution-time-ms]- {:pre [(bytes? query-hash)]}- (or- ;; if there's already a matching Query update the rolling average- (update-rolling-average-execution-time! query query-hash execution-time-ms)- ;; otherwise try adding a new entry. If for some reason there was a race condition and a Query entry was added in- ;; the meantime we'll try updating that existing record- (try (record-new-query-entry! query query-hash execution-time-ms)- (catch Throwable e- (or (update-rolling-average-execution-time! query query-hash execution-time-ms)- ;; rethrow e if updating an existing average execution time failed- (throw e))))))+(defn- rolling-average-coefficients+ "Collapse applying the rolling-average formula (see [[smoothing-factor]]) once per execution time into a single+ linear equation `new-average = c0 * average + c1`."+ [execution-times-ms]+ (reduce (fn [[c0 c1] t]+ [(* decay-factor c0) (+ (* decay-factor c1) (* smoothing-factor t))])+ [1.0 0.0]+ execution-times-ms))++(defn- initial-average-execution-time+ "Average execution time for a new Query entry: the first execution time, with any subsequent ones folded in the same+ way consecutive [[update-rolling-average-execution-time!]] calls would."+ ^long [execution-times-ms]+ (reduce (fn [avg t]+ (Math/round (+ (* decay-factor (double avg)) (* smoothing-factor (double t)))))+ (long (first execution-times-ms))+ (rest execution-times-ms)))++(defn- hash-key+ "Wrap a query-hash byte array so it can be used as a map key: unlike byte arrays themselves, `ByteBuffer`s have+ value-based equality and hash code."+ [hash-bytes]+ (when hash-bytes+ (ByteBuffer/wrap ^bytes hash-bytes)))++(defn- rolling-average-update-expr+ "HoneySQL expression that updates `:average_execution_time` as if the rolling-average formula were applied once per+ execution time, in order."+ [execution-times-ms]+ (let [[c0 c1] (rolling-average-coefficients execution-times-ms)]+ (h2x/cast (int-casting-type)+ (h2x/round (h2x/+ (h2x/* [:inline c0] :average_execution_time)+ [:inline c1])+ [:inline 0]))))++(defn- update-rolling-average-execution-times!+ "Update the rolling average execution times for `groups` (each a sequence of entries sharing a query hash) with a+ single atomic UPDATE."+ [groups]+ (when (seq groups)+ (let [hash+exprs (vec (for [group groups]+ [(:query-hash (first group))+ (rolling-average-update-expr (map :execution-time-ms group))]))]+ (t2/query {:update (t2/table-name :model/Query)+ :set {:average_execution_time (into [:case]+ (mapcat (fn [[query-hash expr]]+ [[:= :query_hash query-hash] expr]))+ hash+exprs)}+ :where [:in :query_hash (mapv first hash+exprs)]}))))++(defn- insert-query-entries!+ "Insert new Query rows for `groups` (each a sequence of entries sharing a query hash) with a single INSERT."+ [groups]+ (t2/insert! :model/Query (for [group groups]+ {:query (:query (first group))+ :query_hash (:query-hash (first group))+ :average_execution_time (initial-average-execution-time+ (map :execution-time-ms group))})))++(defn- query-hash->row-status+ "Fetch which of `query-hashes` already have Query rows. Returns a map of [[hash-key]] -> `:up-to-date` or+ `:needs-query-backfill` (for rows predating 0.31.0 whose `query` column was never set); hashes without a row are+ absent from the map."+ [query-hashes]+ (if (empty? query-hashes)+ {}+ (into {}+ (map (fn [{:keys [query_hash missing_query]}]+ [(hash-key query_hash)+ ;; mysql returns 0/1 instead of booleans+ (if (contains? #{true 1} missing_query)+ :needs-query-backfill+ :up-to-date)]))+ (t2/reducible-query {:select [:query_hash [[:= :query nil] :missing_query]]+ :from [(t2/table-name :model/Query)]+ :where [:in :query_hash query-hashes]}))))++(defn save-queries-and-update-average-execution-times!+ "Update the recorded average execution times (or insert new records as needed) for `entries`, maps with `:query`,+ `:query-hash`, and `:execution-time-ms` keys, using a fixed number of statements rather than several per entry.+ Multiple entries for the same hash are folded into a single atomic update equivalent to applying the rolling-average+ formula once per entry, in order.++ The INSERT of new Query rows runs in a nested transaction (a savepoint), so that if it conflicts inside a caller's+ transaction (e.g. when batch updates run synchronously inside a transaction in tests), the constraint violation+ doesn't abort the outer transaction -- Postgres refuses to run any further statements in a transaction with a failed+ statement otherwise. A conflict means someone else just inserted one of these hashes with its own initial average;+ dropping this batch's samples for a brand-new row doesn't meaningfully change the rolling average, so there is no+ need to retry."+ [entries]+ (when (seq entries)+ (let [groups (vals (group-by (fn [entry] (hash-key (:query-hash entry))) entries))+ hash->status (query-hash->row-status (map (fn [group] (:query-hash (first group))) groups))+ {:keys [up-to-date needs-query-backfill new-entry]}+ (group-by (fn [group]+ (get hash->status (hash-key (:query-hash (first group))) :new-entry))+ groups)]+ (update-rolling-average-execution-times! up-to-date)+ ;; rare legacy path: rows whose `query` still needs backfilling are handled individually+ (doseq [{:keys [query query-hash execution-time-ms]} (apply concat needs-query-backfill)]+ (update-rolling-average-execution-time! query query-hash execution-time-ms))+ (when (seq new-entry)+ (try+ (t2/with-transaction [_conn]+ (insert-query-entries! new-entry))+ (catch Throwable e+ (log/tracef e "Error inserting concurrently created Query entries: %s" (ex-message e))))))))(mr/def ::database-and-table-ids[:mapdiff --git a/src/metabase/query_processor/dashboard.clj b/src/metabase/query_processor/dashboard.cljindex f491a8033a1..de94dd6677d 100644--- a/src/metabase/query_processor/dashboard.clj+++ b/src/metabase/query_processor/dashboard.clj@@ -5,6 +5,7 @@[clojure.string :as str][medley.core :as m][metabase.api.common :as api]+ [metabase.dashboards.models.dashboard :as dashboard][metabase.dashboards.schema :as dashboards.schema][metabase.events.core :as events][metabase.lib.core :as lib]@@ -127,9 +128,11 @@request-params :- [:maybe [:sequential :map]]](log/tracef "Resolving Dashboard %d Card %d query request parameters" dashboard-id card-id)(let [request-params (some-> request-params not-empty (->> (lib/normalize ::dashboards.schema/parameters)))- dashboard (-> (t2/select-one :model/Dashboard :id dashboard-id)- (t2/hydrate :resolved-params)- (api/check-404))+ dashboard (api/check-404 (t2/select-one :model/Dashboard :id dashboard-id))+ dashcard (api/check-404 (t2/select-one [:model/DashboardCard :id :card_id :dashboard_id :parameter_mappings]+ :id dashcard-id+ :dashboard_id dashboard-id))+ resolved-params (dashboard/dashboard->resolved-params (assoc dashboard :dashcards [dashcard]))dashboard-param-id->param (into {};; remove the `:default` values from Dashboard params. We don't ACTUALLY want to;; use these values ourselves -- the expectation is that the frontend will pass@@ -139,7 +142,7 @@;; more information.(map (fn [[param-id param]][param-id (dissoc param :default)]))- (:resolved-params dashboard))+ resolved-params);; ignore default values in request params as well. (#20516)request-param-id->param (into {} (map (juxt :id #(dissoc % :default))) request-params)merged-parameters (vals (merge (dashboard-param-defaults dashboard-param-id->param card-id)diff --git a/src/metabase/query_processor/middleware/cache.clj b/src/metabase/query_processor/middleware/cache.cljindex 0589f663aff..9e5f1e168a2 100644--- a/src/metabase/query_processor/middleware/cache.clj+++ b/src/metabase/query_processor/middleware/cache.clj@@ -12,6 +12,7 @@[clojure.string :as str][java-time.api :as t][medley.core :as m]+ [metabase.batch-processing.core :as grouper][metabase.cache.core :as cache][metabase.config.core :as config][metabase.lib.core :as lib]@@ -45,14 +46,31 @@;;; ------------------------------------------------------ Save -------------------------------------------------------(defn- purge! [backend]- (try- (log/tracef "Purging cache entries older than %s" (u/format-seconds (cache/query-caching-max-ttl)))- (i/purge-old-entries! backend (cache/query-caching-max-ttl))- (log/trace "Successfully purged old cache entries.")- :done- (catch Throwable e- (log/errorf e "Error purging old cache entries: %s" (ex-message e)))))+(def ^:private purge-interval-seconds+ "How often, at most, to purge cache entries older than [[cache/query-caching-max-ttl]]. Purging on every save is+ wasteful: the max TTL is measured in days, and on instances with a high cache-miss rate the repeated DELETEs are+ expensive and contend with concurrent cache writes."+ (* 5 60))++(def ^:private purge-queue-capacity 1000)++(defn- purge!* [backends]+ (doseq [backend (distinct backends)]+ (try+ (log/tracef "Purging cache entries older than %s" (u/format-seconds (cache/query-caching-max-ttl)))+ (i/purge-old-entries! backend (cache/query-caching-max-ttl))+ (log/trace "Successfully purged old cache entries.")+ (catch Throwable e+ (log/errorf e "Error purging old cache entries: %s" (ex-message e))))))++(defonce ^:private purge-queue+ (delay (grouper/start!+ #'purge!*+ :capacity purge-queue-capacity+ :interval (* purge-interval-seconds 1000))))++(defn- schedule-purge! [backend]+ (grouper/submit! @purge-queue backend))(def ^:private ^:dynamic *in-fn*"The `in-fn` provided by [[impl/do-with-serialization]]."@@ -87,7 +105,7 @@(log/trace "Got serialized bytes; saving to cache backend")(i/save-results! *backend* query-hash bytez)(log/debug "Successfully cached results for query.")- (purge! *backend*))))+ (schedule-purge! *backend*)))):done(catch Throwable e(if (= (:type (ex-data e)) ::impl/max-bytes)diff --git a/src/metabase/query_processor/middleware/prefetch_metadata.clj b/src/metabase/query_processor/middleware/prefetch_metadata.cljnew file mode 100644index 00000000000..d33266ecb42--- /dev/null+++ b/src/metabase/query_processor/middleware/prefetch_metadata.clj@@ -0,0 +1,46 @@+(ns metabase.query-processor.middleware.prefetch-metadata+ "Middleware that bulk-loads all the metadata referenced by a query into the metadata provider's cache up front, so+ that later middleware and SQL compilation hit the cache instead of fetching objects from the app DB one at a time."+ (:require+ [clojure.set :as set]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.metadata.protocols :as lib.metadata.protocols]+ [metabase.util.log :as log]))++(defn- card->referenced-entity-ids+ [metadata-provider card]+ (try+ (when (seq (:dataset-query card))+ (lib/all-referenced-entity-ids [(lib/card->underlying-query metadata-provider card)]))+ (catch Throwable e+ ;; prefetching is an optimization: a Card we cannot make sense of here will be dealt with when it is resolved+ (log/debugf e "Error getting referenced entity ids for Card %s" (:id card)))))++(defn- all-referenced-entity-ids+ "Collect the ids of all the entities referenced by `query`, following Card references (source cards and+ `{{#card-id}}` template tags) recursively: the referenced Cards are fetched level by level, one bulk call per level+ of nesting, and the entities their queries reference are folded into the result."+ [metadata-provider query]+ (loop [referenced-ids (lib/all-referenced-entity-ids [query])+ seen-card-ids #{}]+ (let [new-card-ids (set/difference (:card referenced-ids) seen-card-ids)]+ (if (seq new-card-ids)+ (recur (transduce (keep #(card->referenced-entity-ids metadata-provider %))+ (partial merge-with set/union)+ referenced-ids+ (lib.metadata/bulk-metadata metadata-provider :metadata/card new-card-ids))+ (into seen-card-ids new-card-ids))+ referenced-ids))))++(defn prefetch-metadata+ "Bulk-load the metadata for everything referenced by `query` -- tables, cards (recursively), metrics, measures,+ segments, snippets, and the fields referenced by template tags or implicit joins -- into the metadata provider's+ cache using a constant number of app-DB calls per level of Card nesting. Runs before source cards are resolved, so+ that resolving them hits the cache."+ [query]+ (let [metadata-provider (lib.metadata/->metadata-provider query)]+ (when (lib.metadata.protocols/cached-metadata-provider-with-cache? metadata-provider)+ (lib-be/bulk-load-query-metadata! metadata-provider (all-referenced-entity-ids metadata-provider query))))+ query)diff --git a/src/metabase/query_processor/middleware/process_userland_query.clj b/src/metabase/query_processor/middleware/process_userland_query.cljindex 387ed0e094d..379fa350da7 100644--- a/src/metabase/query_processor/middleware/process_userland_query.clj+++ b/src/metabase/query_processor/middleware/process_userland_query.clj@@ -5,13 +5,14 @@ViewLog recording is triggered indirectly by the call to [[events/publish-event!]] with the `:event/card-query`event -- see [[metabase.view-log.events.view-log]]."- (:refer-clojure :exclude [every? empty? get-in])+ (:refer-clojure :exclude [every? empty? get-in not-empty])(:require[java-time.api :as t][metabase.analytics-interface.core :as analytics][metabase.analytics.core :as analytics.core][metabase.analytics.settings :as analytics.settings][metabase.api.common :as api]+ [metabase.batch-processing.core :as grouper][metabase.events.core :as events][metabase.lib.computed :as lib.computed][metabase.queries.models.query :as query]@@ -23,7 +24,7 @@[metabase.util.json :as json][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.performance :refer [every? empty? get-in]]+ [metabase.util.performance :refer [every? empty? get-in not-empty]]^{:clj-kondo/ignore [:discouraged-namespace]}[toucan2.core :as t2]))@@ -40,43 +41,52 @@;;; | Save Query Execution |;;; +----------------------------------------------------------------------------------------------------------------+-;; TODO - I'm not sure whether this should happen async as is currently the case, or should happen synchronously e.g.-;; in the completing arity of the rf-;;-;; Async seems like it makes sense from a performance standpoint, but should we have some sort of shared threadpool-;; for other places where we would want to do async saves (such as results-metadata for Cards?)+(def ^:private save-execution-metadata-interval-seconds 20)++(def ^:private save-execution-metadata-queue-capacity 500)+(defn- save-execution-metadata!*- "Save a `QueryExecution` and update the average execution time for the corresponding `Query`."- [{query :json_query, query-hash :hash, running-time :running_time, context :context :as query-execution}]+ "Save a batch of `QueryExecution`s and update the average execution times for the corresponding `Query`s."+ [query-executions](tracing/with-span :db-app "db-app.save-query-execution" {}- (when-not (:cache_hit query-execution)- (query/save-query-and-update-average-execution-time! query query-hash running-time))- (if-not context- (log/warn "Cannot save QueryExecution, missing :context")- (t2/insert-returning-pk! :model/QueryExecution (dissoc query-execution :json_query)))))+ (log/tracef "Saving %d QueryExecutions" (count query-executions))+ (when-let [entries (not-empty+ (for [{query :json_query, query-hash :hash, running-time :running_time, :as query-execution} query-executions+ :when (and (not (:cache_hit query-execution)) query-hash running-time)]+ {:query query, :query-hash query-hash, :execution-time-ms running-time}))]+ (try+ (query/save-queries-and-update-average-execution-times! entries)+ (catch Throwable e+ (log/error e "Error updating query average execution times"))))+ (try+ (let [{with-context true, no-context false} (group-by (comp some? :context) query-executions)]+ (when (seq no-context)+ (log/warnf "Cannot save %d QueryExecution(s), missing :context" (count no-context)))+ (when (seq with-context)+ (t2/insert! :model/QueryExecution (map #(dissoc % :json_query) with-context))))+ (catch Throwable e+ (log/error e "Error saving query execution info")))))++(defonce ^:private save-execution-metadata-queue+ (delay (grouper/start!+ #'save-execution-metadata!*+ :capacity save-execution-metadata-queue-capacity+ :interval (* save-execution-metadata-interval-seconds 1000))))(defn- save-execution-metadata!- "Save a `QueryExecution` row containing `execution-info`. Done asynchronously when a query is finished."+ "Save a `QueryExecution` row containing `execution-info`. Saves are batched and written asynchronously, so they may+ not be visible for up to [[save-execution-metadata-interval-seconds]] (and may be lost on non-graceful shutdown);+ this is analytics data, so we consider that an acceptable trade for not paying for an INSERT per query execution.+ Tests can bind [[qp.util/*execute-async?*]] to `false` to save synchronously on the current thread."[execution-info]- (let [;; Capture SDK info now, on the request thread, because `with-execute-async` below intentionally- ;; does not convey dynamic var bindings to the background thread (to avoid holding DB connections).- ;; `include-sdk-info` also runs in the `before-insert` hook as a safety net for any code path- ;; that inserts QueryExecution directly (where dynamic vars would still be bound).- execution-info' (analytics.core/include-sdk-info execution-info)]- (qp.util/with-execute-async- ;; 1. Asynchronously save QueryExecution, update query average execution time etc. using the Agent/pooledExecutor- ;; pool, which is a fixed pool of size `nthreads + 2`. This way we don't spin up a ton of threads doing unimportant- ;; background query execution saving (as `future` would do, which uses an unbounded thread pool by default)- ;;- ;; 2. This is on purpose! By *not* using `bound-fn` or `future`, any dynamic variables in play when the task is- ;; submitted, such as `db/*connection*`, won't be in play when the task is actually executed. That way we won't- ;; attempt to use closed DB connections- (fn []- (log/trace "Saving QueryExecution info")- (try- (save-execution-metadata!* (add-running-time execution-info'))- (catch Throwable e- (log/error e "Error saving query execution info")))))))+ (let [;; Capture SDK info and compute the running time now, on the query execution thread: by the time the batch is+ ;; processed the request (and its dynamic bindings and clock) is long gone. `include-sdk-info` also runs in+ ;; the `before-insert` hook as a safety net for any code path that inserts QueryExecution directly (where+ ;; dynamic vars would still be bound).+ execution-info' (add-running-time (analytics.core/include-sdk-info execution-info))]+ (if qp.util/*execute-async?*+ (grouper/submit! @save-execution-metadata-queue execution-info')+ (save-execution-metadata!* [execution-info']))))(defn- save-successful-execution-metadata! [cache-details is-sandboxed? query-execution result-rows](let [qe-map (assoc query-executiondiff --git a/src/metabase/query_processor/preprocess.clj b/src/metabase/query_processor/preprocess.cljindex bb3a215637b..4268f3d1223 100644--- a/src/metabase/query_processor/preprocess.clj+++ b/src/metabase/query_processor/preprocess.clj@@ -31,6 +31,7 @@[metabase.query-processor.middleware.parameters :as parameters][metabase.query-processor.middleware.permissions :as qp.perms][metabase.query-processor.middleware.persistence :as qp.persistence]+ [metabase.query-processor.middleware.prefetch-metadata :as prefetch-metadata][metabase.query-processor.middleware.reconcile-breakout-and-order-by-bucketing :as reconcile-bucketing][metabase.query-processor.middleware.remove-inactive-field-refs :as qp.remove-inactive-field-refs][metabase.query-processor.middleware.resolve-fields :as qp.resolve-fields]@@ -64,6 +65,7 @@#'qp.perms/remove-persisted-info-native-keys#'qp.constraints/maybe-add-default-userland-constraints#'validate/validate-query+ #'prefetch-metadata/prefetch-metadata#'fetch-source-query/resolve-source-cards#'drop-fields-in-summaries/drop-fields-in-summaries#'expand-aggregations/expand-aggregationsdiff --git a/src/metabase/query_processor/util.clj b/src/metabase/query_processor/util.cljindex 41fc37ec065..558723f0710 100644--- a/src/metabase/query_processor/util.clj+++ b/src/metabase/query_processor/util.clj@@ -98,23 +98,11 @@col))))(def ^:dynamic *execute-async?*- "Used to control `with-execute-async` to whether or not execute its body asynchronously."+ "Whether to save QueryExecutions (and other post-execution side effects) asynchronously via the batch-processing+ queue. Bind (or redef, when the query runs on another thread) to `false` in tests to save synchronously on the+ calling thread."true)-(defn do-with-execute-async- "Impl of `with-execute-async`"- [thunk]- (if *execute-async?*- (.submit clojure.lang.Agent/pooledExecutor ^Runnable thunk)- (thunk)))--(defmacro with-execute-async- "Execute body asynchronously in a pooled executor.-- Used for side effects during query execution like saving query execution info."- [thunk]- `(do-with-execute-async ~thunk))-(mu/defn userland-query? :- :boolean"Returns true if the query is an userland query, else false."[query :- ::qp.schema/any-query]diff --git a/src/metabase/view_log/events/view_log.clj b/src/metabase/view_log/events/view_log.cljindex 5df06d7bda9..bb999527165 100644--- a/src/metabase/view_log/events/view_log.clj+++ b/src/metabase/view_log/events/view_log.clj@@ -2,6 +2,7 @@"This namespace is responsible for subscribing to events which should update the view log and view counts."(:require[java-time.api :as t]+ [metabase.analytics.core :as analytics][metabase.api.common :as api][metabase.app-db.cluster-lock :as cluster-lock][metabase.audit-app.core :as audit]@@ -64,11 +65,13 @@(def ^:private increment-view-count-interval-seconds 20)+(def ^:private increment-view-count-queue-capacity 500)+(defonce ^:privateincrease-view-count-queue(delay (grouper/start!#'increment-view-counts!*- :capacity 500+ :capacity increment-view-count-queue-capacity:interval (* increment-view-count-interval-seconds 1000))))(defn increment-view-counts!@@ -76,13 +79,36 @@[model model-id](grouper/submit! @increase-view-count-queue {:model model :id model-id}))+(def ^:private record-view-interval-seconds 20)++(def ^:private record-view-queue-capacity 500)++(defn- record-views!* [views]+ (log/debugf "Recording %d views" (count views))+ (try+ (t2/insert! :model/ViewLog views)+ (catch Exception e+ (log/error e "Failed to record views"))))++(defonce ^:private record-view-queue+ (delay (grouper/start!+ #'record-views!*+ :capacity record-view-queue-capacity+ :interval (* record-view-interval-seconds 1000))))+(mu/defn record-views!- "Simple base function for recording a view of a given `model` and `model-id` by a certain `user`."+ "Simple base function for recording a view of a given `model` and `model-id` by a certain `user`. Views are batched+ and inserted asynchronously, so they may not be visible in the view log for up+ to [[record-view-interval-seconds]] (plus they are lost on non-graceful shutdown); we consider that an acceptable+ trade for not paying for a synchronous INSERT on every read/query request."[view-or-views :- [:or :map [:sequential :map]]](span/with-span!{:name "record-view!"}(when (premium-features/log-enabled?)- (t2/insert! :model/ViewLog view-or-views))))+ (doseq [view (u/one-or-many view-or-views)]+ (grouper/submit! @record-view-queue (-> view+ (assoc :timestamp (t/offset-date-time))+ analytics/include-sdk-info))))))(defn generate-view"Generates a view, given an event map. The event map either has an `object` or a `model` and `object-id`."@@ -116,6 +142,8 @@(def ^:private update-dashboard-last-viewed-at-interval-seconds 20)+(def ^:private update-dashboard-last-viewed-at-queue-capacity 500)+(defn- update-dashboard-last-viewed-at!* [dashboard-id-timestamps](let [dashboard-id->timestamp (update-vals (group-by :id dashboard-id-timestamps)(fn [xs] (apply t/max (map :timestamp xs))))]@@ -138,7 +166,7 @@(def ^:private update-dashboard-last-viewed-at-queue(delay (grouper/start!#'update-dashboard-last-viewed-at!*- :capacity 500+ :capacity update-dashboard-last-viewed-at-queue-capacity:interval (* update-dashboard-last-viewed-at-interval-seconds 1000))))(defn- update-dashboard-last-viewed-at!diff --git a/test/metabase/agent_api/api_test.clj b/test/metabase/agent_api/api_test.cljindex dad419e4af5..4227e841dd6 100644--- a/test/metabase/agent_api/api_test.clj+++ b/test/metabase/agent_api/api_test.clj@@ -1118,19 +1118,20 @@(is (string? (some-> resp :via first :error)))))(testing "A successful query records a QueryExecution audit row tagged :agent";; Bypass-of-userland regression (#1): without `prepare-agent-query` no audit row was- ;; written. Verify a row lands and carries the agent context. The QueryExecution- ;; insert is async, so poll for the new row.- (let [before (t2/count :model/QueryExecution)]- (mt/user-http-request :crowberto :post 202 "agent/v1/execute-sql"- {:database_id (mt/id)- :sql "SELECT 1 AS audit_probe"})- (let [latest (u/poll {:thunk (fn [] (t2/select-one :model/QueryExecution- {:order-by [[:started_at :desc]]}))- :done? (fn [_qe] (> (t2/count :model/QueryExecution) before))- :timeout-ms 5000- :interval-ms 50})]- (is (some? latest) "QueryExecution row should be inserted within 5s")- (is (= :agent (:context latest)))))))+ ;; written. Verify a row lands and carries the agent context. QueryExecutions are saved in async batches by+ ;; default, so save them synchronously to be able to assert on them right away.+ (mt/with-temporary-setting-values [synchronous-batch-updates true]+ (let [before (t2/count :model/QueryExecution)]+ (mt/user-http-request :crowberto :post 202 "agent/v1/execute-sql"+ {:database_id (mt/id)+ :sql "SELECT 1 AS audit_probe"})+ (let [latest (u/poll {:thunk (fn [] (t2/select-one :model/QueryExecution+ {:order-by [[:started_at :desc]]}))+ :done? (fn [_qe] (> (t2/count :model/QueryExecution) before))+ :timeout-ms 5000+ :interval-ms 50})]+ (is (some? latest) "QueryExecution row should be inserted within 5s")+ (is (= :agent (:context latest))))))));;; ------------------------------------------------- Read Resource Tests -----------------------------------------diff --git a/test/metabase/documents/view_log_test.clj b/test/metabase/documents/view_log_test.cljindex 763ddd81900..4a63cee4211 100644--- a/test/metabase/documents/view_log_test.clj+++ b/test/metabase/documents/view_log_test.clj@@ -1,14 +1,19 @@-(ns metabase.documents.view-log-test+(ns ^:synchronous metabase.documents.view-log-test(:require[clojure.test :refer :all][java-time.api :as t][metabase.documents.view-log :as documents.view-log][metabase.events.core :as events][metabase.test :as mt]+ [methodical.core :as methodical][toucan2.core :as t2]))(set! *warn-on-reflection* true)+(use-fixtures :each (fn [thunk]+ (mt/with-temporary-setting-values [synchronous-batch-updates true]+ (thunk))))+(defn latest-view"Returns the most recent view for a given user and document ID"[user-id document-id]@@ -109,13 +114,20 @@:document "{\"type\":\"doc\",\"content\":[]}":creator_id (:id user)}](let [events-published (atom [])]- (mt/with-dynamic-fn-redefs [events/publish-event! (fn [topic event]- (swap! events-published conj topic)- event)]+ (methodical/add-aux-method-with-unique-key!+ #'events/publish-event! :before :default+ (fn [topic _event]+ (swap! events-published conj topic))+ ::publish-event-spy)+ (try(#'documents.view-log/update-document-last-viewed-at!*[{:id (:id document) :timestamp (t/offset-date-time)}])(is (not (contains? (set @events-published) :event/document-update))- "updating last_viewed_at should not publish :event/document-update"))))))+ "updating last_viewed_at should not publish :event/document-update")+ (finally+ (methodical/remove-aux-method-with-unique-key!+ #'events/publish-event! :before :default+ ::publish-event-spy)))))))(deftest document-event-derivation-test(testing "Document events are properly derived from base events"diff --git a/test/metabase/eid_translation/util_test.clj b/test/metabase/eid_translation/util_test.cljindex d81ef2e8249..7d1d92778ed 100644--- a/test/metabase/eid_translation/util_test.clj+++ b/test/metabase/eid_translation/util_test.clj@@ -22,30 +22,31 @@@#'eid-translation.util/api-name->model))))(deftest ->id-test- (#'stats/clear-translation-count!)- (is (= (assoc eid-translation/default-counter :total 0)- (#'stats/get-translation-count)))- (mt/with-temp [:model/Card {card-id :id card-eid :entity_id} {}]- (is (= card-id (eid-translation.util/->id :card card-id)))- (is (= card-id (eid-translation.util/->id :model/Card card-id)))- (is (partial= {:ok 0 :total 0} (#'stats/get-translation-count))- "Translations are not counted when they don't occur")+ (mt/with-temporary-setting-values [synchronous-batch-updates true](#'stats/clear-translation-count!)- (is (= card-id (eid-translation.util/->id :card card-eid)))- (is (= card-id (eid-translation.util/->id :model/Card card-eid)))- (is (partial= {:ok 2 :total 2} (#'stats/get-translation-count))- "Translations are counted when they do occur")- (#'stats/clear-translation-count!))- (let [samples (t2/select-fn->fn :id :entity_id [:model/Card :id :entity_id] {:limit 100})]- (when (seq samples)- (doseq [[card-id entity-id] samples]- (testing (str "card-id: " card-id " entity-id: " entity-id)- (is (= card-id (eid-translation.util/->id :model/Card card-id)))- (is (= card-id (eid-translation.util/->id :card card-id)))- (is (= card-id (eid-translation.util/->id :model/Card entity-id)))- (is (= card-id (eid-translation.util/->id :card entity-id)))))- (is (malli= [:map [:ok pos-int?] [:total pos-int?]]- (#'stats/get-translation-count))))))+ (is (= (assoc eid-translation/default-counter :total 0)+ (#'stats/get-translation-count)))+ (mt/with-temp [:model/Card {card-id :id card-eid :entity_id} {}]+ (is (= card-id (eid-translation.util/->id :card card-id)))+ (is (= card-id (eid-translation.util/->id :model/Card card-id)))+ (is (partial= {:ok 0 :total 0} (#'stats/get-translation-count))+ "Translations are not counted when they don't occur")+ (#'stats/clear-translation-count!)+ (is (= card-id (eid-translation.util/->id :card card-eid)))+ (is (= card-id (eid-translation.util/->id :model/Card card-eid)))+ (is (partial= {:ok 2 :total 2} (#'stats/get-translation-count))+ "Translations are counted when they do occur")+ (#'stats/clear-translation-count!))+ (let [samples (t2/select-fn->fn :id :entity_id [:model/Card :id :entity_id] {:limit 100})]+ (when (seq samples)+ (doseq [[card-id entity-id] samples]+ (testing (str "card-id: " card-id " entity-id: " entity-id)+ (is (= card-id (eid-translation.util/->id :model/Card card-id)))+ (is (= card-id (eid-translation.util/->id :card card-id)))+ (is (= card-id (eid-translation.util/->id :model/Card entity-id)))+ (is (= card-id (eid-translation.util/->id :card entity-id)))))+ (is (malli= [:map [:ok pos-int?] [:total pos-int?]]+ (#'stats/get-translation-count)))))))(deftest ^:parallel entity-id-single-card-translations-test(mt/with-tempdiff --git a/test/metabase/lib/metadata/composed_provider_test.cljc b/test/metabase/lib/metadata/composed_provider_test.cljcindex 0737d48cc7e..b5ecb9d64d9 100644--- a/test/metabase/lib/metadata/composed_provider_test.cljc+++ b/test/metabase/lib/metadata/composed_provider_test.cljc@@ -54,4 +54,4 @@(lib.metadata/field mp (meta/id :orders :tax)))))(testing "Fetching by Name"(is (=? [{:active false}]- (lib.metadata.protocols/metadatas mp {:lib/type :metadata/column, :table-id (meta/id :orders), :name #{"TAX"}})))))))+ (lib.metadata.protocols/metadatas mp {:lib/type :metadata/column, :table-ids #{(meta/id :orders)}, :name #{"TAX"}})))))))diff --git a/test/metabase/lib/test_metadata/graph_provider.cljc b/test/metabase/lib/test_metadata/graph_provider.cljcindex 543b1d5d1d2..752bf9df058 100644--- a/test/metabase/lib/test_metadata/graph_provider.cljc+++ b/test/metabase/lib/test_metadata/graph_provider.cljc@@ -17,17 +17,17 @@(:tables metadata-graph)))(defn- graph-metadatas- [metadata-graph {metadata-type :lib/type, :keys [table-id], :as metadata-spec}]+ [metadata-graph {metadata-type :lib/type, :keys [table-ids], :as metadata-spec}](let [objects (case metadata-type:metadata/table (:tables metadata-graph)- :metadata/column (if table-id- (:fields (find-table metadata-graph table-id))+ :metadata/column (if table-ids+ (mapcat #(:fields (find-table metadata-graph %)) table-ids)(mapcat :fields (:tables metadata-graph)))- :metadata/metric (if table-id- (:metrics (find-table metadata-graph table-id))+ :metadata/metric (if table-ids+ (mapcat #(:metrics (find-table metadata-graph %)) table-ids)(mapcat :metrics (:tables metadata-graph)))- :metadata/segment (if table-id- (:segments (find-table metadata-graph table-id))+ :metadata/segment (if table-ids+ (mapcat #(:segments (find-table metadata-graph %)) table-ids)(mapcat :segments (:tables metadata-graph)))#_else;; not implemented for the simple graph metadata provider.diff --git a/test/metabase/lib_be/metadata/jvm_test.clj b/test/metabase/lib_be/metadata/jvm_test.cljindex 4a515671a49..e4c65b8978c 100644--- a/test/metabase/lib_be/metadata/jvm_test.clj+++ b/test/metabase/lib_be/metadata/jvm_test.clj@@ -249,16 +249,16 @@{:lib/type :metadata/table, :name #{"Table"}}{:lib/type :metadata/column, :id #{1}}{:lib/type :metadata/column, :name #{"Field"}}- {:lib/type :metadata/column, :table-id 1}+ {:lib/type :metadata/column, :table-ids #{1}}{:lib/type :metadata/card, :id #{1}}{:lib/type :metadata/card, :name #{"Card"}}{:lib/type :metadata/metric, :id #{1}}{:lib/type :metadata/metric, :name #{"Metric"}}- {:lib/type :metadata/metric, :table-id 1}- {:lib/type :metadata/metric, :card-id 1}+ {:lib/type :metadata/metric, :table-ids #{1}}+ {:lib/type :metadata/metric, :card-ids #{1}}{:lib/type :metadata/segment, :id #{1}}{:lib/type :metadata/segment, :name #{"Segment"}}- {:lib/type :metadata/segment, :table-id 1}+ {:lib/type :metadata/segment, :table-ids #{1}}{:lib/type :metadata/native-query-snippet, :id #{1}}{:lib/type :metadata/native-query-snippet, :name #{"Snippet"}}))))diff --git a/test/metabase/lib_metric/filter_test.cljc b/test/metabase/lib_metric/filter_test.cljcindex 1cacea0cbc4..3e506256d11 100644--- a/test/metabase/lib_metric/filter_test.cljc+++ b/test/metabase/lib_metric/filter_test.cljc@@ -823,12 +823,15 @@(let [wanted (or id #{1 2})](filterv #(contains? wanted (:id %)) [seg-source-metric-1 seg-source-metric-2])))-(defn- seg-segment-fetcher [{:keys [table-id]}]- (case table-id- 10 [seg-source-segment]- 20 [seg-joined-segment]- 30 [seg-second-source-segment]- []))+(defn- seg-segment-fetcher [{table-ids :table-ids}]+ (into []+ (mapcat (fn [table-id]+ (case table-id+ 10 [seg-source-segment]+ 20 [seg-joined-segment]+ 30 [seg-second-source-segment]+ [])))+ table-ids))(def ^:private seg-mock-provider(lib-metric.metadata.provider/metric-context-metadata-providerdiff --git a/test/metabase/lib_metric/metadata/jvm_test.clj b/test/metabase/lib_metric/metadata/jvm_test.cljindex 4ea1e1b0166..e59ba10468a 100644--- a/test/metabase/lib_metric/metadata/jvm_test.clj+++ b/test/metabase/lib_metric/metadata/jvm_test.clj@@ -40,7 +40,7 @@:query {:source-table (mt/id :orders):aggregation [[:count]]}}}](let [mp (metric-jvm/metadata-provider)- metrics (lib.metadata.protocols/metadatas mp {:lib/type :metadata/metric :table-id (mt/id :orders)})]+ metrics (lib.metadata.protocols/metadatas mp {:lib/type :metadata/metric :table-ids #{(mt/id :orders)}})](is (some #(= (:id metric) (:id %)) metrics))(is (every? #(= (mt/id :orders) (:table-id %)) metrics))))))@@ -66,7 +66,7 @@:query {:source-table (mt/id :orders):aggregation [[:count]]}}}](let [mp (metric-jvm/metadata-provider)- metrics (lib.metadata.protocols/metadatas mp {:lib/type :metadata/metric :table-id (mt/id :orders)})]+ metrics (lib.metadata.protocols/metadatas mp {:lib/type :metadata/metric :table-ids #{(mt/id :orders)}})](is (some #(= (:id active-metric) (:id %)) metrics))(is (not-any? #(= (:id archived-metric) (:id %)) metrics))))))@@ -90,7 +90,7 @@(deftest ^:parallel metadatas-routes-columns-to-database-provider-test(testing "metadatas should route column requests to the appropriate database provider"(let [mp (metric-jvm/metadata-provider)- columns (lib.metadata.protocols/metadatas mp {:lib/type :metadata/column :table-id (mt/id :orders)})]+ columns (lib.metadata.protocols/metadatas mp {:lib/type :metadata/column :table-ids #{(mt/id :orders)}})](is (seq columns))(is (every? #(= (mt/id :orders) (:table-id %)) columns)))))diff --git a/test/metabase/lib_metric/metadata/provider_test.cljc b/test/metabase/lib_metric/metadata/provider_test.cljcindex 1ea975f4e17..e24ddf966aa 100644--- a/test/metabase/lib_metric/metadata/provider_test.cljc+++ b/test/metabase/lib_metric/metadata/provider_test.cljc@@ -37,7 +37,7 @@(defn- mock-metric-fetcher"Mock metric fetcher that filters mock-metrics based on spec."- [{id-set :id, name-set :name, :keys [table-id card-id], :as _spec}]+ [{id-set :id, name-set :name, :keys [table-ids card-ids], :as _spec}](let [active-only? (not (or id-set name-set))](into [](comp@@ -47,12 +47,12 @@(if name-set(filter #(contains? name-set (:name %)))identity)- (if table-id- (filter #(and (= (:table-id %) table-id)+ (if table-ids+ (filter #(and (contains? table-ids (:table-id %))(nil? (:source-card-id %))))identity)- (if card-id- (filter #(= (:source-card-id %) card-id))+ (if card-ids+ (filter #(contains? card-ids (:source-card-id %)))identity)(if active-only?(filter #(not (:archived %)))@@ -69,7 +69,7 @@(reify lib.metadata.protocols/MetadataProvider(database [_this]{:id db-id :name (str "DB-" db-id) :lib/type :metadata/database})- (metadatas [_this {metadata-type :lib/type, id-set :id, :keys [table-id], :as _spec}]+ (metadatas [_this {metadata-type :lib/type, id-set :id, :keys [table-ids], :as _spec}](case metadata-type:metadata/table(if id-set@@ -81,8 +81,8 @@(filter #(= (:db-id %) db-id))(vals mock-tables))):metadata/column- (when table-id- (get mock-columns table-id []))+ (when table-ids+ (into [] (mapcat #(get mock-columns % [])) table-ids))[]))(setting [_this k](get mock-settings k))))@@ -122,7 +122,7 @@(deftest ^:parallel metadatas-fetches-metrics-by-table-id-test(testing "metadatas should fetch metrics by table-id (excludes card-based metrics)"(let [mp (create-mock-provider)- metrics (lib.metadata.protocols/metadatas mp {:lib/type :metadata/metric :table-id 10})]+ metrics (lib.metadata.protocols/metadatas mp {:lib/type :metadata/metric :table-ids #{10}})](is (= 2 (count metrics)))(is (every? #(= 10 (:table-id %)) metrics))(is (every? #(nil? (:source-card-id %)) metrics)))))@@ -130,7 +130,7 @@(deftest ^:parallel metadatas-fetches-metrics-by-card-id-test(testing "metadatas should fetch metrics by card-id"(let [mp (create-mock-provider)- metrics (lib.metadata.protocols/metadatas mp {:lib/type :metadata/metric :card-id 100})]+ metrics (lib.metadata.protocols/metadatas mp {:lib/type :metadata/metric :card-ids #{100}})](is (= 1 (count metrics)))(is (= 100 (:source-card-id (first metrics)))))))@@ -145,7 +145,7 @@(deftest ^:parallel metadatas-routes-columns-to-db-provider-test(testing "metadatas should route column requests to the appropriate database provider"(let [mp (create-mock-provider)- columns (lib.metadata.protocols/metadatas mp {:lib/type :metadata/column :table-id 10})]+ columns (lib.metadata.protocols/metadatas mp {:lib/type :metadata/column :table-ids #{10}})](is (= 2 (count columns)))(is (every? #(= 10 (:table-id %)) columns)))))diff --git a/test/metabase/metrics/api_test.clj b/test/metabase/metrics/api_test.cljindex 55e6b2f9074..f45623566af 100644--- a/test/metabase/metrics/api_test.clj+++ b/test/metabase/metrics/api_test.clj@@ -4,7 +4,6 @@[clojure.test :refer :all][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]- [metabase.query-processor.util :as qp.util][metabase.test :as mt][metabase.test.fixtures :as fixtures][toucan2.core :as t2]))@@ -350,7 +349,7 @@(deftest dataset-leaf-records-query-execution-test(testing "POST /api/metric/dataset (leaf path) writes a QueryExecution row with :context :metric"(mt/test-helpers-set-global-values!- (binding [qp.util/*execute-async?* false]+ (mt/with-temporary-setting-values [synchronous-batch-updates true](mt/with-temp [:model/Card metric {:name "QE Leaf Metric":type :metric:dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}]@@ -366,7 +365,7 @@(deftest dataset-arithmetic-records-one-qe-per-leaf-test(testing "POST /api/metric/dataset (arithmetic) writes one QueryExecution row per leaf"(mt/test-helpers-set-global-values!- (binding [qp.util/*execute-async?* false]+ (mt/with-temporary-setting-values [synchronous-batch-updates true](mt/with-temp [:model/Card metric-a {:name "QE Arith A":type :metric:dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}@@ -388,7 +387,7 @@(deftest breakout-values-records-query-execution-test(testing "POST /api/metric/breakout-values writes a QueryExecution row with :context :metric"(mt/test-helpers-set-global-values!- (binding [qp.util/*execute-async?* false]+ (mt/with-temporary-setting-values [synchronous-batch-updates true](mt/with-temp [:model/Card metric {:name "QE Breakout Metric":type :metric:dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}]diff --git a/test/metabase/queries_rest/api/card_test.clj b/test/metabase/queries_rest/api/card_test.cljindex 3ceabf56cec..5a2e5607e73 100644--- a/test/metabase/queries_rest/api/card_test.clj+++ b/test/metabase/queries_rest/api/card_test.clj@@ -38,7 +38,6 @@[metabase.query-processor.compile :as qp.compile][metabase.query-processor.middleware.constraints :as qp.constraints][metabase.query-processor.pivot.test-util :as api.pivots]- [metabase.query-processor.util :as qp.util][metabase.revisions.models.revision :as revision][metabase.test :as mt][metabase.test.data.users :as test.users]@@ -308,16 +307,18 @@version-string (str "1." (rand-int 1000) "." (rand-int 1000))](mt/with-temp [:model/Card card-1 {:name "Card 1":dataset_query (mt/mbql-query venues {:limit 1})}]- (mt/with-premium-features #{:audit-app}- (mt/user-http-request :crowberto :post 202 (str "card/" (u/the-id card-1) "/query")- {:request-options {:headers {"x-metabase-client" client-string- "x-metabase-client-version" version-string}}}))+ (mt/with-temporary-setting-values [synchronous-batch-updates true]+ (mt/with-premium-features #{:audit-app}+ (mt/user-http-request :crowberto :post 202 (str "card/" (u/the-id card-1) "/query")+ {:request-options {:headers {"x-metabase-client" client-string+ "x-metabase-client-version" version-string}}})))(is (= {:embedding_client client-string, :embedding_sdk_version version-string}(t2/select-one [:model/ViewLog :embedding_client :embedding_sdk_version] :model "card" :model_id (u/the-id card-1))))))))(deftest embedding-sdk-info-saves-query-execution-test(testing "GET /api/card with embedding headers set"- (binding [qp.util/*execute-async?* false]+ ;; QueryExecutions are saved in async batches by default; save them synchronously so we can assert on them+ (mt/with-temporary-setting-values [synchronous-batch-updates true](mt/with-temp [:model/Card card-1 {:name "Card 1";; This query is just to make sure the card actually runs a query, otherwise;; there won't be a QueryExecution record to check!diff --git a/test/metabase/query_processor/api_test.clj b/test/metabase/query_processor/api_test.cljindex 2741b16fca3..56dd7227422 100644--- a/test/metabase/query_processor/api_test.clj+++ b/test/metabase/query_processor/api_test.clj@@ -84,115 +84,118 @@(deftest basic-query-test(testing "POST /api/dataset"(testing "\nJust a basic sanity check to make sure Query Processor endpoint is still working correctly."- (let [query (mt/mbql-query checkins- {:aggregation [[:count]]})- result (mt/user-http-request :crowberto :post 202 "dataset" query)]- (testing "\nAPI Response"- (is (=?- {:data {:rows [[1000]]- :cols [(mt/obj->json->obj (qp.test-util/aggregate-col :count))]- :native_form true- :results_timezone "UTC"}- :row_count 1- :status "completed"- :context "ad-hoc"- :json_query (-> (mt/mbql-query checkins- {:aggregation [[:count]]})- (assoc-in [:query :aggregation] [["count"]])- (assoc :type "query")- (merge query-defaults))- :started_at true- :running_time true- :average_execution_time nil- :database_id (mt/id)}- (format-response result))))- (testing "\nSaved QueryExecution"- (is (= {:hash true- :row_count 1- :result_rows 1- :context :ad-hoc- :executor_id (mt/user->id :crowberto)- :native false- :pulse_id nil- :card_id nil- :is_sandboxed false- :dashboard_id nil- :transform_id nil- :lens_id nil- :lens_params nil- :error nil- :id true- :action_id nil- :cache_hit false- :cache_hash false- :parameterized false- :database_id (mt/id)- :started_at true- :running_time true- :embedding_client nil- :embedding_hostname nil- :embedding_path nil- :embedding_route nil- :embedding_sdk_version nil- :metabase_version true- :auth_method true- :ip_address nil- :is_db_routed false- :is_impersonated false- :parameters nil- :tenant_id nil- :user_agent nil- :sanitized_user_agent nil}- (format-response (most-recent-query-execution-for-query query)))))))))+ (mt/with-temporary-setting-values [synchronous-batch-updates true]+ (let [query (mt/mbql-query checkins+ {:aggregation [[:count]]})+ result (mt/user-http-request :crowberto :post 202 "dataset" query)]+ (testing "\nAPI Response"+ (is (=?+ {:data {:rows [[1000]]+ :cols [(mt/obj->json->obj (qp.test-util/aggregate-col :count))]+ :native_form true+ :results_timezone "UTC"}+ :row_count 1+ :status "completed"+ :context "ad-hoc"+ :json_query (-> (mt/mbql-query checkins+ {:aggregation [[:count]]})+ (assoc-in [:query :aggregation] [["count"]])+ (assoc :type "query")+ (merge query-defaults))+ :started_at true+ :running_time true+ :average_execution_time nil+ :database_id (mt/id)}+ (format-response result))))+ (testing "\nSaved QueryExecution"+ (is (= {:hash true+ :row_count 1+ :result_rows 1+ :context :ad-hoc+ :executor_id (mt/user->id :crowberto)+ :native false+ :pulse_id nil+ :card_id nil+ :is_sandboxed false+ :dashboard_id nil+ :transform_id nil+ :lens_id nil+ :lens_params nil+ :error nil+ :id true+ :action_id nil+ :cache_hit false+ :cache_hash false+ :parameterized false+ :database_id (mt/id)+ :started_at true+ :running_time true+ :embedding_client nil+ :embedding_hostname nil+ :embedding_path nil+ :embedding_route nil+ :embedding_sdk_version nil+ :metabase_version true+ :auth_method true+ :ip_address nil+ :is_db_routed false+ :is_impersonated false+ :parameters nil+ :tenant_id nil+ :user_agent nil+ :sanitized_user_agent nil}+ (format-response (most-recent-query-execution-for-query query))))))))))(deftest failure-test;; clear out recent query executions!(t2/delete! :model/QueryExecution)(testing "POST /api/dataset"(testing "\nA failed query should return a 400 response from the API"- ;; Error message's format can differ a bit depending on DB version and the comment we prepend to it, so check- ;; that it exists and contains the substring "Syntax error in SQL statement"- (let [query {:database (mt/id)- :type "native"- :native {:query "foobar"}}- result (mt/user-http-request :crowberto :post 400 "dataset" query)]- (testing "\nAPI Response"- (is (malli= [:map- [:data [:map- [:rows [:= []]]- [:cols [:= []]]]]- [:row_count [:= 0]]- [:status [:= "failed"]]- [:context [:= "ad-hoc"]]- [:error #"Syntax error in SQL statement"]- [:json_query [:map- [:database [:= (mt/id)]]- [:type [:= "native"]]- [:native [:map- [:query [:= "foobar"]]]]- [:middleware [:map- [:add-default-userland-constraints? [:= true]]- [:js-int-to-string? [:= true]]]]]]- [:database_id [:= (mt/id)]]- [:state [:= "42000"]]- [:class [:= "class org.h2.jdbc.JdbcSQLSyntaxErrorException"]]]- result)))- (testing "\nSaved QueryExecution"- (is (malli=- [:map- [:hash (ms/InstanceOfClass (Class/forName "[B"))]- [:id ms/PositiveInt]- [:result_rows [:= 0]]- [:row_count [:= 0]]- [:context [:= :ad-hoc]]- [:error #"Syntax error in SQL statement"]- [:database_id [:= (mt/id)]]- [:executor_id [:= (mt/user->id :crowberto)]]- [:native [:= true]]- [:pulse_id nil?]- [:card_id nil?]- [:dashboard_id nil?]]- (most-recent-query-execution-for-query query))))))))+ ;; QueryExecutions are saved in async batches by default; save them synchronously so we can assert on them+ (mt/with-temporary-setting-values [synchronous-batch-updates true]+ ;; Error message's format can differ a bit depending on DB version and the comment we prepend to it, so check+ ;; that it exists and contains the substring "Syntax error in SQL statement"+ (let [query {:database (mt/id)+ :type "native"+ :native {:query "foobar"}}+ result (mt/user-http-request :crowberto :post 400 "dataset" query)]+ (testing "\nAPI Response"+ (is (malli= [:map+ [:data [:map+ [:rows [:= []]]+ [:cols [:= []]]]]+ [:row_count [:= 0]]+ [:status [:= "failed"]]+ [:context [:= "ad-hoc"]]+ [:error #"Syntax error in SQL statement"]+ [:json_query [:map+ [:database [:= (mt/id)]]+ [:type [:= "native"]]+ [:native [:map+ [:query [:= "foobar"]]]]+ [:middleware [:map+ [:add-default-userland-constraints? [:= true]]+ [:js-int-to-string? [:= true]]]]]]+ [:database_id [:= (mt/id)]]+ [:state [:= "42000"]]+ [:class [:= "class org.h2.jdbc.JdbcSQLSyntaxErrorException"]]]+ result)))+ (testing "\nSaved QueryExecution"+ (is (malli=+ [:map+ [:hash (ms/InstanceOfClass (Class/forName "[B"))]+ [:id ms/PositiveInt]+ [:result_rows [:= 0]]+ [:row_count [:= 0]]+ [:context [:= :ad-hoc]]+ [:error #"Syntax error in SQL statement"]+ [:database_id [:= (mt/id)]]+ [:executor_id [:= (mt/user->id :crowberto)]]+ [:native [:= true]]+ [:pulse_id nil?]+ [:card_id nil?]+ [:dashboard_id nil?]]+ (most-recent-query-execution-for-query query)))))))))(defn- test-download-response-headers[url]diff --git a/test/metabase/query_processor/middleware/cache_test.clj b/test/metabase/query_processor/middleware/cache_test.cljindex 0f2ce2c1eae..213b05a2d90 100644--- a/test/metabase/query_processor/middleware/cache_test.clj+++ b/test/metabase/query_processor/middleware/cache_test.clj@@ -104,7 +104,8 @@(defn do-with-mock-cache! [f](mt/with-open-channels [save-chan (a/chan 10)purge-chan (a/chan 10)]- (mt/with-temporary-setting-values [query-caching-max-ttl 60]+ (mt/with-temporary-setting-values [query-caching-max-ttl 60+ synchronous-batch-updates true](binding [cache/*backend* (test-backend save-chan purge-chan)*save-chan* save-chan*purge-chan* purge-chan]@@ -420,17 +421,17 @@update-avg-execution-count (atom 0)called-promise (promise)save-execution-metadata-original (mt/original-fn #'process-userland-query/save-execution-metadata!*)- save-query-update-avg-time-original (mt/original-fn #'query/save-query-and-update-average-execution-time!)]- ;; save-execution-metadata!* and save-query-and-update-average-execution-time! are invoked from+ save-query-update-avg-time-original (mt/original-fn #'query/save-queries-and-update-average-execution-times!)]+ ;; save-execution-metadata!* and save-queries-and-update-average-execution-times! are invoked from;; the QP pipeline on worker threads that don't inherit *local-redefs* — use with-redefs.#_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}- (with-redefs [process-userland-query/save-execution-metadata!* (fn [& args]- (swap! save-execution-metadata-count inc)- (apply save-execution-metadata-original args)- (deliver called-promise true))- query/save-query-and-update-average-execution-time! (fn [& args]- (swap! update-avg-execution-count inc)- (apply save-query-update-avg-time-original args))]+ (with-redefs [process-userland-query/save-execution-metadata!* (fn [& args]+ (swap! save-execution-metadata-count inc)+ (apply save-execution-metadata-original args)+ (deliver called-promise true))+ query/save-queries-and-update-average-execution-times! (fn [entries]+ (swap! update-avg-execution-count + (count entries))+ (save-query-update-avg-time-original entries))](let [query (assoc (mt/mbql-query venues {:order-by [[:asc $id]] :limit 42}):cache-strategy (assoc (ttl-strategy) :multiplier 5000))q-hash (qp.util/query-hash query)]diff --git a/test/metabase/query_processor/middleware/prefetch_metadata_test.clj b/test/metabase/query_processor/middleware/prefetch_metadata_test.cljnew file mode 100644index 00000000000..2839e906ca5--- /dev/null+++ b/test/metabase/query_processor/middleware/prefetch_metadata_test.clj@@ -0,0 +1,128 @@+(ns metabase.query-processor.middleware.prefetch-metadata-test+ ;; these tests count real app-DB calls, so they have to use the real application-database metadata provider (and+ ;; `with-temp` for source cards) rather than a mock metadata provider+ {:clj-kondo/config '{:linters {:discouraged-var {metabase.test/with-temp {:level :off}+ toucan2.tools.with-temp/with-temp {:level :off}}}}}+ (:require+ [clojure.test :refer :all]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.query-processor.preprocess :as qp.preprocess]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(defn- preprocess-app-db-call-count+ "Preprocess `query` and return the number of app-DB calls made. Binds a fresh metadata-provider cache, like the REST+ API does for each HTTP request."+ [query]+ (lib-be/with-metadata-provider-cache+ (t2/with-call-count [call-count]+ (qp.preprocess/preprocess (dissoc query :lib/metadata))+ (call-count))))++(deftest prefetch-metadata-mbql-one-table-call-count-test+ (mt/dataset test-data+ (let [mp (mt/metadata-provider)+ query (-> (lib/query mp (lib.metadata/table mp (mt/id :venues)))+ (lib/limit 1))]+ ;; the baseline:+ ;; - fetch the Database+ ;; - fetch the Tables in bulk+ ;; - fetch the Tables' columns in bulk+ ;; - the permissions check (EE only)+ ;; - the database-routing check (EE only)+ ;; - the table-remapping check (EE only)+ (is (<= (preprocess-app-db-call-count query) 6)))))++(deftest prefetch-metadata-mbql-three-tables-call-count-test+ (mt/dataset test-data+ (let [mp (mt/metadata-provider)+ query (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ (lib/join (lib/join-clause (lib.metadata/table mp (mt/id :products))+ [(lib/= (lib.metadata/field mp (mt/id :orders :product_id))+ (lib.metadata/field mp (mt/id :products :id)))]))+ (lib/join (lib/join-clause (lib.metadata/table mp (mt/id :people))+ [(lib/= (lib.metadata/field mp (mt/id :orders :user_id))+ (lib.metadata/field mp (mt/id :people :id)))]))+ (lib/limit 1))]+ ;; the same number of calls as for a single-table query: all the referenced tables and their columns are+ ;; bulk-loaded, so the count does not depend on the number of tables+ (is (<= (preprocess-app-db-call-count query) 6)))))++(deftest prefetch-metadata-native-one-field-filter-call-count-test+ (mt/dataset test-data+ (let [mp (mt/metadata-provider)+ query (-> (lib/native-query mp "SELECT * FROM VENUES WHERE {{ff1}}")+ (lib/with-template-tags+ {"ff1" {:type :dimension+ :name "ff1"+ :display-name "ff1"+ :id "id1"+ :dimension (lib/ref (lib.metadata/field mp (mt/id :venues :price)))+ :widget-type :id}})+ (assoc :parameters [{:type :id+ :target [:dimension [:template-tag "ff1"]]+ :value [1]}]))]+ ;; one more call than for MBQL queries: the fields referenced by the template tags are bulk-loaded by ID, which+ ;; is also how their tables are discovered+ (is (<= (preprocess-app-db-call-count query) 7)))))++(deftest prefetch-metadata-native-three-field-filters-call-count-test+ (mt/dataset test-data+ (let [mp (mt/metadata-provider)+ tag (fn [tag-name field-id]+ {:type :dimension+ :name tag-name+ :display-name tag-name+ :id tag-name+ :dimension (lib/ref (lib.metadata/field mp field-id))+ :widget-type :id})+ query (-> (lib/native-query mp "SELECT * FROM VENUES WHERE {{ff1}} AND {{ff2}} AND {{ff3}}")+ (lib/with-template-tags+ {"ff1" (tag "ff1" (mt/id :venues :price))+ "ff2" (tag "ff2" (mt/id :venues :category_id))+ "ff3" (tag "ff3" (mt/id :venues :name))})+ (assoc :parameters [{:type :id+ :target [:dimension [:template-tag "ff1"]]+ :value [1]}]))]+ ;; the same calls as with a single field filter: all the template-tag Fields are loaded with a single bulk+ ;; call, while without prefetching parameter substitution would resolve them one at a time+ (is (<= (preprocess-app-db-call-count query) 7)))))++(deftest prefetch-metadata-card-source-call-count-test+ (mt/dataset test-data+ (let [mp (mt/metadata-provider)]+ (mt/with-temp [:model/Card card {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :venues)))}]+ (let [query (-> (lib/query mp (lib.metadata/card mp (:id card)))+ (lib/limit 1))]+ ;; the baseline calls, plus:+ ;; - one bulk fetch of the referenced Cards; resolving the source card afterwards hits the cache, and the+ ;; card's table and columns are folded into the baseline bulk loads+ (is (<= (preprocess-app-db-call-count query) 7)))))))++(deftest prefetch-metadata-nested-card-source-call-count-test+ (mt/dataset test-data+ (let [mp (mt/metadata-provider)]+ (mt/with-temp [:model/Card inner-card {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :venues)))}]+ (mt/with-temp [:model/Card outer-card {:dataset_query (lib/query mp (lib.metadata/card mp (:id inner-card)))}]+ (let [query (-> (lib/query mp (lib.metadata/card mp (:id outer-card)))+ (lib/limit 1))]+ ;; one more call than for a single source card: referenced Cards are prefetched with one bulk fetch per+ ;; level of nesting, and both cards' tables and columns are folded into the same bulk loads+ (is (<= (preprocess-app-db-call-count query) 8))))))))++(deftest prefetch-metadata-card-source-with-join-call-count-test+ (mt/dataset test-data+ (let [mp (mt/metadata-provider)]+ (mt/with-temp [:model/Card card {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :venues)))}]+ (let [query (-> (lib/query mp (lib.metadata/card mp (:id card)))+ (lib/join (lib/join-clause (lib.metadata/table mp (mt/id :categories))+ [(lib/= (lib.metadata/field mp (mt/id :venues :category_id))+ (lib.metadata/field mp (mt/id :categories :id)))]))+ (lib/limit 1))]+ ;; the same calls as for a plain source card: the joined Table and its columns are folded into the same+ ;; bulk loads+ (is (<= (preprocess-app-db-call-count query) 7)))))))diff --git a/test/metabase/query_processor/middleware/process_userland_query_test.clj b/test/metabase/query_processor/middleware/process_userland_query_test.cljindex e6bac3abdbb..b63b03dcb83 100644--- a/test/metabase/query_processor/middleware/process_userland_query_test.clj+++ b/test/metabase/query_processor/middleware/process_userland_query_test.clj@@ -28,13 +28,17 @@result (promise)];; save-execution-metadata!* is invoked from the QP pipeline transducer, which runs on a thread;; that doesn't inherit *local-redefs* — use with-redefs so worker threads see the replacement.+ ;; `*execute-async?*` is redef'd (not bound) for the same reason: it makes the save happen synchronously+ ;; on whatever thread runs the query instead of going through the (batched, async) grouper queue.#_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}- (with-redefs [process-userland-query/save-execution-metadata!*- (fn [query-execution]- (when-let [^bytes qe-hash (:hash query-execution)]+ (with-redefs [qp.util/*execute-async?* false+ process-userland-query/save-execution-metadata!*+ (fn [query-executions]+ (doseq [{qe-hash :hash, :as query-execution} query-executions+ :when qe-hash](deliverresult- (if (java.util.Arrays/equals qe-hash original-hash)+ (if (java.util.Arrays/equals ^bytes qe-hash original-hash)query-execution;; if you're seeing this there is probably some;; bug that is causing query hashes to get@@ -44,7 +48,7 @@{:query query:original-hash (some-> original-hash codecs/bytes->hex):query-execution query-execution- :query-execution-hash (some-> qe-hash codecs/bytes->hex)+ :query-execution-hash (some-> ^bytes qe-hash codecs/bytes->hex):query-execution-query (:json_query query-execution)})))))](run(fn qe-result* []diff --git a/test/metabase/view_log/events/view_log_test.clj b/test/metabase/view_log/events/view_log_test.cljindex 87dd520abfe..3d38461c21f 100644--- a/test/metabase/view_log/events/view_log_test.clj+++ b/test/metabase/view_log/events/view_log_test.clj@@ -1,4 +1,4 @@-(ns metabase.view-log.events.view-log-test+(ns ^:synchronous metabase.view-log.events.view-log-test{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.view-log.events.view-log-test]}}}}}}(:require[clojure.test :refer :all]@@ -22,6 +22,10 @@(set! *warn-on-reflection* true)+(use-fixtures :each (fn [thunk]+ (mt/with-temporary-setting-values [synchronous-batch-updates true]+ (thunk))))+(defn latest-view"Returns the most recent view for a given user and model ID"[user-id model-id]