Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[perf] Drop `:dimensions` backfill from `/query_metadata` responses (#83960)

Braden Shepherdson303be6290c86
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 7e418f9790c..0dabd1ebcce 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -3373,6 +3373,7 @@
parameters.schema
permissions
premium-features
+ queries
query-processor
remote-sync
search
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.edn
index 96b2f42912e..b04d522bd53 100644
--- a/.clj-kondo/ratchets.edn
+++ b/.clj-kondo/ratchets.edn
@@ -25,7 +25,7 @@
:discouraged-java-method 4
:duplicate-require 1
:main-without-gen-class 1
- :metabase/discourage-dynamic-vars 324
+ :metabase/discourage-dynamic-vars 325
:metabase/discourage-millis-duration 1
:metabase/modules 23
:metabase/namespace-name 3
diff --git a/src/metabase/queries/card_schema.clj b/src/metabase/queries/card_schema.clj
index c046e931650..9912bd4bc15 100644
--- a/src/metabase/queries/card_schema.clj
+++ b/src/metabase/queries/card_schema.clj
@@ -49,3 +49,28 @@
([] base-selection)
([extras]
(into base-selection extras)))
+
+#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}
+(def ^:dynamic *skip-dimension-backfill?*
+ "When true, the `:card_schema` 24 upgrade leaves a pre-curation metric's `:dimensions` and
+ `:dimension_mappings` exactly as stored instead of backfilling them.
+
+ Bind it around a read whose caller does not look at a metric's dimensions -- one that wants an id, a name
+ and a link. The backfill builds the metric's full implicitly-joined dimension set, which costs time and
+ allocation proportional to (metrics x columns); hydrating a wide table's metrics for `query_metadata` did
+ that once per metric and discarded all of it, taking minutes and exhausting the heap (#83937).
+
+ Deliberately narrow:
+
+ - **Only upgrade 24's backfill is skipped.** The upgrades to 21, 22 and 23 still run, so `:result_metadata`
+ is still modernized, and `:dataset_query` is still normalized to current MBQL.
+
+ - **`:card_schema` is still stamped to the current version.** The returned row therefore claims a vintage
+ whose dimensions it is not carrying. That is in-memory only -- no read path writes it back -- and it
+ cannot mislead `t2/update!`, which decides what to upgrade from the row's *stored* `card_schema` (read
+ with the upgrades suppressed), never from an incoming card.
+
+ - **Do not bind it around a write.** Writes legitimately need the real backfill. Note also that feeding a
+ whole card read under this flag into `t2/update!` would put `:dimensions nil` into the changes and clobber
+ the stored dimension set -- true of any card read whose `:dimensions` you pass back, but worth knowing."
+ false)
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index 42686acba1c..89181c75e95 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -692,6 +692,10 @@
(defmethod upgrade-card-schema-to 24
[{:keys [dataset_query dimensions entity_id] :as card} _schema-version]
(cond
+ ;; The caller declared it does not read a metric's dimensions, so don't pay to backfill them.
+ ;; See [[card-schema/*skip-dimension-backfill?*]].
+ card-schema/*skip-dimension-backfill?* card
+
(not= :metric (keyword (:type card))) card ; Ignore non-:metric cards
(empty? dataset_query) card ; And those without real queries
diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.clj
index 35cdca1c661..169e5a18b81 100644
--- a/src/metabase/warehouse_schema/models/table.clj
+++ b/src/metabase/warehouse_schema/models/table.clj
@@ -9,6 +9,7 @@
[metabase.models.serialization :as serdes]
[metabase.permissions.core :as perms]
[metabase.premium-features.core :refer [defenterprise]]
+ [metabase.queries.card-schema :as card-schema]
[metabase.remote-sync.core :as remote-sync]
[metabase.search.spec :as search.spec]
[metabase.util :as u]
@@ -530,11 +531,18 @@
(mi/define-batched-hydration-method with-metrics
:metrics
- "Efficiently hydrate the Metrics for a collection of `tables`."
+ "Efficiently hydrate the Metrics for a collection of `tables`.
+
+ Consumers of a Table's `:metrics` want an id, a name and a link, never a metric's `:dimensions`, so the
+ `card_schema` 24 backfill is skipped. Without that the upgrade builds the whole implicitly-joined dimension set
+ for every pre-curation metric on these tables -- work proportional to (metrics x columns), then discarded --
+ which is minutes of CPU and enough garbage to exhaust the heap on a wide table with many old metrics (#83937).
+ Every other `card_schema` upgrade still runs."
[tables]
(with-objects :metrics
(fn [table-ids]
- (->> (warehouse-schema.db/unarchived-metric-cards-for-tables table-ids)
+ (->> (binding [card-schema/*skip-dimension-backfill?* true]
+ (warehouse-schema.db/unarchived-metric-cards-for-tables table-ids))
(filter mi/can-read?)))
tables))
diff --git a/test/metabase/queries/models/card_test.clj b/test/metabase/queries/models/card_test.clj
index ec2504ed0b0..9df7f7df9a9 100644
--- a/test/metabase/queries/models/card_test.clj
+++ b/test/metabase/queries/models/card_test.clj
@@ -1,6 +1,7 @@
(ns metabase.queries.models.card-test
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.queries.models.card-test]}}}}}}
(:require
+ [clojure.string :as str]
[clojure.test :refer :all]
[java-time.api :as t]
[metabase.api.common :as api]
@@ -2240,3 +2241,41 @@
(t2/update! :model/Card (:id card) {:visualization_settings {:timeline.selected_timeline_ids []}})
(is (= [] (get-in (t2/select-one :model/Card (:id card))
[:visualization_settings :timeline.selected_timeline_ids])))))))
+
+(deftest skip-dimension-backfill-skips-only-upgrade-24-test
+ (testing "card-schema/*skip-dimension-backfill?* skips the 24 dimension backfill and nothing else (#83937)"
+ (mt/with-temp [:model/Card {card-id :id} {:type :metric
+ :database_id (mt/id)
+ :table_id (mt/id :venues)
+ :dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}]
+ ;; A pre-curation metric: schema 23, no stored dimensions, and a `:result_metadata` still carrying one of
+ ;; the abandoned `:ident`s that the upgrade to 22 exists to strip. Written raw because before-insert
+ ;; forces `:card_schema` to current and normalizes the card.
+ (t2/query-one {:update :report_card
+ :set {:card_schema 23
+ :dimensions nil
+ :dimension_mappings nil
+ :result_metadata (json/encode [{:name "count"
+ :ident "abandoned-ident"
+ :base_type "type/Integer"}])}
+ :where [:= :id card-id]})
+ (testing "precondition: the stored row really does carry the stale ident"
+ (is (str/includes? (:result_metadata (t2/query-one {:select [:result_metadata]
+ :from [:report_card]
+ :where [:= :id card-id]}))
+ "abandoned-ident")))
+ (testing "without the flag the backfill runs"
+ (is (seq (:dimensions (t2/select-one :model/Card card-id)))))
+ (binding [card-schema/*skip-dimension-backfill?* true]
+ (let [card (t2/select-one :model/Card card-id)]
+ (testing "the expensive 24 backfill is skipped"
+ (is (nil? (:dimensions card)))
+ (is (nil? (:dimension_mappings card))))
+ (testing "the other upgrades still run -- 22 strips the abandoned :ident"
+ (is (every? #(not (contains? % :ident)) (:result_metadata card))))
+ (testing ":dataset_query is still normalized to current MBQL"
+ (is (=? {:lib/type :mbql/query} (:dataset_query card))))
+ (testing "and the row still reports the current schema version"
+ (is (= @#'card/current-schema-version (:card_schema card))))))
+ (testing "the stored row is untouched by any of this"
+ (is (= 23 (stored-card-schema card-id)))))))
diff --git a/test/metabase/warehouse_schema_rest/api/table_test.clj b/test/metabase/warehouse_schema_rest/api/table_test.clj
index cbe5ef2895b..f6e13342d5f 100644
--- a/test/metabase/warehouse_schema_rest/api/table_test.clj
+++ b/test/metabase/warehouse_schema_rest/api/table_test.clj
@@ -12,6 +12,7 @@
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.util :as driver.u]
[metabase.events.core :as events]
+ [metabase.lib-metric.dimension.jvm :as lib-metric.dimension.jvm]
[metabase.lib.core :as lib]
[metabase.permissions.models.data-permissions :as data-perms]
[metabase.permissions.models.permissions :as perms]
@@ -1606,3 +1607,69 @@
{:ids [t1 t2] :visibility_type nil})
(is (= [nil nil]
(map #(user-table-fn :visibility_type %) [t1 t2]))))))
+
+;;; ---------------------------------- #83937 dimension-backfill canary -----------------------------------
+
+(defn- counting-dimension-pairs
+ "Calls `thunk` with [[lib-metric.dimension.jvm/compute-dimension-pairs]] counted, returning
+ `[result call-count]`.
+
+ That function is the expensive primitive behind the `card_schema` 24 dimension backfill: it runs
+ `lib/visible-columns` over the metric's table *plus* every FK-reachable table and fetches every one of those
+ Fields. Running it once per metric is what made `query_metadata` for a wide table take minutes and exhaust the
+ heap in #83937, so these tests assert it never runs at all on that path."
+ [thunk]
+ (let [calls (atom 0)
+ orig lib-metric.dimension.jvm/compute-dimension-pairs]
+ (mt/with-dynamic-fn-redefs [lib-metric.dimension.jvm/compute-dimension-pairs
+ (fn [& args]
+ (clojure.core/swap! calls inc)
+ (apply orig args))]
+ [(thunk) @calls])))
+
+(defn- make-pre-curation-metric!
+ "Put `card-id` back into the pre-64 shape: `card_schema` 23 with no stored dimensions, so reading it is what
+ triggers the backfill. Needs a raw UPDATE because before-insert forces `:card_schema` to current."
+ [card-id]
+ (t2/query-one {:update :report_card
+ :set {:card_schema 23, :dimensions nil, :dimension_mappings nil}
+ :where [:= :id card-id]}))
+
+(deftest query-metadata-does-not-backfill-metric-dimensions-test
+ (testing "query_metadata must not run the pre-64 dimension backfill for a table's metrics (#83937)"
+ (mt/with-temp [:model/Card {metric-id :id} {:type :metric
+ :database_id (mt/id)
+ :table_id (mt/id :venues)
+ :dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}
+ :model/Card {question-id :id} {:database_id (mt/id)
+ :table_id (mt/id :venues)
+ :dataset_query (mt/mbql-query venues)}
+ :model/Dashboard {dash-id :id} {}
+ :model/DashboardCard _ {:dashboard_id dash-id, :card_id question-id}]
+ (make-pre-curation-metric! metric-id)
+ ;; Without this the whole test could pass vacuously: if the fixture ever stopped being a genuine
+ ;; pre-curation metric, nothing would backfill and the zero-call assertions below would be meaningless.
+ (testing "the fixture really is a metric whose dimensions get backfilled on an ordinary read"
+ (is (pos? (second (counting-dimension-pairs
+ #(t2/select-one :model/Card :id metric-id))))))
+ (testing "GET /api/table/:id/query_metadata"
+ (let [[response calls] (counting-dimension-pairs
+ #(mt/user-http-request :crowberto :get 200
+ (format "table/%d/query_metadata" (mt/id :venues))))
+ metrics (:metrics response)]
+ (is (zero? calls)
+ "the dimension backfill ran while hydrating a table's metrics")
+ (is (some #(= metric-id (:id %)) metrics)
+ "the metric should still be hydrated, just without its backfilled dimensions")
+ (is (every? (comp nil? :dimensions) metrics)
+ "no metric should come back carrying a backfilled dimension set")))
+ (testing "GET /api/dashboard/:id/query_metadata"
+ (let [[_response calls] (counting-dimension-pairs
+ #(mt/user-http-request :crowberto :get 200
+ (format "dashboard/%d/query_metadata" dash-id)))]
+ (is (zero? calls)
+ "the dimension backfill ran while loading dashboard query metadata")))
+ (testing "the stored row is untouched — still schema 23, so a later write still upgrades it"
+ (is (= 23 (:card_schema (t2/query-one {:select [:card_schema]
+ :from [:report_card]
+ :where [:= :id metric-id]}))))))))