The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[perf] Drop `:dimensions` backfill from `/query_metadata` responses (#83960)
Braden Shepherdson303be6290c86
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 7e418f9790c..0dabd1ebcce 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -3373,6 +3373,7 @@parameters.schemapermissionspremium-features+ queriesquery-processorremote-syncsearchdiff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.ednindex 96b2f42912e..b04d522bd53 100644--- a/.clj-kondo/ratchets.edn+++ b/.clj-kondo/ratchets.edn@@ -25,7 +25,7 @@:discouraged-java-method 4:duplicate-require 1:main-without-gen-class 1- :metabase/discourage-dynamic-vars 324+ :metabase/discourage-dynamic-vars 325:metabase/discourage-millis-duration 1:metabase/modules 23:metabase/namespace-name 3diff --git a/src/metabase/queries/card_schema.clj b/src/metabase/queries/card_schema.cljindex c046e931650..9912bd4bc15 100644--- a/src/metabase/queries/card_schema.clj+++ b/src/metabase/queries/card_schema.clj@@ -49,3 +49,28 @@([] base-selection)([extras](into base-selection extras)))++#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}+(def ^:dynamic *skip-dimension-backfill?*+ "When true, the `:card_schema` 24 upgrade leaves a pre-curation metric's `:dimensions` and+ `:dimension_mappings` exactly as stored instead of backfilling them.++ Bind it around a read whose caller does not look at a metric's dimensions -- one that wants an id, a name+ and a link. The backfill builds the metric's full implicitly-joined dimension set, which costs time and+ allocation proportional to (metrics x columns); hydrating a wide table's metrics for `query_metadata` did+ that once per metric and discarded all of it, taking minutes and exhausting the heap (#83937).++ Deliberately narrow:++ - **Only upgrade 24's backfill is skipped.** The upgrades to 21, 22 and 23 still run, so `:result_metadata`+ is still modernized, and `:dataset_query` is still normalized to current MBQL.++ - **`:card_schema` is still stamped to the current version.** The returned row therefore claims a vintage+ whose dimensions it is not carrying. That is in-memory only -- no read path writes it back -- and it+ cannot mislead `t2/update!`, which decides what to upgrade from the row's *stored* `card_schema` (read+ with the upgrades suppressed), never from an incoming card.++ - **Do not bind it around a write.** Writes legitimately need the real backfill. Note also that feeding a+ whole card read under this flag into `t2/update!` would put `:dimensions nil` into the changes and clobber+ the stored dimension set -- true of any card read whose `:dimensions` you pass back, but worth knowing."+ false)diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex 42686acba1c..89181c75e95 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -692,6 +692,10 @@(defmethod upgrade-card-schema-to 24[{:keys [dataset_query dimensions entity_id] :as card} _schema-version](cond+ ;; The caller declared it does not read a metric's dimensions, so don't pay to backfill them.+ ;; See [[card-schema/*skip-dimension-backfill?*]].+ card-schema/*skip-dimension-backfill?* card+(not= :metric (keyword (:type card))) card ; Ignore non-:metric cards(empty? dataset_query) card ; And those without real queriesdiff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.cljindex 35cdca1c661..169e5a18b81 100644--- a/src/metabase/warehouse_schema/models/table.clj+++ b/src/metabase/warehouse_schema/models/table.clj@@ -9,6 +9,7 @@[metabase.models.serialization :as serdes][metabase.permissions.core :as perms][metabase.premium-features.core :refer [defenterprise]]+ [metabase.queries.card-schema :as card-schema][metabase.remote-sync.core :as remote-sync][metabase.search.spec :as search.spec][metabase.util :as u]@@ -530,11 +531,18 @@(mi/define-batched-hydration-method with-metrics:metrics- "Efficiently hydrate the Metrics for a collection of `tables`."+ "Efficiently hydrate the Metrics for a collection of `tables`.++ Consumers of a Table's `:metrics` want an id, a name and a link, never a metric's `:dimensions`, so the+ `card_schema` 24 backfill is skipped. Without that the upgrade builds the whole implicitly-joined dimension set+ for every pre-curation metric on these tables -- work proportional to (metrics x columns), then discarded --+ which is minutes of CPU and enough garbage to exhaust the heap on a wide table with many old metrics (#83937).+ Every other `card_schema` upgrade still runs."[tables](with-objects :metrics(fn [table-ids]- (->> (warehouse-schema.db/unarchived-metric-cards-for-tables table-ids)+ (->> (binding [card-schema/*skip-dimension-backfill?* true]+ (warehouse-schema.db/unarchived-metric-cards-for-tables table-ids))(filter mi/can-read?)))tables))diff --git a/test/metabase/queries/models/card_test.clj b/test/metabase/queries/models/card_test.cljindex ec2504ed0b0..9df7f7df9a9 100644--- a/test/metabase/queries/models/card_test.clj+++ b/test/metabase/queries/models/card_test.clj@@ -1,6 +1,7 @@(ns metabase.queries.models.card-test{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.queries.models.card-test]}}}}}}(:require+ [clojure.string :as str][clojure.test :refer :all][java-time.api :as t][metabase.api.common :as api]@@ -2240,3 +2241,41 @@(t2/update! :model/Card (:id card) {:visualization_settings {:timeline.selected_timeline_ids []}})(is (= [] (get-in (t2/select-one :model/Card (:id card))[:visualization_settings :timeline.selected_timeline_ids])))))))++(deftest skip-dimension-backfill-skips-only-upgrade-24-test+ (testing "card-schema/*skip-dimension-backfill?* skips the 24 dimension backfill and nothing else (#83937)"+ (mt/with-temp [:model/Card {card-id :id} {:type :metric+ :database_id (mt/id)+ :table_id (mt/id :venues)+ :dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}]+ ;; A pre-curation metric: schema 23, no stored dimensions, and a `:result_metadata` still carrying one of+ ;; the abandoned `:ident`s that the upgrade to 22 exists to strip. Written raw because before-insert+ ;; forces `:card_schema` to current and normalizes the card.+ (t2/query-one {:update :report_card+ :set {:card_schema 23+ :dimensions nil+ :dimension_mappings nil+ :result_metadata (json/encode [{:name "count"+ :ident "abandoned-ident"+ :base_type "type/Integer"}])}+ :where [:= :id card-id]})+ (testing "precondition: the stored row really does carry the stale ident"+ (is (str/includes? (:result_metadata (t2/query-one {:select [:result_metadata]+ :from [:report_card]+ :where [:= :id card-id]}))+ "abandoned-ident")))+ (testing "without the flag the backfill runs"+ (is (seq (:dimensions (t2/select-one :model/Card card-id)))))+ (binding [card-schema/*skip-dimension-backfill?* true]+ (let [card (t2/select-one :model/Card card-id)]+ (testing "the expensive 24 backfill is skipped"+ (is (nil? (:dimensions card)))+ (is (nil? (:dimension_mappings card))))+ (testing "the other upgrades still run -- 22 strips the abandoned :ident"+ (is (every? #(not (contains? % :ident)) (:result_metadata card))))+ (testing ":dataset_query is still normalized to current MBQL"+ (is (=? {:lib/type :mbql/query} (:dataset_query card))))+ (testing "and the row still reports the current schema version"+ (is (= @#'card/current-schema-version (:card_schema card))))))+ (testing "the stored row is untouched by any of this"+ (is (= 23 (stored-card-schema card-id)))))))diff --git a/test/metabase/warehouse_schema_rest/api/table_test.clj b/test/metabase/warehouse_schema_rest/api/table_test.cljindex cbe5ef2895b..f6e13342d5f 100644--- a/test/metabase/warehouse_schema_rest/api/table_test.clj+++ b/test/metabase/warehouse_schema_rest/api/table_test.clj@@ -12,6 +12,7 @@[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.util :as driver.u][metabase.events.core :as events]+ [metabase.lib-metric.dimension.jvm :as lib-metric.dimension.jvm][metabase.lib.core :as lib][metabase.permissions.models.data-permissions :as data-perms][metabase.permissions.models.permissions :as perms]@@ -1606,3 +1607,69 @@{:ids [t1 t2] :visibility_type nil})(is (= [nil nil](map #(user-table-fn :visibility_type %) [t1 t2]))))))++;;; ---------------------------------- #83937 dimension-backfill canary -----------------------------------++(defn- counting-dimension-pairs+ "Calls `thunk` with [[lib-metric.dimension.jvm/compute-dimension-pairs]] counted, returning+ `[result call-count]`.++ That function is the expensive primitive behind the `card_schema` 24 dimension backfill: it runs+ `lib/visible-columns` over the metric's table *plus* every FK-reachable table and fetches every one of those+ Fields. Running it once per metric is what made `query_metadata` for a wide table take minutes and exhaust the+ heap in #83937, so these tests assert it never runs at all on that path."+ [thunk]+ (let [calls (atom 0)+ orig lib-metric.dimension.jvm/compute-dimension-pairs]+ (mt/with-dynamic-fn-redefs [lib-metric.dimension.jvm/compute-dimension-pairs+ (fn [& args]+ (clojure.core/swap! calls inc)+ (apply orig args))]+ [(thunk) @calls])))++(defn- make-pre-curation-metric!+ "Put `card-id` back into the pre-64 shape: `card_schema` 23 with no stored dimensions, so reading it is what+ triggers the backfill. Needs a raw UPDATE because before-insert forces `:card_schema` to current."+ [card-id]+ (t2/query-one {:update :report_card+ :set {:card_schema 23, :dimensions nil, :dimension_mappings nil}+ :where [:= :id card-id]}))++(deftest query-metadata-does-not-backfill-metric-dimensions-test+ (testing "query_metadata must not run the pre-64 dimension backfill for a table's metrics (#83937)"+ (mt/with-temp [:model/Card {metric-id :id} {:type :metric+ :database_id (mt/id)+ :table_id (mt/id :venues)+ :dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}+ :model/Card {question-id :id} {:database_id (mt/id)+ :table_id (mt/id :venues)+ :dataset_query (mt/mbql-query venues)}+ :model/Dashboard {dash-id :id} {}+ :model/DashboardCard _ {:dashboard_id dash-id, :card_id question-id}]+ (make-pre-curation-metric! metric-id)+ ;; Without this the whole test could pass vacuously: if the fixture ever stopped being a genuine+ ;; pre-curation metric, nothing would backfill and the zero-call assertions below would be meaningless.+ (testing "the fixture really is a metric whose dimensions get backfilled on an ordinary read"+ (is (pos? (second (counting-dimension-pairs+ #(t2/select-one :model/Card :id metric-id))))))+ (testing "GET /api/table/:id/query_metadata"+ (let [[response calls] (counting-dimension-pairs+ #(mt/user-http-request :crowberto :get 200+ (format "table/%d/query_metadata" (mt/id :venues))))+ metrics (:metrics response)]+ (is (zero? calls)+ "the dimension backfill ran while hydrating a table's metrics")+ (is (some #(= metric-id (:id %)) metrics)+ "the metric should still be hydrated, just without its backfilled dimensions")+ (is (every? (comp nil? :dimensions) metrics)+ "no metric should come back carrying a backfilled dimension set")))+ (testing "GET /api/dashboard/:id/query_metadata"+ (let [[_response calls] (counting-dimension-pairs+ #(mt/user-http-request :crowberto :get 200+ (format "dashboard/%d/query_metadata" dash-id)))]+ (is (zero? calls)+ "the dimension backfill ran while loading dashboard query metadata")))+ (testing "the stored row is untouched — still schema 23, so a later write still upgrades it"+ (is (= 23 (:card_schema (t2/query-one {:select [:card_schema]+ :from [:report_card]+ :where [:= :id metric-id]}))))))))