Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Preserve card metadata when recomputing fails (#85067)

metambenc195c3ff08ec
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/api/database_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/api/database_test.clj
index 8e472f889c9..483b73b9236 100644
--- a/enterprise/backend/test/metabase_enterprise/sandbox/api/database_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sandbox/api/database_test.clj
@@ -29,7 +29,8 @@
[:report_card :c] [:= :c.id :s.card_id]]
:where [:= :pg.id (u/the-id group)]})
{:keys [metadata metadata-future]} (@#'card.metadata/maybe-async-recomputed-metadata
- (:dataset_query card))]
+ (:dataset_query card)
+ (:result_metadata card))]
(mt/with-test-user :crowberto
(if metadata
(t2/update! :model/Card :id (u/the-id card) {:result_metadata metadata})
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/api/table_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/api/table_test.clj
index ac8bfbc446e..928f2284267 100644
--- a/enterprise/backend/test/metabase_enterprise/sandbox/api/table_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sandbox/api/table_test.clj
@@ -77,7 +77,9 @@
:join [[:permissions_group :pg] [:= :s.group_id :pg.id]
[:report_card :c] [:= :c.id :s.card_id]]
:where [:= :pg.id (u/the-id &group)]})
- {:keys [metadata metadata-future]} (@#'card.metadata/maybe-async-recomputed-metadata (:dataset_query card))]
+ {:keys [metadata metadata-future]} (@#'card.metadata/maybe-async-recomputed-metadata
+ (:dataset_query card)
+ (:result_metadata card))]
(mt/with-test-user :crowberto
(if metadata
(t2/update! :model/Card :id (u/the-id card) {:result_metadata metadata})
diff --git a/src/metabase/legacy_mbql/schema.cljc b/src/metabase/legacy_mbql/schema.cljc
index aa595ee9f5d..e94f011300c 100644
--- a/src/metabase/legacy_mbql/schema.cljc
+++ b/src/metabase/legacy_mbql/schema.cljc
@@ -2409,6 +2409,7 @@
[:user-parameters {:optional true} [:maybe [:ref ::lib.schema.parameter/parameters]]]
[:qp/source-card-id {:optional true} [:ref ::lib.schema.id/card]]
[:qp/skip-result-metadata-persistence {:optional true} :boolean]
+ [:qp/non-default-parameterized? {:optional true} :boolean]
[:qp.pivot/unremapped-breakout-combination {:optional true} [:sequential [:int {:min 0}]]]
[:qp.pivot/remapped-breakout-combination {:optional true} [:maybe [:sequential [:int {:min 0}]]]]
[:qp.pivot/num-remapped-cols {:optional true} [:int {:min 0}]]
diff --git a/src/metabase/lib/schema.cljc b/src/metabase/lib/schema.cljc
index 4e32c1a1049..e78b45d0532 100644
--- a/src/metabase/lib/schema.cljc
+++ b/src/metabase/lib/schema.cljc
@@ -735,6 +735,7 @@
[:qp/compiled-inline {:optional true} [:ref ::compiled-native-query]]
[:qp/source-card-id {:optional true} [:ref ::id/card]]
[:qp/skip-result-metadata-persistence {:optional true} :boolean]
+ [:qp/non-default-parameterized? {:optional true} :boolean]
[:qp.pivot/unremapped-breakout-combination {:optional true} [:sequential [:int {:min 0}]]]
[:qp.pivot/remapped-breakout-combination {:optional true} [:maybe [:sequential [:int {:min 0}]]]]
[:qp.pivot/num-remapped-cols {:optional true} [:int {:min 0}]]
diff --git a/src/metabase/queries/models/card/metadata.clj b/src/metabase/queries/models/card/metadata.clj
index ee3efc02ac7..ef3670add74 100644
--- a/src/metabase/queries/models/card/metadata.clj
+++ b/src/metabase/queries/models/card/metadata.clj
@@ -33,7 +33,8 @@
(qp.metadata/legacy-result-metadata query api/*current-user-id*)
(catch Throwable e
(log/errorf "Error calculating result metadata for Card: %s" (ex-message e))
- []))))
+ ;; sentinel: callers distinguish this from "the query produced no columns"
+ ::recompute-failed))))
(def ^:private metadata-sync-wait-ms
"Duration in milliseconds to wait for the metadata before saving the card without the metadata. That metadata will be
@@ -72,24 +73,35 @@ saved later when it is ready."
;; existing usage, don't use this going forward
#_{:clj-kondo/ignore [:deprecated-var]}
(qp.util/combine-metadata metadata')))]
- (if (= result ::timed-out)
+ (cond
+ (= result ::timed-out)
{:metadata-future (future
(try
- (combiner @futur)
+ (let [awaited @futur]
+ (if (= awaited ::recompute-failed)
+ metadata'
+ (combiner awaited)))
(catch Throwable e
(future-cancel futur)
(log/errorf "Error blending model metadata: %s" (ex-message e))
metadata')))}
+
+ (= result ::recompute-failed)
+ {:metadata metadata'}
+
+ :else
{:metadata (combiner result)})))
(mu/defn- maybe-async-recomputed-metadata :- ::maybe-async-result-metadata
- [query :- ::lib-be.schema/maybe-legacy-or-empty-query]
+ [query :- ::lib-be.schema/maybe-legacy-or-empty-query
+ original-metadata :- [:maybe analyze/ResultsMetadata]]
(log/debug "Querying for metadata")
- (let [futur (legacy-result-metadata-future query)
+ (let [futur (legacy-result-metadata-future query)
result (deref futur metadata-sync-wait-ms ::timed-out)]
- (if (= result ::timed-out)
- {:metadata-future futur}
- {:metadata result})))
+ (cond
+ (= result ::timed-out) {:metadata-future futur}
+ (= result ::recompute-failed) {:metadata original-metadata}
+ :else {:metadata result})))
(defn normalize-dataset-query
"Normalize the query `dataset-query` received via an HTTP call.
@@ -151,7 +163,7 @@ saved later when it is ready."
(maybe-async-model-result-metadata (assoc options :valid-metadata? valid-metadata?))
:else
- (maybe-async-recomputed-metadata query))))
+ (maybe-async-recomputed-metadata query original-metadata))))
(def ^:private metadata-async-timeout-ms
"Duration in milliseconds to wait for the metadata before abandoning the asynchronous metadata saving. Default is 15
@@ -173,6 +185,9 @@ saved later when it is ready."
(log/infof "Metadata not ready in %s, abandoning" (u/format-milliseconds metadata-async-timeout-ms))
(future-cancel result-metadata-future))
+ (= metadata ::recompute-failed)
+ (log/infof "Not updating metadata asynchronously for card %s because the recompute failed" id)
+
(not (seq metadata))
(log/infof "Not updating metadata asynchronously for card %s because no metadata" (u/the-id card))
diff --git a/src/metabase/query_processor/middleware/parameters.clj b/src/metabase/query_processor/middleware/parameters.clj
index e479029376d..b5d5c9c7fd7 100644
--- a/src/metabase/query_processor/middleware/parameters.clj
+++ b/src/metabase/query_processor/middleware/parameters.clj
@@ -116,15 +116,12 @@
(not= value default)))))]
(some non-default-param? params)))))
-(defn- maybe-skip-result-metadata-persistence
- "Sets :qp/skip-result-metadata-persistence to true/false."
+(defn- mark-non-default-parameterized
+ "Set `:qp/non-default-parameterized?` to true when any stage carries parameters whose values diverge from
+ their template tag defaults."
[query]
- (->> (:stages query)
- (some stage-has-non-default-params?)
- boolean
- ;; `qp/skip-result-metadata-persistence` is used by [[metabase.query-processor.middleware.results-metadata/record-metadata!]]
- ;; to decide whether to record metadata.
- (assoc query :qp/skip-result-metadata-persistence)))
+ (assoc query :qp/non-default-parameterized?
+ (boolean (some stage-has-non-default-params? (:stages query)))))
(mu/defn- move-top-level-params-to-stage :- ::lib.schema/query
"Move any top-level parameters to the stage they affect."
@@ -144,7 +141,7 @@
(seq parameters)
(-> (move-top-level-params-to-stage* parameters)
- maybe-skip-result-metadata-persistence)))
+ mark-non-default-parameterized)))
(mu/defn- expand-parameters :- ::lib.schema/query
"If any parameters were supplied then substitute them into the query."
diff --git a/src/metabase/query_processor/middleware/results_metadata.clj b/src/metabase/query_processor/middleware/results_metadata.clj
index 9c4d5358fc8..9b833a7c457 100644
--- a/src/metabase/query_processor/middleware/results_metadata.clj
+++ b/src/metabase/query_processor/middleware/results_metadata.clj
@@ -29,32 +29,38 @@
;;; +----------------------------------------------------------------------------------------------------------------+
(mu/defn- comparable-metadata
- "Smooth out any unimportant differences in metadata so we can do an easy equality check."
- [metadata :- [:maybe [:sequential ::lib.schema.metadata/lib-or-legacy-column]]]
- (letfn [(remove-underscore-nil-keys
- ;; Sometimes we get an underscore version of a key with a nil value which is a duplicate of a key with a
- ;; dash. Remove the nil value
- [m]
- (reduce-kv
- (fn [acc k v]
- (let [kebab-cased-key (when (str/includes? (name k) "_")
- (keyword (str/replace (name k) "_" "-")))]
- (if (and kebab-cased-key
- (nil? v)
- (contains? m kebab-cased-key))
- acc
- (assoc acc k v))))
- {} m))
- (standardize-metadata [m]
- (cond
- (keyword? m) (u/qualified-name m)
- (map? m) (-> (update-vals m standardize-metadata)
- (dissoc :ident) ; `:ident` is deprecated and should no longer be present, but better safe than sorry.
- (remove-underscore-nil-keys))
- (sequential? m) (mapv standardize-metadata m)
- (set? m) (into #{} (map standardize-metadata) m)
- :else m))]
- (mapv standardize-metadata metadata)))
+ "A comparable representation of `metadata`, normalized so inconsequential differences don't register as a
+ change. With `parameter-independent?` true, two runs over different parameter values compare equal when the
+ column structure matches."
+ ([metadata :- [:maybe [:sequential ::lib.schema.metadata/lib-or-legacy-column]]]
+ (comparable-metadata metadata false))
+ ([metadata :- [:maybe [:sequential ::lib.schema.metadata/lib-or-legacy-column]]
+ parameter-independent? :- :boolean]
+ (letfn [(remove-underscore-nil-keys
+ ;; Sometimes we get an underscore version of a key with a nil value which is a duplicate of a key with a
+ ;; dash. Remove the nil value
+ [m]
+ (reduce-kv
+ (fn [acc k v]
+ (let [kebab-cased-key (when (str/includes? (name k) "_")
+ (keyword (str/replace (name k) "_" "-")))]
+ (if (and kebab-cased-key
+ (nil? v)
+ (contains? m kebab-cased-key))
+ acc
+ (assoc acc k v))))
+ {} m))
+ (standardize-metadata [m]
+ (cond
+ (keyword? m) (u/qualified-name m)
+ (map? m) (cond-> (-> (update-vals m standardize-metadata)
+ (dissoc :ident) ; `:ident` is deprecated and should no longer be present, but better safe than sorry.
+ (remove-underscore-nil-keys))
+ parameter-independent? (dissoc :fingerprint))
+ (sequential? m) (mapv standardize-metadata m)
+ (set? m) (into #{} (map standardize-metadata) m)
+ :else m))]
+ (mapv standardize-metadata metadata))))
(mu/defn- record-metadata!
[{{:keys [card-id]} :info, :as query} :- ::lib.schema/query
@@ -70,21 +76,23 @@
driver/*driver*
;; pivot queries can run multiple queries, only record metadata for the main query
(not= actual-metadata :none)
- ;; Don't persist result_metadata for parameterized queries with non-default values.
- ;; MBQL queries with any parameters are skipped because filters change fingerprints.
- ;; Native queries are only skipped when parameter values differ from template tag defaults.
- ;; See QUE2-502 for details.
- (not (:qp/skip-result-metadata-persistence query))
(driver.u/supports? driver/*driver* :nested-queries (lib.metadata/database query))
card-id
;; don't want to update metadata when we use a Card as a source Card.
(not (:qp/source-card-id query))
- ;; Only update changed metadata
- (not= (comparable-metadata actual-metadata)
- (comparable-metadata
- ;; existing usage -- don't use going forward
- #_{:clj-kondo/ignore [:deprecated-var]}
- (qp.store/miscellaneous-value [::card-stored-metadata]))))
+ ;; `:qp/skip-result-metadata-persistence` is an absolute skip set by callers (e.g.
+ ;; `qp.card/process-query-for-card` when a `card-transform` rewrites the query) to tell us
+ ;; this run's output is not the Card's canonical shape.
+ (not (:qp/skip-result-metadata-persistence query))
+ ;; Only update changed metadata; a non-default parameterized run writes only on
+ ;; parameter-independent column changes, not fingerprint drift (#83734, see also #72007).
+ (let [parameter-independent? (boolean (:qp/non-default-parameterized? query))]
+ (not= (comparable-metadata actual-metadata parameter-independent?)
+ (comparable-metadata
+ ;; existing usage -- don't use going forward
+ #_{:clj-kondo/ignore [:deprecated-var]}
+ (qp.store/miscellaneous-value [::card-stored-metadata])
+ parameter-independent?))))
(when-let [error (me/humanize (mr/explain [:sequential ::lib.schema.metadata/lib-or-legacy-column] actual-metadata))]
(throw (ex-info "Invalid result metadata!" {:error error, :metadata actual-metadata})))
(query-processor.db/set-card-result-metadata! card-id actual-metadata)))
diff --git a/test/metabase/lib/serialize_test.cljc b/test/metabase/lib/serialize_test.cljc
index 8b9e7b2f70e..4e2dbc3dfba 100644
--- a/test/metabase/lib/serialize_test.cljc
+++ b/test/metabase/lib/serialize_test.cljc
@@ -257,6 +257,7 @@
:qp/compiled-inline {:query "SELECT 1"}
:qp/source-card-id 1
:qp/skip-result-metadata-persistence true
+ :qp/non-default-parameterized? true
:qp.pivot/unremapped-breakout-combination [0 1]
:qp.pivot/remapped-breakout-combination [0]
:qp.pivot/num-remapped-cols 3
diff --git a/test/metabase/queries/models/card/metadata_test.clj b/test/metabase/queries/models/card/metadata_test.clj
index 79f749a0772..a14d63c34e3 100644
--- a/test/metabase/queries/models/card/metadata_test.clj
+++ b/test/metabase/queries/models/card/metadata_test.clj
@@ -93,6 +93,35 @@
(let [mp (mt/metadata-provider)]
(lib/query mp (lib.metadata/table mp (mt/id :venues)))))
+(deftest maybe-async-result-metadata-recompute-failure-preserves-original-test
+ (testing "when the metadata recompute throws (missing required native parameter), preserve original-metadata (#83734)"
+ (let [mp (mt/metadata-provider)
+ state-tag {:id "_STATE_"
+ :name "state"
+ :display-name "State"
+ :type :text
+ :required true}
+ original-metadata [{:name "ID" :base_type :type/BigInteger :display_name "ID"
+ :field_ref [:field "ID" {:base-type :type/BigInteger}]}
+ {:name "NAME" :base_type :type/Text :display_name "Name"
+ :field_ref [:field "NAME" {:base-type :type/Text}]}
+ {:name "EMAIL" :base_type :type/Text :display_name "Email"
+ :field_ref [:field "EMAIL" {:base-type :type/Text}]}]
+ with-default (-> (lib/native-query mp "SELECT ID, NAME, EMAIL FROM PEOPLE WHERE STATE = {{state}}")
+ (lib/with-template-tags {"state" (assoc state-tag :default "CA")}))
+ without-default (-> (lib/native-query
+ mp
+ "SELECT ID, NAME, EMAIL FROM PEOPLE WHERE STATE = {{state}}\n-- edited")
+ (lib/with-template-tags {"state" state-tag}))
+ result (card.metadata/maybe-async-result-metadata
+ {:original-query with-default
+ :query without-default
+ :original-metadata original-metadata
+ :model? false})]
+ (is (= (map :name original-metadata)
+ (map :name (:metadata result)))
+ "existing columns should be preserved when the recompute fails on missing required parameter"))))
+
(deftest backfill-untyped-model-metadata!-test
(testing "backfill-untyped-model-metadata! infers the untyped metadata of the models on a database (GHY-4213)"
(mt/with-temp [:model/Table {inactive-table-id :id} {:db_id (mt/id) :name "GHY_4213_NOT_SYNCED" :active false}
diff --git a/test/metabase/queries_rest/api/card_test.clj b/test/metabase/queries_rest/api/card_test.clj
index 1e4033e7058..3e2e503d0be 100644
--- a/test/metabase/queries_rest/api/card_test.clj
+++ b/test/metabase/queries_rest/api/card_test.clj
@@ -5889,3 +5889,31 @@
(data-perms/set-database-permission! (perms-group/all-users) (mt/id) :perms/view-data :unrestricted)
(data-perms/set-database-permission! (perms-group/all-users) (mt/id) :perms/create-queries :no)
(is (= [[9]] (mt/rows (mt/user-http-request :rasta :post 202 (format "card/%d/query" (u/the-id outer)))))))))))))
+
+(deftest sql-change-preserves-metadata-when-required-param-has-no-default-test
+ (testing "editing a native card with a required-no-default parameter preserves existing result_metadata on save (#83734)"
+ (let [mp (mt/metadata-provider)
+ state-tag {:id "_STATE_"
+ :name "state"
+ :display-name "State"
+ :type :text
+ :required true}
+ query-with-default (-> (lib/native-query mp "SELECT ID, NAME, EMAIL FROM PEOPLE WHERE STATE = {{state}}")
+ (lib/with-template-tags {"state" (assoc state-tag :default "CA")}))
+ query-without-default (-> (lib/native-query
+ mp
+ "SELECT ID, NAME, EMAIL FROM PEOPLE WHERE STATE = {{state}}\n-- edited")
+ (lib/with-template-tags {"state" state-tag}))]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query query-with-default}]
+ ;; First, run the card with the default value so result_metadata gets populated server-side.
+ (mt/user-http-request :rasta :post 202 (format "card/%d/query" card-id))
+ (let [metadata-before (t2/select-one-fn :result_metadata :model/Card card-id)]
+ (is (seq metadata-before)
+ "baseline: a run with the default value should populate result_metadata")
+ ;; Now edit the SQL and remove the default — the recompute at save time can't run the query
+ (mt/user-http-request :crowberto :put 200 (format "card/%d" card-id)
+ {:dataset_query query-without-default})
+ (let [metadata-after (t2/select-one-fn :result_metadata :model/Card card-id)]
+ (is (= (map :name metadata-before)
+ (map :name metadata-after))
+ "result_metadata should be preserved across SQL edits when the recompute cannot run")))))))
diff --git a/test/metabase/query_processor/middleware/results_metadata_test.clj b/test/metabase/query_processor/middleware/results_metadata_test.clj
index 58abb3132a7..929fa81d3f3 100644
--- a/test/metabase/query_processor/middleware/results_metadata_test.clj
+++ b/test/metabase/query_processor/middleware/results_metadata_test.clj
@@ -11,12 +11,14 @@
[clojure.test :refer :all]
[malli.error :as me]
[metabase.analyze.query-results :as qr]
+ [metabase.lib.core :as lib]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.test-util :as lib.tu]
[metabase.permissions.models.permissions :as perms]
[metabase.permissions.models.permissions-group :as perms-group]
[metabase.query-processor.card :as qp.card]
[metabase.query-processor.compile :as qp.compile]
+ [metabase.query-processor.db :as query-processor.db]
[metabase.query-processor.middleware.results-metadata :as middleware.results-metadata]
[metabase.query-processor.preprocess :as qp.preprocess]
;; binds mock metadata providers via the ambient store, which the code under test reads
@@ -565,35 +567,68 @@
"result_metadata should update when the query changes back")))))
(deftest parameterized-queries-do-not-thrash-result-metadata-test
- ;; When parameters are applied via :parameters, result_metadata should not be updated.
- ;; See QUE2-502 for details.
- (mt/with-temp [:model/Card {card-id :id, :as card}
+ ;; A parameterized run that changes nothing but the fingerprint-worthy slice of the data should not rewrite
+ ;; result_metadata (see #72007).
+ (mt/with-temp [:model/Card {card-id :id}
{:dataset_query (mt/mbql-query orders
{:aggregation [[:count] [:sum $total]]
:breakout [$product_id]})
:result_metadata nil}]
(let [run-with-parameters!
(fn [product-id]
- ;; Bind *allow-arbitrary-mbql-parameters* as the dashboard code path does.
+ ;; Refetch the card each time so `store-previous-result-metadata!` sees what the previous run wrote.
(binding [qp.card/*allow-arbitrary-mbql-parameters* true]
(mt/as-admin
(qp/process-query-for-card
- card :api
+ (t2/select-one :model/Card :id card-id) :api
:parameters [{:id "product-id-param"
:type :id
:target [:dimension (mt/$ids orders $product_id)]
:value [product-id]}]
:make-run (constantly
(fn [query info]
- (qp/process-query (assoc query :info info)))))))
- (t2/select-one-fn :result_metadata :model/Card :id card-id))
- ;; First run establishes baseline metadata
+ (qp/process-query (assoc query :info info))))))))
+ ;; First run can legitimately update metadata — the before-insert inference under-types some columns
+ ;; (e.g. count as Integer instead of BigInteger). Snapshot after that run is the baseline.
_ (run-with-parameters! 1)
meta-before (t2/select-one-fn :result_metadata :model/Card :id card-id)
- ;; Second run with different parameter
_ (run-with-parameters! 50)
meta-after (t2/select-one-fn :result_metadata :model/Card :id card-id)]
(is (some? meta-before)
"Baseline metadata should be established by first run")
(is (= meta-before meta-after)
- "result_metadata should not change for parameterized queries"))))
+ "a second parameterized run should not change result_metadata when only fingerprints differ"))))
+
+(deftest parameterized-queries-still-write-when-column-schema-differs-test
+ ;; The #72007 skip for parameterized runs is overridden when the run's columns differ from stored in anything
+ ;; other than fingerprint, so a card doesn't get stuck with stale columns (#83734).
+ (let [mp (mt/metadata-provider)
+ pid-tag {:id "_pid_" :name "pid" :display-name "PID" :type :number
+ :required true :default "1"}
+ ;; `reduced-query` is the Card's actual query; `baseline-query` represents a wider earlier version
+ ;; whose metadata we install on the Card to simulate a stale save-time recompute (the #83734 shape).
+ reduced-query (-> (lib/native-query
+ mp
+ "SELECT PRODUCT_ID, COUNT(*) AS cnt FROM ORDERS WHERE PRODUCT_ID = {{pid}} GROUP BY PRODUCT_ID")
+ (lib/with-template-tags {"pid" pid-tag}))
+ baseline-query (-> (lib/native-query
+ mp
+ "SELECT PRODUCT_ID, COUNT(*) AS cnt, SUM(TOTAL) AS total_sum FROM ORDERS WHERE PRODUCT_ID = {{pid}} GROUP BY PRODUCT_ID")
+ (lib/with-template-tags {"pid" pid-tag}))
+ baseline (mt/as-admin
+ (get-in (qp/process-query baseline-query)
+ [:data :results_metadata :columns]))]
+ (is (seq baseline))
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query reduced-query}]
+ ;; set-card-result-metadata! bypasses populate hooks, so the stale baseline sticks.
+ (query-processor.db/set-card-result-metadata! card-id baseline)
+ (mt/as-admin
+ (qp/process-query-for-card
+ (t2/select-one :model/Card card-id) :api
+ :parameters [{:id "_pid_" :type :number :target [:variable [:template-tag "pid"]] :value "50"}]
+ :make-run (constantly
+ (fn [query info]
+ (qp/process-query (assoc query :info info))))))
+ (let [after (t2/select-one-fn :result_metadata :model/Card card-id)]
+ (is (= #{"product_id" "cnt"} (into #{} (map (comp u/lower-case-en :name)) after))
+ "parameter-independent schema change should persist despite the parameter skip")))))