The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Preserve card metadata when recomputing fails (#85067)
metambenc195c3ff08ec
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/api/database_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/api/database_test.cljindex 8e472f889c9..483b73b9236 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/api/database_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/api/database_test.clj@@ -29,7 +29,8 @@[:report_card :c] [:= :c.id :s.card_id]]:where [:= :pg.id (u/the-id group)]}){:keys [metadata metadata-future]} (@#'card.metadata/maybe-async-recomputed-metadata- (:dataset_query card))]+ (:dataset_query card)+ (:result_metadata card))](mt/with-test-user :crowberto(if metadata(t2/update! :model/Card :id (u/the-id card) {:result_metadata metadata})diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/api/table_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/api/table_test.cljindex ac8bfbc446e..928f2284267 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/api/table_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/api/table_test.clj@@ -77,7 +77,9 @@:join [[:permissions_group :pg] [:= :s.group_id :pg.id][:report_card :c] [:= :c.id :s.card_id]]:where [:= :pg.id (u/the-id &group)]})- {:keys [metadata metadata-future]} (@#'card.metadata/maybe-async-recomputed-metadata (:dataset_query card))]+ {:keys [metadata metadata-future]} (@#'card.metadata/maybe-async-recomputed-metadata+ (:dataset_query card)+ (:result_metadata card))](mt/with-test-user :crowberto(if metadata(t2/update! :model/Card :id (u/the-id card) {:result_metadata metadata})diff --git a/src/metabase/legacy_mbql/schema.cljc b/src/metabase/legacy_mbql/schema.cljcindex aa595ee9f5d..e94f011300c 100644--- a/src/metabase/legacy_mbql/schema.cljc+++ b/src/metabase/legacy_mbql/schema.cljc@@ -2409,6 +2409,7 @@[:user-parameters {:optional true} [:maybe [:ref ::lib.schema.parameter/parameters]]][:qp/source-card-id {:optional true} [:ref ::lib.schema.id/card]][:qp/skip-result-metadata-persistence {:optional true} :boolean]+ [:qp/non-default-parameterized? {:optional true} :boolean][:qp.pivot/unremapped-breakout-combination {:optional true} [:sequential [:int {:min 0}]]][:qp.pivot/remapped-breakout-combination {:optional true} [:maybe [:sequential [:int {:min 0}]]]][:qp.pivot/num-remapped-cols {:optional true} [:int {:min 0}]]diff --git a/src/metabase/lib/schema.cljc b/src/metabase/lib/schema.cljcindex 4e32c1a1049..e78b45d0532 100644--- a/src/metabase/lib/schema.cljc+++ b/src/metabase/lib/schema.cljc@@ -735,6 +735,7 @@[:qp/compiled-inline {:optional true} [:ref ::compiled-native-query]][:qp/source-card-id {:optional true} [:ref ::id/card]][:qp/skip-result-metadata-persistence {:optional true} :boolean]+ [:qp/non-default-parameterized? {:optional true} :boolean][:qp.pivot/unremapped-breakout-combination {:optional true} [:sequential [:int {:min 0}]]][:qp.pivot/remapped-breakout-combination {:optional true} [:maybe [:sequential [:int {:min 0}]]]][:qp.pivot/num-remapped-cols {:optional true} [:int {:min 0}]]diff --git a/src/metabase/queries/models/card/metadata.clj b/src/metabase/queries/models/card/metadata.cljindex ee3efc02ac7..ef3670add74 100644--- a/src/metabase/queries/models/card/metadata.clj+++ b/src/metabase/queries/models/card/metadata.clj@@ -33,7 +33,8 @@(qp.metadata/legacy-result-metadata query api/*current-user-id*)(catch Throwable e(log/errorf "Error calculating result metadata for Card: %s" (ex-message e))- []))))+ ;; sentinel: callers distinguish this from "the query produced no columns"+ ::recompute-failed))))(def ^:private metadata-sync-wait-ms"Duration in milliseconds to wait for the metadata before saving the card without the metadata. That metadata will be@@ -72,24 +73,35 @@ saved later when it is ready.";; existing usage, don't use this going forward#_{:clj-kondo/ignore [:deprecated-var]}(qp.util/combine-metadata metadata')))]- (if (= result ::timed-out)+ (cond+ (= result ::timed-out){:metadata-future (future(try- (combiner @futur)+ (let [awaited @futur]+ (if (= awaited ::recompute-failed)+ metadata'+ (combiner awaited)))(catch Throwable e(future-cancel futur)(log/errorf "Error blending model metadata: %s" (ex-message e))metadata')))}++ (= result ::recompute-failed)+ {:metadata metadata'}++ :else{:metadata (combiner result)})))(mu/defn- maybe-async-recomputed-metadata :- ::maybe-async-result-metadata- [query :- ::lib-be.schema/maybe-legacy-or-empty-query]+ [query :- ::lib-be.schema/maybe-legacy-or-empty-query+ original-metadata :- [:maybe analyze/ResultsMetadata]](log/debug "Querying for metadata")- (let [futur (legacy-result-metadata-future query)+ (let [futur (legacy-result-metadata-future query)result (deref futur metadata-sync-wait-ms ::timed-out)]- (if (= result ::timed-out)- {:metadata-future futur}- {:metadata result})))+ (cond+ (= result ::timed-out) {:metadata-future futur}+ (= result ::recompute-failed) {:metadata original-metadata}+ :else {:metadata result})))(defn normalize-dataset-query"Normalize the query `dataset-query` received via an HTTP call.@@ -151,7 +163,7 @@ saved later when it is ready."(maybe-async-model-result-metadata (assoc options :valid-metadata? valid-metadata?)):else- (maybe-async-recomputed-metadata query))))+ (maybe-async-recomputed-metadata query original-metadata))))(def ^:private metadata-async-timeout-ms"Duration in milliseconds to wait for the metadata before abandoning the asynchronous metadata saving. Default is 15@@ -173,6 +185,9 @@ saved later when it is ready."(log/infof "Metadata not ready in %s, abandoning" (u/format-milliseconds metadata-async-timeout-ms))(future-cancel result-metadata-future))+ (= metadata ::recompute-failed)+ (log/infof "Not updating metadata asynchronously for card %s because the recompute failed" id)+(not (seq metadata))(log/infof "Not updating metadata asynchronously for card %s because no metadata" (u/the-id card))diff --git a/src/metabase/query_processor/middleware/parameters.clj b/src/metabase/query_processor/middleware/parameters.cljindex e479029376d..b5d5c9c7fd7 100644--- a/src/metabase/query_processor/middleware/parameters.clj+++ b/src/metabase/query_processor/middleware/parameters.clj@@ -116,15 +116,12 @@(not= value default)))))](some non-default-param? params)))))-(defn- maybe-skip-result-metadata-persistence- "Sets :qp/skip-result-metadata-persistence to true/false."+(defn- mark-non-default-parameterized+ "Set `:qp/non-default-parameterized?` to true when any stage carries parameters whose values diverge from+ their template tag defaults."[query]- (->> (:stages query)- (some stage-has-non-default-params?)- boolean- ;; `qp/skip-result-metadata-persistence` is used by [[metabase.query-processor.middleware.results-metadata/record-metadata!]]- ;; to decide whether to record metadata.- (assoc query :qp/skip-result-metadata-persistence)))+ (assoc query :qp/non-default-parameterized?+ (boolean (some stage-has-non-default-params? (:stages query)))))(mu/defn- move-top-level-params-to-stage :- ::lib.schema/query"Move any top-level parameters to the stage they affect."@@ -144,7 +141,7 @@(seq parameters)(-> (move-top-level-params-to-stage* parameters)- maybe-skip-result-metadata-persistence)))+ mark-non-default-parameterized)))(mu/defn- expand-parameters :- ::lib.schema/query"If any parameters were supplied then substitute them into the query."diff --git a/src/metabase/query_processor/middleware/results_metadata.clj b/src/metabase/query_processor/middleware/results_metadata.cljindex 9c4d5358fc8..9b833a7c457 100644--- a/src/metabase/query_processor/middleware/results_metadata.clj+++ b/src/metabase/query_processor/middleware/results_metadata.clj@@ -29,32 +29,38 @@;;; +----------------------------------------------------------------------------------------------------------------+(mu/defn- comparable-metadata- "Smooth out any unimportant differences in metadata so we can do an easy equality check."- [metadata :- [:maybe [:sequential ::lib.schema.metadata/lib-or-legacy-column]]]- (letfn [(remove-underscore-nil-keys- ;; Sometimes we get an underscore version of a key with a nil value which is a duplicate of a key with a- ;; dash. Remove the nil value- [m]- (reduce-kv- (fn [acc k v]- (let [kebab-cased-key (when (str/includes? (name k) "_")- (keyword (str/replace (name k) "_" "-")))]- (if (and kebab-cased-key- (nil? v)- (contains? m kebab-cased-key))- acc- (assoc acc k v))))- {} m))- (standardize-metadata [m]- (cond- (keyword? m) (u/qualified-name m)- (map? m) (-> (update-vals m standardize-metadata)- (dissoc :ident) ; `:ident` is deprecated and should no longer be present, but better safe than sorry.- (remove-underscore-nil-keys))- (sequential? m) (mapv standardize-metadata m)- (set? m) (into #{} (map standardize-metadata) m)- :else m))]- (mapv standardize-metadata metadata)))+ "A comparable representation of `metadata`, normalized so inconsequential differences don't register as a+ change. With `parameter-independent?` true, two runs over different parameter values compare equal when the+ column structure matches."+ ([metadata :- [:maybe [:sequential ::lib.schema.metadata/lib-or-legacy-column]]]+ (comparable-metadata metadata false))+ ([metadata :- [:maybe [:sequential ::lib.schema.metadata/lib-or-legacy-column]]+ parameter-independent? :- :boolean]+ (letfn [(remove-underscore-nil-keys+ ;; Sometimes we get an underscore version of a key with a nil value which is a duplicate of a key with a+ ;; dash. Remove the nil value+ [m]+ (reduce-kv+ (fn [acc k v]+ (let [kebab-cased-key (when (str/includes? (name k) "_")+ (keyword (str/replace (name k) "_" "-")))]+ (if (and kebab-cased-key+ (nil? v)+ (contains? m kebab-cased-key))+ acc+ (assoc acc k v))))+ {} m))+ (standardize-metadata [m]+ (cond+ (keyword? m) (u/qualified-name m)+ (map? m) (cond-> (-> (update-vals m standardize-metadata)+ (dissoc :ident) ; `:ident` is deprecated and should no longer be present, but better safe than sorry.+ (remove-underscore-nil-keys))+ parameter-independent? (dissoc :fingerprint))+ (sequential? m) (mapv standardize-metadata m)+ (set? m) (into #{} (map standardize-metadata) m)+ :else m))]+ (mapv standardize-metadata metadata))))(mu/defn- record-metadata![{{:keys [card-id]} :info, :as query} :- ::lib.schema/query@@ -70,21 +76,23 @@driver/*driver*;; pivot queries can run multiple queries, only record metadata for the main query(not= actual-metadata :none)- ;; Don't persist result_metadata for parameterized queries with non-default values.- ;; MBQL queries with any parameters are skipped because filters change fingerprints.- ;; Native queries are only skipped when parameter values differ from template tag defaults.- ;; See QUE2-502 for details.- (not (:qp/skip-result-metadata-persistence query))(driver.u/supports? driver/*driver* :nested-queries (lib.metadata/database query))card-id;; don't want to update metadata when we use a Card as a source Card.(not (:qp/source-card-id query))- ;; Only update changed metadata- (not= (comparable-metadata actual-metadata)- (comparable-metadata- ;; existing usage -- don't use going forward- #_{:clj-kondo/ignore [:deprecated-var]}- (qp.store/miscellaneous-value [::card-stored-metadata]))))+ ;; `:qp/skip-result-metadata-persistence` is an absolute skip set by callers (e.g.+ ;; `qp.card/process-query-for-card` when a `card-transform` rewrites the query) to tell us+ ;; this run's output is not the Card's canonical shape.+ (not (:qp/skip-result-metadata-persistence query))+ ;; Only update changed metadata; a non-default parameterized run writes only on+ ;; parameter-independent column changes, not fingerprint drift (#83734, see also #72007).+ (let [parameter-independent? (boolean (:qp/non-default-parameterized? query))]+ (not= (comparable-metadata actual-metadata parameter-independent?)+ (comparable-metadata+ ;; existing usage -- don't use going forward+ #_{:clj-kondo/ignore [:deprecated-var]}+ (qp.store/miscellaneous-value [::card-stored-metadata])+ parameter-independent?))))(when-let [error (me/humanize (mr/explain [:sequential ::lib.schema.metadata/lib-or-legacy-column] actual-metadata))](throw (ex-info "Invalid result metadata!" {:error error, :metadata actual-metadata})))(query-processor.db/set-card-result-metadata! card-id actual-metadata)))diff --git a/test/metabase/lib/serialize_test.cljc b/test/metabase/lib/serialize_test.cljcindex 8b9e7b2f70e..4e2dbc3dfba 100644--- a/test/metabase/lib/serialize_test.cljc+++ b/test/metabase/lib/serialize_test.cljc@@ -257,6 +257,7 @@:qp/compiled-inline {:query "SELECT 1"}:qp/source-card-id 1:qp/skip-result-metadata-persistence true+ :qp/non-default-parameterized? true:qp.pivot/unremapped-breakout-combination [0 1]:qp.pivot/remapped-breakout-combination [0]:qp.pivot/num-remapped-cols 3diff --git a/test/metabase/queries/models/card/metadata_test.clj b/test/metabase/queries/models/card/metadata_test.cljindex 79f749a0772..a14d63c34e3 100644--- a/test/metabase/queries/models/card/metadata_test.clj+++ b/test/metabase/queries/models/card/metadata_test.clj@@ -93,6 +93,35 @@(let [mp (mt/metadata-provider)](lib/query mp (lib.metadata/table mp (mt/id :venues)))))+(deftest maybe-async-result-metadata-recompute-failure-preserves-original-test+ (testing "when the metadata recompute throws (missing required native parameter), preserve original-metadata (#83734)"+ (let [mp (mt/metadata-provider)+ state-tag {:id "_STATE_"+ :name "state"+ :display-name "State"+ :type :text+ :required true}+ original-metadata [{:name "ID" :base_type :type/BigInteger :display_name "ID"+ :field_ref [:field "ID" {:base-type :type/BigInteger}]}+ {:name "NAME" :base_type :type/Text :display_name "Name"+ :field_ref [:field "NAME" {:base-type :type/Text}]}+ {:name "EMAIL" :base_type :type/Text :display_name "Email"+ :field_ref [:field "EMAIL" {:base-type :type/Text}]}]+ with-default (-> (lib/native-query mp "SELECT ID, NAME, EMAIL FROM PEOPLE WHERE STATE = {{state}}")+ (lib/with-template-tags {"state" (assoc state-tag :default "CA")}))+ without-default (-> (lib/native-query+ mp+ "SELECT ID, NAME, EMAIL FROM PEOPLE WHERE STATE = {{state}}\n-- edited")+ (lib/with-template-tags {"state" state-tag}))+ result (card.metadata/maybe-async-result-metadata+ {:original-query with-default+ :query without-default+ :original-metadata original-metadata+ :model? false})]+ (is (= (map :name original-metadata)+ (map :name (:metadata result)))+ "existing columns should be preserved when the recompute fails on missing required parameter"))))+(deftest backfill-untyped-model-metadata!-test(testing "backfill-untyped-model-metadata! infers the untyped metadata of the models on a database (GHY-4213)"(mt/with-temp [:model/Table {inactive-table-id :id} {:db_id (mt/id) :name "GHY_4213_NOT_SYNCED" :active false}diff --git a/test/metabase/queries_rest/api/card_test.clj b/test/metabase/queries_rest/api/card_test.cljindex 1e4033e7058..3e2e503d0be 100644--- a/test/metabase/queries_rest/api/card_test.clj+++ b/test/metabase/queries_rest/api/card_test.clj@@ -5889,3 +5889,31 @@(data-perms/set-database-permission! (perms-group/all-users) (mt/id) :perms/view-data :unrestricted)(data-perms/set-database-permission! (perms-group/all-users) (mt/id) :perms/create-queries :no)(is (= [[9]] (mt/rows (mt/user-http-request :rasta :post 202 (format "card/%d/query" (u/the-id outer)))))))))))))++(deftest sql-change-preserves-metadata-when-required-param-has-no-default-test+ (testing "editing a native card with a required-no-default parameter preserves existing result_metadata on save (#83734)"+ (let [mp (mt/metadata-provider)+ state-tag {:id "_STATE_"+ :name "state"+ :display-name "State"+ :type :text+ :required true}+ query-with-default (-> (lib/native-query mp "SELECT ID, NAME, EMAIL FROM PEOPLE WHERE STATE = {{state}}")+ (lib/with-template-tags {"state" (assoc state-tag :default "CA")}))+ query-without-default (-> (lib/native-query+ mp+ "SELECT ID, NAME, EMAIL FROM PEOPLE WHERE STATE = {{state}}\n-- edited")+ (lib/with-template-tags {"state" state-tag}))]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query query-with-default}]+ ;; First, run the card with the default value so result_metadata gets populated server-side.+ (mt/user-http-request :rasta :post 202 (format "card/%d/query" card-id))+ (let [metadata-before (t2/select-one-fn :result_metadata :model/Card card-id)]+ (is (seq metadata-before)+ "baseline: a run with the default value should populate result_metadata")+ ;; Now edit the SQL and remove the default — the recompute at save time can't run the query+ (mt/user-http-request :crowberto :put 200 (format "card/%d" card-id)+ {:dataset_query query-without-default})+ (let [metadata-after (t2/select-one-fn :result_metadata :model/Card card-id)]+ (is (= (map :name metadata-before)+ (map :name metadata-after))+ "result_metadata should be preserved across SQL edits when the recompute cannot run")))))))diff --git a/test/metabase/query_processor/middleware/results_metadata_test.clj b/test/metabase/query_processor/middleware/results_metadata_test.cljindex 58abb3132a7..929fa81d3f3 100644--- a/test/metabase/query_processor/middleware/results_metadata_test.clj+++ b/test/metabase/query_processor/middleware/results_metadata_test.clj@@ -11,12 +11,14 @@[clojure.test :refer :all][malli.error :as me][metabase.analyze.query-results :as qr]+ [metabase.lib.core :as lib][metabase.lib.schema.id :as lib.schema.id][metabase.lib.test-util :as lib.tu][metabase.permissions.models.permissions :as perms][metabase.permissions.models.permissions-group :as perms-group][metabase.query-processor.card :as qp.card][metabase.query-processor.compile :as qp.compile]+ [metabase.query-processor.db :as query-processor.db][metabase.query-processor.middleware.results-metadata :as middleware.results-metadata][metabase.query-processor.preprocess :as qp.preprocess];; binds mock metadata providers via the ambient store, which the code under test reads@@ -565,35 +567,68 @@"result_metadata should update when the query changes back")))))(deftest parameterized-queries-do-not-thrash-result-metadata-test- ;; When parameters are applied via :parameters, result_metadata should not be updated.- ;; See QUE2-502 for details.- (mt/with-temp [:model/Card {card-id :id, :as card}+ ;; A parameterized run that changes nothing but the fingerprint-worthy slice of the data should not rewrite+ ;; result_metadata (see #72007).+ (mt/with-temp [:model/Card {card-id :id}{:dataset_query (mt/mbql-query orders{:aggregation [[:count] [:sum $total]]:breakout [$product_id]}):result_metadata nil}](let [run-with-parameters!(fn [product-id]- ;; Bind *allow-arbitrary-mbql-parameters* as the dashboard code path does.+ ;; Refetch the card each time so `store-previous-result-metadata!` sees what the previous run wrote.(binding [qp.card/*allow-arbitrary-mbql-parameters* true](mt/as-admin(qp/process-query-for-card- card :api+ (t2/select-one :model/Card :id card-id) :api:parameters [{:id "product-id-param":type :id:target [:dimension (mt/$ids orders $product_id)]:value [product-id]}]:make-run (constantly(fn [query info]- (qp/process-query (assoc query :info info)))))))- (t2/select-one-fn :result_metadata :model/Card :id card-id))- ;; First run establishes baseline metadata+ (qp/process-query (assoc query :info info))))))))+ ;; First run can legitimately update metadata — the before-insert inference under-types some columns+ ;; (e.g. count as Integer instead of BigInteger). Snapshot after that run is the baseline._ (run-with-parameters! 1)meta-before (t2/select-one-fn :result_metadata :model/Card :id card-id)- ;; Second run with different parameter_ (run-with-parameters! 50)meta-after (t2/select-one-fn :result_metadata :model/Card :id card-id)](is (some? meta-before)"Baseline metadata should be established by first run")(is (= meta-before meta-after)- "result_metadata should not change for parameterized queries"))))+ "a second parameterized run should not change result_metadata when only fingerprints differ"))))++(deftest parameterized-queries-still-write-when-column-schema-differs-test+ ;; The #72007 skip for parameterized runs is overridden when the run's columns differ from stored in anything+ ;; other than fingerprint, so a card doesn't get stuck with stale columns (#83734).+ (let [mp (mt/metadata-provider)+ pid-tag {:id "_pid_" :name "pid" :display-name "PID" :type :number+ :required true :default "1"}+ ;; `reduced-query` is the Card's actual query; `baseline-query` represents a wider earlier version+ ;; whose metadata we install on the Card to simulate a stale save-time recompute (the #83734 shape).+ reduced-query (-> (lib/native-query+ mp+ "SELECT PRODUCT_ID, COUNT(*) AS cnt FROM ORDERS WHERE PRODUCT_ID = {{pid}} GROUP BY PRODUCT_ID")+ (lib/with-template-tags {"pid" pid-tag}))+ baseline-query (-> (lib/native-query+ mp+ "SELECT PRODUCT_ID, COUNT(*) AS cnt, SUM(TOTAL) AS total_sum FROM ORDERS WHERE PRODUCT_ID = {{pid}} GROUP BY PRODUCT_ID")+ (lib/with-template-tags {"pid" pid-tag}))+ baseline (mt/as-admin+ (get-in (qp/process-query baseline-query)+ [:data :results_metadata :columns]))]+ (is (seq baseline))+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query reduced-query}]+ ;; set-card-result-metadata! bypasses populate hooks, so the stale baseline sticks.+ (query-processor.db/set-card-result-metadata! card-id baseline)+ (mt/as-admin+ (qp/process-query-for-card+ (t2/select-one :model/Card card-id) :api+ :parameters [{:id "_pid_" :type :number :target [:variable [:template-tag "pid"]] :value "50"}]+ :make-run (constantly+ (fn [query info]+ (qp/process-query (assoc query :info info))))))+ (let [after (t2/select-one-fn :result_metadata :model/Card card-id)]+ (is (= #{"product_id" "cnt"} (into #{} (map (comp u/lower-case-en :name)) after))+ "parameter-independent schema change should persist despite the parameter skip")))))