The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GHY-3339: serdes export should drop most `result_metadata` (#72560)
Timothy S. Dean800db3ff13ce
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/dump_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/dump_test.cljindex 6a657054ba5..ba6cc3ebeee 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/dump_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/dump_test.clj@@ -17,18 +17,6 @@(def example-card{:source_card_id "skoPT2xiuEcUV8vFkHE6S",:table_id ["Internal Metabase Database" "public" "v_alerts"],- :result_metadata '({:semantic_type :type/Quantity,- :lib/deduplicated-name "count",- :lib/original-name "count",- :name "count",- :lib/source :source/aggregations,- :lib/source-column-alias "count",- :source :aggregation,- :field_ref [:aggregation 0],- :effective_type :type/Integer,- :lib/desired-column-alias "count",- :display_name "Count",- :base_type :type/Integer}),:card_schema 23,:database_id "Internal Metabase Database",:collection_id "vG58R8k-QddHWA7_47umn",@@ -136,21 +124,6 @@ dataset_query:source-card: skoPT2xiuEcUV8vFkHE6Slib/type: mbql.stage/mbqllib/type: mbql/query-result_metadata:-- base_type: type/Integer- display_name: Count- effective_type: type/Integer- field_ref:- - aggregation- - 0- name: count- semantic_type: type/Quantity- source: aggregation- lib/deduplicated-name: count- lib/desired-column-alias: count- lib/original-name: count- lib/source: source/aggregations- lib/source-column-alias: countvisualization_settings:column_settings: nulltable.cell_column: recipient_externaldiff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.cljindex 5278b0e63a6..4eac8dad945 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj@@ -14,6 +14,7 @@[metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.models.serialization :as serdes]+ [metabase.query-processor.preprocess :as qp.preprocess][metabase.query-processor.test :as qp][metabase.search.test-util :as search.tu][metabase.test :as mt]@@ -1957,20 +1958,41 @@(is (= 5 (qc))))))))(deftest result-metadata-test- (mt/with-temp [:model/Card c {:dataset_query (mt/query venues)}]- (let [res (qp/process-query- (qp/userland-query- (:dataset_query c)- {:card-id (:id c)}))]- (when-not (= (:status res) :completed)- (throw (ex-info "Query failed" res)))- (let [ser (serdes/extract-one "Card" nil (t2/select-one :model/Card (:id c)))]- (is (=? {:base_type :type/Integer- :id [string? "PUBLIC" "VENUES" "CATEGORY_ID"]- :fk_target_field_id [string? "PUBLIC" "CATEGORIES" "ID"]- :field_ref [:field [string? "PUBLIC" "VENUES" "CATEGORY_ID"] nil]}- (->> (:result_metadata ser)- (u/seek #(= (:display_name %) "Category ID")))))))))+ (testing "model Card extraction portablizes :fk_target_field_id in :result_metadata"+ (mt/with-temp [:model/Card c {:type :model :dataset_query (mt/query venues)}]+ (let [res (qp/process-query+ (qp/userland-query+ (:dataset_query c)+ {:card-id (:id c)}))]+ (when-not (= (:status res) :completed)+ (throw (ex-info "Query failed" res)))+ (let [ser (serdes/extract-one "Card" nil (t2/select-one :model/Card (:id c)))]+ (is (=? {:fk_target_field_id [string? "PUBLIC" "CATEGORIES" "ID"]}+ (->> (:result_metadata ser)+ (u/seek #(and (= (:name %) "CATEGORY_ID")+ (= (:display_name %) "Category ID")))))))))))++(deftest model-preserved-keys-extract-test+ (testing "model Card preserved-key overrides survive extract"+ (let [base-cols (qp.preprocess/query->expected-cols (mt/query venues))+ overridden (mapv (fn [col]+ (cond-> (assoc col+ :display_name (str "Custom " (:name col))+ :visibility_type :normal+ :description "user-set")+ (= "CATEGORY_ID" (:name col)) (assoc :semantic_type :type/Category)))+ base-cols)]+ (mt/with-temp+ [:model/Card {card-id :id} {:type :model+ :dataset_query (mt/query venues)+ :result_metadata overridden}]+ (let [extracted (serdes/extract-one "Card" nil (t2/select-one :model/Card card-id))+ by-name (u/index-by :name (:result_metadata extracted))]+ (is (every? #(re-matches #"Custom .*" (:display_name %))+ (:result_metadata extracted)))+ (is (= "user-set" (:description (get by-name "ID"))))+ (is (= :type/Category (:semantic_type (get by-name "CATEGORY_ID"))))+ (is (vector? (:fk_target_field_id (get by-name "CATEGORY_ID")))))))))(deftest extract-single-collection-test(mt/with-empty-h2-app-db!diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex 8e1122c8408..3c16b6bc795 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -1257,14 +1257,18 @@;;; ------------------------------------------------- Serialization ---------------------------------------------------(defn- export-result-metadata [metadata]- (when metadata- (for [m metadata]- (-> (dissoc m :fingerprint)- (m/update-existing :table_id serdes/*export-table-fk*)- (m/update-existing :id serdes/*export-field-fk*)- (m/update-existing :field_ref serdes/export-mbql)- (m/update-existing :fk_target_field_id serdes/*export-field-fk*)))))+(defn- export-result-metadata [card _k metadata]+ (if (and (seq metadata) (model? card))+ (let [native? (lib/native? (:dataset_query card))+ keep-keys (into #{:name}+ (map u/->snake_case_en)+ (lib/model-preserved-keys native?))]+ (mapv (fn [m]+ (-> (select-keys m keep-keys)+ (m/update-existing :fk_target_field_id serdes/*export-field-fk*)+ (m/update-existing :id serdes/*export-field-fk*)))+ metadata))+ ::serdes/skip))(defn- import-result-metadata [metadata](when metadata@@ -1338,7 +1342,7 @@:parameters {:export serdes/export-parameters :import serdes/import-parameters}:parameter_mappings {:export serdes/export-parameter-mappings :import serdes/import-parameter-mappings}:visualization_settings {:export serdes/export-visualization-settings :import serdes/import-visualization-settings}- :result_metadata {:export export-result-metadata :import import-result-metadata}}+ :result_metadata {:export-with-context export-result-metadata :import import-result-metadata}}:defaults {:archived false:archived_directly false:collection_preview truediff --git a/test/metabase/queries/models/card_test.clj b/test/metabase/queries/models/card_test.cljindex e31aff88cd2..12a8a056aa8 100644--- a/test/metabase/queries/models/card_test.clj+++ b/test/metabase/queries/models/card_test.clj@@ -1,5 +1,6 @@(ns metabase.queries.models.card-test(:require+ [clojure.set :as set][clojure.test :refer :all][java-time.api :as t][metabase.api.common :as api]@@ -730,24 +731,70 @@(is (= {["Card" (:id card1)] {"Card" (:id card2)}}(serdes/descendants "Card" (:id card2) {}))))))-(deftest ^:parallel extract-test- (let [metadata (qp.preprocess/query->expected-cols (mt/mbql-query venues))- query (mt/mbql-query venues)]- (testing "every card retains result_metadata"- (mt/with-temp [:model/Card {card1-id :id} {:dataset_query query- :result_metadata metadata}- :model/Card {card2-id :id} {:type :model- :dataset_query query- :result_metadata metadata}]- (doseq [card-id [card1-id card2-id]]- (let [extracted (serdes/extract-one "Card" nil (t2/select-one :model/Card :id card-id))]- ;; card2 is model, but card1 is not- (is (= (= card-id card2-id)- (= :model (:type extracted))))- (is (string? (:display_name (first (:result_metadata extracted)))))- ;; this is a quick comparison, since the actual stored metadata is quite complex- (is (= (map :display_name metadata)- (map :display_name (:result_metadata extracted))))))))))+(deftest ^:parallel extract-result-metadata-non-model-test+ (testing "non-model Card extraction drops :result_metadata entirely"+ (let [metadata (qp.preprocess/query->expected-cols (mt/mbql-query venues))+ query (mt/mbql-query venues)]+ (doseq [card-type [:question :metric]]+ (testing (str "Card with :type " card-type)+ (mt/with-temp [:model/Card {card-id :id} {:type card-type+ :dataset_query query+ :result_metadata metadata}]+ (let [extracted (serdes/extract-one "Card" nil (t2/select-one :model/Card :id card-id))]+ (is (not (contains? extracted :result_metadata))))))))))++(deftest ^:parallel extract-result-metadata-model-test+ (testing "model Card extraction keeps :name + snake-cased model-preserved-keys only"+ (let [base (qp.preprocess/query->expected-cols (mt/mbql-query venues))+ metadata (mapv #(assoc %+ :display_name "Custom!"+ :semantic_type :type/Category+ :visibility_type :normal+ :description "desc"+ :fingerprint {:global {:distinct-count 100}}+ :lib/desired-column-alias "bloat"+ :qp/internal-flag true)+ base)+ query (mt/mbql-query venues)]+ (mt/with-temp [:model/Card {card-id :id} {:type :model+ :dataset_query query+ :result_metadata metadata}]+ (let [extracted (serdes/extract-one "Card" nil (t2/select-one :model/Card :id card-id))+ cols (:result_metadata extracted)+ ;; mirror the export's transformation exactly so this test stays in sync with+ ;; production if (lib/model-preserved-keys false) ever changes+ allowed (into #{:name} (map u/->snake_case_en) (lib/model-preserved-keys false))+ leaked (into #{} (mapcat #(remove allowed (keys %))) cols)]+ (is (seq cols))+ (is (= #{} leaked)+ "no key outside the allowed set leaks through (including bloat keys deliberately seeded)")+ (is (every? #(= "Custom!" (:display_name %)) cols)+ "user-set :display_name survives"))))))++(deftest ^:parallel extract-result-metadata-native-model-test+ (testing "native model Card extraction also preserves :id (as a field FK)"+ (mt/with-temp [:model/Card {card-id :id}+ {:type :model+ :dataset_query (mt/native-query {:query "SELECT ID FROM VENUES"})+ :result_metadata [{:name "ID"+ :id (mt/id :venues :id)+ :display_name "Venue ID"+ :semantic_type :type/PK+ :base_type :type/BigInteger}]}]+ (let [extracted (serdes/extract-one "Card" nil (t2/select-one :model/Card :id card-id))+ col (first (:result_metadata extracted))]+ (is (= #{:name :id :display_name :semantic_type}+ (set (keys col)))+ "exact set of keys preserved for this fixture (one col with these inputs)")+ (is (= "Venue ID" (:display_name col)))+ ;; :id should be portablized to a Field FK path: [db-name schema table-name field-name]+ (is (=? [string? "PUBLIC" "VENUES" "ID"] (:id col)))+ ;; cross-reference: nothing outside the snake-cased model-preserved-keys for native models.+ ;; If `model-preserved-keys` ever changes, the exact-set assertion above stops matching;+ ;; this guard catches unexpected drift (a new key sneaking in) on the way.+ (let [allowed (into #{:name} (map u/->snake_case_en) (lib/model-preserved-keys true))+ leaked (set/difference (set (keys col)) allowed)]+ (is (= #{} leaked) "no key outside the native-model preserved set"))))))(deftest ^:parallel upgrade-to-v2-db-test(testing ":visualization_settings v. 1 should be upgraded to v. 2 on select"