Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

GHY-3339: serdes export should drop most `result_metadata` (#72560)

Timothy S. Dean800db3ff13ce
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/dump_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/dump_test.clj
index 6a657054ba5..ba6cc3ebeee 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/dump_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/dump_test.clj
@@ -17,18 +17,6 @@
(def example-card
{:source_card_id "skoPT2xiuEcUV8vFkHE6S",
:table_id ["Internal Metabase Database" "public" "v_alerts"],
- :result_metadata '({:semantic_type :type/Quantity,
- :lib/deduplicated-name "count",
- :lib/original-name "count",
- :name "count",
- :lib/source :source/aggregations,
- :lib/source-column-alias "count",
- :source :aggregation,
- :field_ref [:aggregation 0],
- :effective_type :type/Integer,
- :lib/desired-column-alias "count",
- :display_name "Count",
- :base_type :type/Integer}),
:card_schema 23,
:database_id "Internal Metabase Database",
:collection_id "vG58R8k-QddHWA7_47umn",
@@ -136,21 +124,6 @@ dataset_query:
source-card: skoPT2xiuEcUV8vFkHE6S
lib/type: mbql.stage/mbql
lib/type: mbql/query
-result_metadata:
-- base_type: type/Integer
- display_name: Count
- effective_type: type/Integer
- field_ref:
- - aggregation
- - 0
- name: count
- semantic_type: type/Quantity
- source: aggregation
- lib/deduplicated-name: count
- lib/desired-column-alias: count
- lib/original-name: count
- lib/source: source/aggregations
- lib/source-column-alias: count
visualization_settings:
column_settings: null
table.cell_column: recipient_external
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
index 5278b0e63a6..4eac8dad945 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
@@ -14,6 +14,7 @@
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.models.serialization :as serdes]
+ [metabase.query-processor.preprocess :as qp.preprocess]
[metabase.query-processor.test :as qp]
[metabase.search.test-util :as search.tu]
[metabase.test :as mt]
@@ -1957,20 +1958,41 @@
(is (= 5 (qc))))))))
(deftest result-metadata-test
- (mt/with-temp [:model/Card c {:dataset_query (mt/query venues)}]
- (let [res (qp/process-query
- (qp/userland-query
- (:dataset_query c)
- {:card-id (:id c)}))]
- (when-not (= (:status res) :completed)
- (throw (ex-info "Query failed" res)))
- (let [ser (serdes/extract-one "Card" nil (t2/select-one :model/Card (:id c)))]
- (is (=? {:base_type :type/Integer
- :id [string? "PUBLIC" "VENUES" "CATEGORY_ID"]
- :fk_target_field_id [string? "PUBLIC" "CATEGORIES" "ID"]
- :field_ref [:field [string? "PUBLIC" "VENUES" "CATEGORY_ID"] nil]}
- (->> (:result_metadata ser)
- (u/seek #(= (:display_name %) "Category ID")))))))))
+ (testing "model Card extraction portablizes :fk_target_field_id in :result_metadata"
+ (mt/with-temp [:model/Card c {:type :model :dataset_query (mt/query venues)}]
+ (let [res (qp/process-query
+ (qp/userland-query
+ (:dataset_query c)
+ {:card-id (:id c)}))]
+ (when-not (= (:status res) :completed)
+ (throw (ex-info "Query failed" res)))
+ (let [ser (serdes/extract-one "Card" nil (t2/select-one :model/Card (:id c)))]
+ (is (=? {:fk_target_field_id [string? "PUBLIC" "CATEGORIES" "ID"]}
+ (->> (:result_metadata ser)
+ (u/seek #(and (= (:name %) "CATEGORY_ID")
+ (= (:display_name %) "Category ID")))))))))))
+
+(deftest model-preserved-keys-extract-test
+ (testing "model Card preserved-key overrides survive extract"
+ (let [base-cols (qp.preprocess/query->expected-cols (mt/query venues))
+ overridden (mapv (fn [col]
+ (cond-> (assoc col
+ :display_name (str "Custom " (:name col))
+ :visibility_type :normal
+ :description "user-set")
+ (= "CATEGORY_ID" (:name col)) (assoc :semantic_type :type/Category)))
+ base-cols)]
+ (mt/with-temp
+ [:model/Card {card-id :id} {:type :model
+ :dataset_query (mt/query venues)
+ :result_metadata overridden}]
+ (let [extracted (serdes/extract-one "Card" nil (t2/select-one :model/Card card-id))
+ by-name (u/index-by :name (:result_metadata extracted))]
+ (is (every? #(re-matches #"Custom .*" (:display_name %))
+ (:result_metadata extracted)))
+ (is (= "user-set" (:description (get by-name "ID"))))
+ (is (= :type/Category (:semantic_type (get by-name "CATEGORY_ID"))))
+ (is (vector? (:fk_target_field_id (get by-name "CATEGORY_ID")))))))))
(deftest extract-single-collection-test
(mt/with-empty-h2-app-db!
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index 8e1122c8408..3c16b6bc795 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -1257,14 +1257,18 @@
;;; ------------------------------------------------- Serialization --------------------------------------------------
-(defn- export-result-metadata [metadata]
- (when metadata
- (for [m metadata]
- (-> (dissoc m :fingerprint)
- (m/update-existing :table_id serdes/*export-table-fk*)
- (m/update-existing :id serdes/*export-field-fk*)
- (m/update-existing :field_ref serdes/export-mbql)
- (m/update-existing :fk_target_field_id serdes/*export-field-fk*)))))
+(defn- export-result-metadata [card _k metadata]
+ (if (and (seq metadata) (model? card))
+ (let [native? (lib/native? (:dataset_query card))
+ keep-keys (into #{:name}
+ (map u/->snake_case_en)
+ (lib/model-preserved-keys native?))]
+ (mapv (fn [m]
+ (-> (select-keys m keep-keys)
+ (m/update-existing :fk_target_field_id serdes/*export-field-fk*)
+ (m/update-existing :id serdes/*export-field-fk*)))
+ metadata))
+ ::serdes/skip))
(defn- import-result-metadata [metadata]
(when metadata
@@ -1338,7 +1342,7 @@
:parameters {:export serdes/export-parameters :import serdes/import-parameters}
:parameter_mappings {:export serdes/export-parameter-mappings :import serdes/import-parameter-mappings}
:visualization_settings {:export serdes/export-visualization-settings :import serdes/import-visualization-settings}
- :result_metadata {:export export-result-metadata :import import-result-metadata}}
+ :result_metadata {:export-with-context export-result-metadata :import import-result-metadata}}
:defaults {:archived false
:archived_directly false
:collection_preview true
diff --git a/test/metabase/queries/models/card_test.clj b/test/metabase/queries/models/card_test.clj
index e31aff88cd2..12a8a056aa8 100644
--- a/test/metabase/queries/models/card_test.clj
+++ b/test/metabase/queries/models/card_test.clj
@@ -1,5 +1,6 @@
(ns metabase.queries.models.card-test
(:require
+ [clojure.set :as set]
[clojure.test :refer :all]
[java-time.api :as t]
[metabase.api.common :as api]
@@ -730,24 +731,70 @@
(is (= {["Card" (:id card1)] {"Card" (:id card2)}}
(serdes/descendants "Card" (:id card2) {}))))))
-(deftest ^:parallel extract-test
- (let [metadata (qp.preprocess/query->expected-cols (mt/mbql-query venues))
- query (mt/mbql-query venues)]
- (testing "every card retains result_metadata"
- (mt/with-temp [:model/Card {card1-id :id} {:dataset_query query
- :result_metadata metadata}
- :model/Card {card2-id :id} {:type :model
- :dataset_query query
- :result_metadata metadata}]
- (doseq [card-id [card1-id card2-id]]
- (let [extracted (serdes/extract-one "Card" nil (t2/select-one :model/Card :id card-id))]
- ;; card2 is model, but card1 is not
- (is (= (= card-id card2-id)
- (= :model (:type extracted))))
- (is (string? (:display_name (first (:result_metadata extracted)))))
- ;; this is a quick comparison, since the actual stored metadata is quite complex
- (is (= (map :display_name metadata)
- (map :display_name (:result_metadata extracted))))))))))
+(deftest ^:parallel extract-result-metadata-non-model-test
+ (testing "non-model Card extraction drops :result_metadata entirely"
+ (let [metadata (qp.preprocess/query->expected-cols (mt/mbql-query venues))
+ query (mt/mbql-query venues)]
+ (doseq [card-type [:question :metric]]
+ (testing (str "Card with :type " card-type)
+ (mt/with-temp [:model/Card {card-id :id} {:type card-type
+ :dataset_query query
+ :result_metadata metadata}]
+ (let [extracted (serdes/extract-one "Card" nil (t2/select-one :model/Card :id card-id))]
+ (is (not (contains? extracted :result_metadata))))))))))
+
+(deftest ^:parallel extract-result-metadata-model-test
+ (testing "model Card extraction keeps :name + snake-cased model-preserved-keys only"
+ (let [base (qp.preprocess/query->expected-cols (mt/mbql-query venues))
+ metadata (mapv #(assoc %
+ :display_name "Custom!"
+ :semantic_type :type/Category
+ :visibility_type :normal
+ :description "desc"
+ :fingerprint {:global {:distinct-count 100}}
+ :lib/desired-column-alias "bloat"
+ :qp/internal-flag true)
+ base)
+ query (mt/mbql-query venues)]
+ (mt/with-temp [:model/Card {card-id :id} {:type :model
+ :dataset_query query
+ :result_metadata metadata}]
+ (let [extracted (serdes/extract-one "Card" nil (t2/select-one :model/Card :id card-id))
+ cols (:result_metadata extracted)
+ ;; mirror the export's transformation exactly so this test stays in sync with
+ ;; production if (lib/model-preserved-keys false) ever changes
+ allowed (into #{:name} (map u/->snake_case_en) (lib/model-preserved-keys false))
+ leaked (into #{} (mapcat #(remove allowed (keys %))) cols)]
+ (is (seq cols))
+ (is (= #{} leaked)
+ "no key outside the allowed set leaks through (including bloat keys deliberately seeded)")
+ (is (every? #(= "Custom!" (:display_name %)) cols)
+ "user-set :display_name survives"))))))
+
+(deftest ^:parallel extract-result-metadata-native-model-test
+ (testing "native model Card extraction also preserves :id (as a field FK)"
+ (mt/with-temp [:model/Card {card-id :id}
+ {:type :model
+ :dataset_query (mt/native-query {:query "SELECT ID FROM VENUES"})
+ :result_metadata [{:name "ID"
+ :id (mt/id :venues :id)
+ :display_name "Venue ID"
+ :semantic_type :type/PK
+ :base_type :type/BigInteger}]}]
+ (let [extracted (serdes/extract-one "Card" nil (t2/select-one :model/Card :id card-id))
+ col (first (:result_metadata extracted))]
+ (is (= #{:name :id :display_name :semantic_type}
+ (set (keys col)))
+ "exact set of keys preserved for this fixture (one col with these inputs)")
+ (is (= "Venue ID" (:display_name col)))
+ ;; :id should be portablized to a Field FK path: [db-name schema table-name field-name]
+ (is (=? [string? "PUBLIC" "VENUES" "ID"] (:id col)))
+ ;; cross-reference: nothing outside the snake-cased model-preserved-keys for native models.
+ ;; If `model-preserved-keys` ever changes, the exact-set assertion above stops matching;
+ ;; this guard catches unexpected drift (a new key sneaking in) on the way.
+ (let [allowed (into #{:name} (map u/->snake_case_en) (lib/model-preserved-keys true))
+ leaked (set/difference (set (keys col)) allowed)]
+ (is (= #{} leaked) "no key outside the native-model preserved set"))))))
(deftest ^:parallel upgrade-to-v2-db-test
(testing ":visualization_settings v. 1 should be upgraded to v. 2 on select"