Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Preserve native model column types during serialization (GHY-4043) (#77405)

Eric Normand950f45353522
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
index c507a46b1e0..496711b23bb 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
@@ -1760,6 +1760,36 @@
(is (= :type/Category (:semantic_type (get by-name "CATEGORY_ID"))))
(is (vector? (:fk_target_field_id (get by-name "CATEGORY_ID")))))))))
+(deftest native-model-preserves-column-types-extract-test
+ (testing "native model Card extraction preserves structural column types (GHY-4043)"
+ ;; Native model columns can't be re-derived from the query at import time without executing
+ ;; the SQL, so :base_type/:effective_type must survive extract — otherwise filters on the
+ ;; read-only target instance collapse to Is empty / Not empty.
+ (mt/with-temp
+ [:model/Card {card-id :id}
+ {:type :model
+ :dataset_query (mt/native-query
+ {:query "SELECT project_id, STRING_AGG(region_name, ', ') AS names FROM project_regions GROUP BY project_id"})
+ :result_metadata [{:name "project_id"
+ :display_name "project_id"
+ :base_type :type/Integer
+ :effective_type :type/Integer
+ :semantic_type :type/PK}
+ {:name "names"
+ :display_name "names"
+ :base_type :type/Text
+ :effective_type :type/Text
+ :semantic_type :type/Title}]}]
+ (let [extracted (serdes/extract-one "Card" nil (t2/select-one :model/Card card-id))
+ by-name (u/index-by :name (:result_metadata extracted))]
+ (testing "physical passthrough column keeps its types"
+ (is (= :type/Integer (:base_type (get by-name "project_id"))))
+ (is (= :type/Integer (:effective_type (get by-name "project_id")))))
+ (testing "aggregated projection column keeps its types and soft keys"
+ (is (= :type/Text (:base_type (get by-name "names"))))
+ (is (= :type/Text (:effective_type (get by-name "names"))))
+ (is (= :type/Title (:semantic_type (get by-name "names")))))))))
+
(deftest extract-single-collection-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index 3c627d2ff7e..b1aff765748 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -1302,22 +1302,11 @@
(empty? metadata)
::serdes/skip
- (model? card)
- (let [native? (lib/native? (:dataset_query card))
- keep-keys (into #{:name}
- (map u/->snake_case_en)
- (lib/model-preserved-keys native?))]
- (mapv (fn [m]
- (-> (select-keys m keep-keys)
- (m/update-existing :fk_target_field_id serdes/*export-field-fk*)
- (m/update-existing :id serdes/*export-field-fk*)))
- metadata))
-
- ;; Native non-model cards keep their result_metadata. Unlike MBQL cards, their columns can't be re-derived
- ;; from the query at import time without executing the SQL, and downstream questions that join them depend
- ;; on this metadata to resolve join columns. We whitelist the portable keys (dropping computed `:lib/*`,
+ ;; Native cards — models included — keep their result_metadata. Unlike MBQL cards, their columns can't be
+ ;; re-derived from the query at import time without executing the SQL, and downstream questions that join them
+ ;; depend on this metadata to resolve join columns. We whitelist the portable keys (dropping computed `:lib/*`,
;; `:fingerprint`, etc.) rather than the model soft-key set, since native columns also need their structural
- ;; type info preserved.
+ ;; type info preserved. This must precede the model branch, whose soft-key set omits type info.
(lib/native? (:dataset_query card))
(let [keep-keys #{:name :base_type :effective_type :field_ref :database_type
:display_name :semantic_type :description :visibility_type :settings
@@ -1330,6 +1319,16 @@
(m/update-existing :fk_target_field_id serdes/*export-field-fk*)))
metadata))
+ (model? card)
+ (let [keep-keys (into #{:name}
+ (map u/->snake_case_en)
+ (lib/model-preserved-keys false))]
+ (mapv (fn [m]
+ (-> (select-keys m keep-keys)
+ (m/update-existing :fk_target_field_id serdes/*export-field-fk*)
+ (m/update-existing :id serdes/*export-field-fk*)))
+ metadata))
+
:else
::serdes/skip))
diff --git a/test/metabase/queries/models/card_test.clj b/test/metabase/queries/models/card_test.clj
index d0c6cb59bb1..f35b696db53 100644
--- a/test/metabase/queries/models/card_test.clj
+++ b/test/metabase/queries/models/card_test.clj
@@ -1,7 +1,6 @@
(ns metabase.queries.models.card-test
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.queries.models.card-test]}}}}}}
(:require
- [clojure.set :as set]
[clojure.test :refer :all]
[java-time.api :as t]
[metabase.api.common :as api]
@@ -808,7 +807,10 @@
"user-set :display_name survives"))))))
(deftest ^:parallel extract-result-metadata-native-model-test
- (testing "native model Card extraction also preserves :id (as a field FK)"
+ (testing "native model Card extraction preserves :id (as a field FK) and structural column types"
+ ;; Native model columns can't be re-derived from the query at import time without executing the
+ ;; SQL, so their :base_type/:effective_type must survive extract (GHY-4043). Unlike MBQL models,
+ ;; native models serialize through the native-card whitelist, not model-preserved-keys.
(mt/with-temp [:model/Card {card-id :id}
{:type :model
:dataset_query (mt/native-query {:query "SELECT ID FROM VENUES"})
@@ -819,18 +821,14 @@
:base_type :type/BigInteger}]}]
(let [extracted (serdes/extract-one "Card" nil (t2/select-one :model/Card :id card-id))
col (first (:result_metadata extracted))]
- (is (= #{:name :id :display_name :semantic_type}
+ (is (= #{:name :id :display_name :semantic_type :base_type}
(set (keys col)))
"exact set of keys preserved for this fixture (one col with these inputs)")
(is (= "Venue ID" (:display_name col)))
+ (is (= :type/BigInteger (:base_type col))
+ "native model keeps structural type info the target can't re-derive")
;; :id should be portablized to a Field FK path: [db-name schema table-name field-name]
- (is (=? [string? "PUBLIC" "VENUES" "ID"] (:id col)))
- ;; cross-reference: nothing outside the snake-cased model-preserved-keys for native models.
- ;; If `model-preserved-keys` ever changes, the exact-set assertion above stops matching;
- ;; this guard catches unexpected drift (a new key sneaking in) on the way.
- (let [allowed (into #{:name} (map u/->snake_case_en) (lib/model-preserved-keys true))
- leaked (set/difference (set (keys col)) allowed)]
- (is (= #{} leaked) "no key outside the native-model preserved set"))))))
+ (is (=? [string? "PUBLIC" "VENUES" "ID"] (:id col)))))))
(deftest ^:parallel upgrade-to-v2-db-test
(testing ":visualization_settings v. 1 should be upgraded to v. 2 on select"