The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Preserve native model column types during serialization (GHY-4043) (#77405)
Eric Normand950f45353522
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.cljindex c507a46b1e0..496711b23bb 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj@@ -1760,6 +1760,36 @@(is (= :type/Category (:semantic_type (get by-name "CATEGORY_ID"))))(is (vector? (:fk_target_field_id (get by-name "CATEGORY_ID")))))))))+(deftest native-model-preserves-column-types-extract-test+ (testing "native model Card extraction preserves structural column types (GHY-4043)"+ ;; Native model columns can't be re-derived from the query at import time without executing+ ;; the SQL, so :base_type/:effective_type must survive extract — otherwise filters on the+ ;; read-only target instance collapse to Is empty / Not empty.+ (mt/with-temp+ [:model/Card {card-id :id}+ {:type :model+ :dataset_query (mt/native-query+ {:query "SELECT project_id, STRING_AGG(region_name, ', ') AS names FROM project_regions GROUP BY project_id"})+ :result_metadata [{:name "project_id"+ :display_name "project_id"+ :base_type :type/Integer+ :effective_type :type/Integer+ :semantic_type :type/PK}+ {:name "names"+ :display_name "names"+ :base_type :type/Text+ :effective_type :type/Text+ :semantic_type :type/Title}]}]+ (let [extracted (serdes/extract-one "Card" nil (t2/select-one :model/Card card-id))+ by-name (u/index-by :name (:result_metadata extracted))]+ (testing "physical passthrough column keeps its types"+ (is (= :type/Integer (:base_type (get by-name "project_id"))))+ (is (= :type/Integer (:effective_type (get by-name "project_id")))))+ (testing "aggregated projection column keeps its types and soft keys"+ (is (= :type/Text (:base_type (get by-name "names"))))+ (is (= :type/Text (:effective_type (get by-name "names"))))+ (is (= :type/Title (:semantic_type (get by-name "names")))))))))+(deftest extract-single-collection-test(mt/with-empty-h2-app-db!(ts/with-temp-dpcdiff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex 3c627d2ff7e..b1aff765748 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -1302,22 +1302,11 @@(empty? metadata)::serdes/skip- (model? card)- (let [native? (lib/native? (:dataset_query card))- keep-keys (into #{:name}- (map u/->snake_case_en)- (lib/model-preserved-keys native?))]- (mapv (fn [m]- (-> (select-keys m keep-keys)- (m/update-existing :fk_target_field_id serdes/*export-field-fk*)- (m/update-existing :id serdes/*export-field-fk*)))- metadata))-- ;; Native non-model cards keep their result_metadata. Unlike MBQL cards, their columns can't be re-derived- ;; from the query at import time without executing the SQL, and downstream questions that join them depend- ;; on this metadata to resolve join columns. We whitelist the portable keys (dropping computed `:lib/*`,+ ;; Native cards — models included — keep their result_metadata. Unlike MBQL cards, their columns can't be+ ;; re-derived from the query at import time without executing the SQL, and downstream questions that join them+ ;; depend on this metadata to resolve join columns. We whitelist the portable keys (dropping computed `:lib/*`,;; `:fingerprint`, etc.) rather than the model soft-key set, since native columns also need their structural- ;; type info preserved.+ ;; type info preserved. This must precede the model branch, whose soft-key set omits type info.(lib/native? (:dataset_query card))(let [keep-keys #{:name :base_type :effective_type :field_ref :database_type:display_name :semantic_type :description :visibility_type :settings@@ -1330,6 +1319,16 @@(m/update-existing :fk_target_field_id serdes/*export-field-fk*)))metadata))+ (model? card)+ (let [keep-keys (into #{:name}+ (map u/->snake_case_en)+ (lib/model-preserved-keys false))]+ (mapv (fn [m]+ (-> (select-keys m keep-keys)+ (m/update-existing :fk_target_field_id serdes/*export-field-fk*)+ (m/update-existing :id serdes/*export-field-fk*)))+ metadata))+:else::serdes/skip))diff --git a/test/metabase/queries/models/card_test.clj b/test/metabase/queries/models/card_test.cljindex d0c6cb59bb1..f35b696db53 100644--- a/test/metabase/queries/models/card_test.clj+++ b/test/metabase/queries/models/card_test.clj@@ -1,7 +1,6 @@(ns metabase.queries.models.card-test{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.queries.models.card-test]}}}}}}(:require- [clojure.set :as set][clojure.test :refer :all][java-time.api :as t][metabase.api.common :as api]@@ -808,7 +807,10 @@"user-set :display_name survives"))))))(deftest ^:parallel extract-result-metadata-native-model-test- (testing "native model Card extraction also preserves :id (as a field FK)"+ (testing "native model Card extraction preserves :id (as a field FK) and structural column types"+ ;; Native model columns can't be re-derived from the query at import time without executing the+ ;; SQL, so their :base_type/:effective_type must survive extract (GHY-4043). Unlike MBQL models,+ ;; native models serialize through the native-card whitelist, not model-preserved-keys.(mt/with-temp [:model/Card {card-id :id}{:type :model:dataset_query (mt/native-query {:query "SELECT ID FROM VENUES"})@@ -819,18 +821,14 @@:base_type :type/BigInteger}]}](let [extracted (serdes/extract-one "Card" nil (t2/select-one :model/Card :id card-id))col (first (:result_metadata extracted))]- (is (= #{:name :id :display_name :semantic_type}+ (is (= #{:name :id :display_name :semantic_type :base_type}(set (keys col)))"exact set of keys preserved for this fixture (one col with these inputs)")(is (= "Venue ID" (:display_name col)))+ (is (= :type/BigInteger (:base_type col))+ "native model keeps structural type info the target can't re-derive");; :id should be portablized to a Field FK path: [db-name schema table-name field-name]- (is (=? [string? "PUBLIC" "VENUES" "ID"] (:id col)))- ;; cross-reference: nothing outside the snake-cased model-preserved-keys for native models.- ;; If `model-preserved-keys` ever changes, the exact-set assertion above stops matching;- ;; this guard catches unexpected drift (a new key sneaking in) on the way.- (let [allowed (into #{:name} (map u/->snake_case_en) (lib/model-preserved-keys true))- leaked (set/difference (set (keys col)) allowed)]- (is (= #{} leaked) "no key outside the native-model preserved set"))))))+ (is (=? [string? "PUBLIC" "VENUES" "ID"] (:id col)))))))(deftest ^:parallel upgrade-to-v2-db-test(testing ":visualization_settings v. 1 should be upgraded to v. 2 on select"