The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GHY-3801 preserve result_metadata for native cards on serdes export (#75259)
Eric Normand18899f65e92c
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.cljindex a84e0c7811a..66b4fb7b23e 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj@@ -16,6 +16,8 @@[metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.models.serialization :as serdes]+ [metabase.query-processor :as qp]+ [metabase.query-processor.compile :as qp.compile][metabase.search.core :as search][metabase.settings.core :as setting][metabase.test :as mt]@@ -930,6 +932,62 @@:breakout [[:field {} (mt/id :orders :user_id)]]}]}}(t2/select-one :model/Card :name "Metric Consuming Question Card"))))))))))))+(deftest gui-question-joined-to-native-source-card-survives-roundtrip-test+ (testing "GUI question joining a native source-card should still run after serdes export+import (GHY-3801)"+ (ts/with-random-dump-dir [dump-dir "serdesv2-"]+ (ts/with-dbs [source-db dest-db]+ (ts/with-db source-db+ (mt/with-temp+ [:model/Collection {coll-id :id} {:name "GHY-3801"}+ :model/Card {native-id :id+ native-eid :entity_id} {:name "Native Products"+ :collection_id coll-id+ :type :question+ :display :table+ :dataset_query {:type :native+ :database (mt/id)+ :native {:query "SELECT * FROM PRODUCTS"}}}+ :model/Card {gui-eid :entity_id} {:name "GUI Joins Native"+ :collection_id coll-id+ :type :question+ :display :table+ :dataset_query (mt/mbql-query nil+ {:source-table (str "card__" native-id)+ :joins [{:fields :all+ :strategy :left-join+ :alias "Products"+ :source-table $$products+ :condition [:=+ [:field "CATEGORY" {:base-type :type/Text}]+ [:field %products.category {:join-alias "Products"}]]}]})}]+ ;; Populate the native source card's result_metadata the way the app does when a user runs and+ ;; saves the query. This is the state serdes must preserve across the round-trip.+ (let [source-cols (-> (qp/process-query (t2/select-one-fn :dataset_query [:model/Card :dataset_query] native-id))+ (get-in [:data :results_metadata :columns]))+ source-names (mapv :name source-cols)]+ (t2/update! :model/Card native-id {:result_metadata source-cols})+ (let [extraction (serdes/with-cache (into [] (extract/extract {})))]+ (storage/store! (seq extraction) (storage.files/file-writer dump-dir)))+ (ts/with-db dest-db+ (is (serdes/with-cache (serdes.load/load-metabase! (ingest/ingest-yaml dump-dir)))+ "serdes ingest succeeded")+ (let [imported-native (t2/select-one :model/Card :entity_id native-eid)+ imported-gui (t2/select-one :model/Card :entity_id gui-eid)]+ (testing "imported native card preserves its result_metadata columns"+ (is (= source-names (mapv :name (:result_metadata imported-native)))+ "Native source card lost result_metadata columns during serdes round-trip"))+ (testing "imported GUI question's query compiles without 'add alias info to join' errors"+ ;; Compile (not execute): the bug surfaces in add-alias-info during compilation, and serdes+ ;; strips warehouse connection details so the imported card can't actually be run here.+ (let [result (try+ (qp.compile/compile (:dataset_query imported-gui))+ (catch Throwable e+ {:error (ex-message e)+ :data (ex-data e)}))]+ (is (not (and (map? result) (:error result)))+ (str "Expected query to compile but got error: "+ (when (map? result) (:error result)))))))))))))))+(deftest schema-coercion-test(ts/with-random-dump-dir [dump-dir "serdesv2-"](mt/with-empty-h2-app-db!diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex 7ba3c3e93b7..e11dc62c171 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -1259,7 +1259,11 @@;;; ------------------------------------------------- Serialization --------------------------------------------------(defn- export-result-metadata [card _k metadata]- (if (and (seq metadata) (model? card))+ (cond+ (empty? metadata)+ ::serdes/skip++ (model? card)(let [native? (lib/native? (:dataset_query card))keep-keys (into #{:name}(map u/->snake_case_en)@@ -1269,6 +1273,25 @@(m/update-existing :fk_target_field_id serdes/*export-field-fk*)(m/update-existing :id serdes/*export-field-fk*)))metadata))++ ;; Native non-model cards keep their result_metadata. Unlike MBQL cards, their columns can't be re-derived+ ;; from the query at import time without executing the SQL, and downstream questions that join them depend+ ;; on this metadata to resolve join columns. We whitelist the portable keys (dropping computed `:lib/*`,+ ;; `:fingerprint`, etc.) rather than the model soft-key set, since native columns also need their structural+ ;; type info preserved.+ (lib/native? (:dataset_query card))+ (let [keep-keys #{:name :base_type :effective_type :field_ref :database_type+ :display_name :semantic_type :description :visibility_type :settings+ :fk_target_field_id :id :table_id}]+ (mapv (fn [m]+ (-> (select-keys m keep-keys)+ (m/update-existing :table_id serdes/*export-table-fk*)+ (m/update-existing :id serdes/*export-field-fk*)+ (m/update-existing :field_ref serdes/export-mbql)+ (m/update-existing :fk_target_field_id serdes/*export-field-fk*)))+ metadata))++ :else::serdes/skip))(defn- import-result-metadata [metadata]