The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Copy model metadata overrides to Fields during models-to-transforms (#72129)
bryandb672b40435d
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 659f56ab9d6..5843e1eccd2 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2001,7 +2001,8 @@source-swap{:team "Graphy":api #{metabase.source-swap.core- metabase.source-swap.schema}+ metabase.source-swap.schema+ metabase.source-swap.util}:uses #{driverlibsql-tools@@ -3044,6 +3045,7 @@tasktransformsutil+ warehouse-schemaenterprise/dependencies}:model-imports #{:model/Card:model/Dashboarddiff --git a/enterprise/backend/src/metabase_enterprise/replacement/runner.clj b/enterprise/backend/src/metabase_enterprise/replacement/runner.cljindex db95550d234..718285be410 100644--- a/enterprise/backend/src/metabase_enterprise/replacement/runner.clj+++ b/enterprise/backend/src/metabase_enterprise/replacement/runner.clj@@ -1,6 +1,7 @@(ns metabase-enterprise.replacement.runner(:require[clojure.string :as str]+ [medley.core :as m][metabase-enterprise.replacement.field-refs :as replacement.field-refs][metabase-enterprise.replacement.protocols :as replacement.protocols][metabase-enterprise.replacement.source-swap :as replacement.source-swap]@@ -10,8 +11,11 @@[metabase.lib.core :as lib][metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.model-persistence.core :as model-persistence]+ [metabase.source-swap.util :as source-swap.util][metabase.transforms.core :as transforms]+ [metabase.util :as u][metabase.util.log :as log]+ [metabase.warehouse-schema.models.field-user-settings :as field-user-settings][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -165,6 +169,26 @@(run-swap* {:all-transitive-dependents all-transitive}old-source new-source progress))))+(def ^:private metadata-override-keys+ "Field columns that correspond to user-editable model metadata overrides.+ These are in snake_case matching both result_metadata storage and Field columns."+ [:description :display_name :semantic_type :fk_target_field_id :settings :visibility_type])++(defn- copy-model-metadata-overrides!+ "Copy user-edited metadata from a model's result_metadata onto the Fields of the+ output table. Writes to both Field and FieldUserSettings so overrides survive sync."+ [card-id table-id]+ (let [card (t2/select-one :model/Card :id card-id)+ result-metadata (:result_metadata card)+ fields (t2/select :model/Field :table_id table-id :active true)+ field-by-name (m/index-by :name fields)]+ (doseq [col-meta result-metadata+ :let [field (field-by-name (source-swap.util/column-match-key col-meta))+ overrides (u/select-keys-when col-meta :non-nil metadata-override-keys)]+ :when (and field (seq overrides))]+ (t2/update! :model/Field (:id field) overrides)+ (field-user-settings/upsert-user-settings field overrides))))+(defn run-swap-model-with-transform!"Execute a transform, find the output table, then swap all dependents of the cardto point at the new table. Finally un-persist and convert the card to a saved question.@@ -181,10 +205,11 @@(transforms/execute! transform (cond-> {:run-method :manual}user-id (assoc :user-id user-id)))- ;; phase 2: find the output table and swap sources+ ;; phase 2: find the output table, copy metadata overrides, and swap sources(let [table (or (transforms/output-table transform)(throw (ex-info "Output table not found after transform execution"{:transform-id (:id transform)})))]+ (copy-model-metadata-overrides! card-id (:id table))(run-swap-source! [:card card-id] [:table (:id table)] progress));; phase 3: unpersist the model if it was persisteddiff --git a/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/runner_test.cljindex c63b164bd68..e52e6130f0a 100644--- a/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj+++ b/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj@@ -12,6 +12,8 @@[metabase.lib.metadata :as lib.metadata][metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.test :as mt]+ [metabase.util.json :as json]+ [metabase.warehouse-schema.models.field-user-settings][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -275,3 +277,96 @@(testing "child-1 retains original source (swap failed)"(is (= old-id (get-in (t2/select-one-fn :dataset_query :model/Card :id child-1-id)[:stages 0 :source-card]))))))))))))))++(deftest copy-model-metadata-overrides!-test+ (testing "copies user-edited metadata from model result_metadata to Field and FieldUserSettings"+ (mt/with-temp [:model/Table {table-id :id} {:name "transform_output"+ :db_id (mt/id)+ :active true}+ :model/Field {field-1-id :id} {:name "TOTAL"+ :table_id table-id+ :base_type :type/Float+ :display_name "Total"+ :description nil+ :semantic_type nil}+ :model/Field {field-2-id :id} {:name "CREATED_AT"+ :table_id table-id+ :base_type :type/DateTimeWithLocalTZ+ :display_name "Created At"+ :description nil+ :semantic_type nil}+ :model/Card {card-id :id} {:type :model+ :database_id (mt/id)+ :dataset_query (mt/mbql-query orders)}]+ ;; Set result_metadata directly via SQL to bypass Card hooks that recompute metadata+ (t2/query-one {:update :report_card+ :set {:result_metadata+ (json/encode [{:name "TOTAL"+ :display_name "Order Total"+ :description "The total amount"+ :semantic_type "type/Currency"+ :base_type "type/Float"}+ {:name "CREATED_AT"+ :display_name "Order Date"+ :semantic_type "type/CreationTimestamp"+ :base_type "type/DateTimeWithLocalTZ"}])}+ :where [:= :id card-id]})++ (#'replacement.runner/copy-model-metadata-overrides! card-id table-id)++ (testing "Field records are updated with overrides from model metadata"+ (let [field-1 (t2/select-one :model/Field :id field-1-id)+ field-2 (t2/select-one :model/Field :id field-2-id)]+ (is (= "Order Total" (:display_name field-1)))+ (is (= "The total amount" (:description field-1)))+ (is (= :type/Currency (:semantic_type field-1)))++ (is (= "Order Date" (:display_name field-2)))+ (is (= :type/CreationTimestamp (:semantic_type field-2)))))++ (testing "FieldUserSettings are created so overrides survive sync"+ (let [fus-1 (t2/select-one :model/FieldUserSettings :field_id field-1-id)+ fus-2 (t2/select-one :model/FieldUserSettings :field_id field-2-id)]+ (is (some? fus-1))+ (is (= "Order Total" (:display_name fus-1)))+ (is (= "The total amount" (:description fus-1)))+ (is (= :type/Currency (:semantic_type fus-1)))++ (is (some? fus-2))+ (is (= "Order Date" (:display_name fus-2)))+ (is (= :type/CreationTimestamp (:semantic_type fus-2)))))))++ (testing "matches joined columns using :lib/desired-column-alias instead of :name"+ (mt/with-temp [:model/Table {table-id :id} {:name "transform_joined_output"+ :db_id (mt/id)+ :active true}+ ;; The output table has a field named Products__ID (from the join)+ :model/Field {field-id :id} {:name "Products__ID"+ :table_id table-id+ :base_type :type/Integer+ :display_name "Products ID"+ :description nil+ :semantic_type nil}+ :model/Card {card-id :id} {:type :model+ :database_id (mt/id)+ :dataset_query (mt/mbql-query orders)}]+ ;; result_metadata has :name "ID" but :lib/desired-column-alias "Products__ID"+ (t2/query-one {:update :report_card+ :set {:result_metadata+ (json/encode [{:name "ID"+ :lib/desired-column-alias "Products__ID"+ :display_name "Product ID"+ :description "The product identifier"+ :base_type "type/Integer"}])}+ :where [:= :id card-id]})++ (#'replacement.runner/copy-model-metadata-overrides! card-id table-id)++ (let [field (t2/select-one :model/Field :id field-id)]+ (is (= "Product ID" (:display_name field)))+ (is (= "The product identifier" (:description field))))++ (let [fus (t2/select-one :model/FieldUserSettings :field_id field-id)]+ (is (some? fus))+ (is (= "Product ID" (:display_name fus)))+ (is (= "The product identifier" (:description fus)))))))diff --git a/src/metabase/source_swap/util.clj b/src/metabase/source_swap/util.cljindex a2d4136328e..ee480f76d95 100644--- a/src/metabase/source_swap/util.clj+++ b/src/metabase/source_swap/util.clj@@ -11,7 +11,7 @@(mu/defn column-match-key :- :string"Gets the column match key from a column. This is used to match columns by name or alias."- [column :- ::lib.schema.metadata/column]+ [column :- [:map [:name :string]]](or (:lib/desired-column-alias column) (:name column)))(mu/defn source-columns :- [:sequential ::lib.schema.metadata/column]