The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Allow loading of 58 serialized tables in 59+ (#70486)
Nicola Mometto8f1606adc504
diff --git a/src/metabase/models/interface.clj b/src/metabase/models/interface.cljindex 862c9cfc7cb..68a48692572 100644--- a/src/metabase/models/interface.clj+++ b/src/metabase/models/interface.clj@@ -264,6 +264,23 @@(throw (ex-info (format "Must be a namespaced keyword under :%s, got: %s" qualified-ns value) {:status-code 400:value value}))))+(defn transform-validator-with-fixes+ "Like [[transform-validator]], but applies `fixes` (a fn of old-value->new-value) before validation on both read+ and write. Use this when an enum has been renamed and old values may still exist in the database or in+ serialization exports."+ [tf assert-fn fixes]+ (-> tf+ (update :out (fn [f]+ (fn [x]+ (let [out (fixes (f x))]+ (assert-fn out)+ out))))+ (update :in (fn [f]+ (fn [x]+ (let [x (fixes x)]+ (assert-fn x)+ (f x)))))))+(defn transform-validator"Given a transform, returns a transform that call `assert-fn` on the \"out\" value.@@ -273,18 +290,7 @@(when-not (-> x namespace some?)(throw (ex-info \"Value is not namespaced\")))))"[tf assert-fn]- (-> tf- ;; deserialization- (update :out (fn [f]- (fn [x]- (let [out (f x)]- (assert-fn out)- out))))- ;; serialization- (update :in (fn [f]- (fn [x]- (assert-fn x)- (f x))))))+ (transform-validator-with-fixes tf assert-fn identity))(def encrypted-json-in"Serialize encrypted json."diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.cljindex dfbd7472c59..f33cc097576 100644--- a/src/metabase/warehouse_schema/models/table.clj+++ b/src/metabase/warehouse_schema/models/table.clj@@ -110,25 +110,21 @@:status-code 400}))))(some-> value name))})-(def ^:private transform-data-layer- {:in (:in mi/transform-keyword)- :out (comp (some-fn data-layers- ;; some databases may retain prior medallion values- ;; this is due to race conditions with migrations (migration runs can contend with running writes)- ;; e.g. we saw the prior values retained for the sync of a new search index table in stats.- ;; we might later have a second migration and risk removing this map or find a better way- ;; to make breaking enum migrations.- {:copper :hidden- :bronze :final- :silver :final- :gold :final}- identity)- (:out mi/transform-keyword))})+(def ^:private legacy-data-layer->current+ "Map old medallion data_layer values to current values.+ Used to handle values from pre-v59 databases or serialization exports."+ {:copper :hidden+ :bronze :final+ :silver :final+ :gold :final})(t2/deftransforms :model/Table{:entity_type mi/transform-keyword:visibility_type mi/transform-keyword- :data_layer (mi/transform-validator transform-data-layer (partial mi/assert-optional-enum data-layers))+ :data_layer (mi/transform-validator-with-fixes+ mi/transform-keyword+ (partial mi/assert-optional-enum data-layers)+ (some-fn legacy-data-layer->current identity)):field_order mi/transform-keyword:data_source (mi/transform-validator mi/transform-keyword (partial mi/assert-optional-enum data-sources));; Warning: by using a transform to handle unexpected enum values, serialization becomes lossydiff --git a/test/metabase/warehouse_schema/models/table_test.clj b/test/metabase/warehouse_schema/models/table_test.cljindex 0a1505689e9..5cbd7658c6e 100644--- a/test/metabase/warehouse_schema/models/table_test.clj+++ b/test/metabase/warehouse_schema/models/table_test.clj@@ -717,3 +717,20 @@(let [descendants (serdes/descendants "Table" table-id {})](is (contains? descendants ["Segment" active-seg-id]))(is (contains? descendants ["Segment" archived-seg-id])))))))++(deftest data-layer-legacy-serdes-round-trip-test+ (testing "Importing a v58 serialization export with legacy medallion data_layer values maps them to current values"+ (mt/with-temp [:model/Database {db-id :id} {:name "Test DB"}+ :model/Table {table-id :id} {:db_id db-id}]+ (let [table (t2/select-one :model/Table :id table-id)+ extracted (serdes/extract-one "Table" {} table)]+ (doseq [[legacy-value expected] {"copper" "hidden"+ "bronze" "final"+ "silver" "final"+ "gold" "final"}]+ (testing (format "legacy data_layer %s -> %s" legacy-value expected)+ ;; simulate a v58 export by substituting the legacy value+ (let [ingested (assoc extracted :data_layer legacy-value)]+ (serdes/load-one! ingested table)+ (is (= (keyword expected)+ (t2/select-one-fn :data_layer :model/Table :id table-id))))))))))