Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Allow loading of 58 serialized tables in 59+ (#70486)

Nicola Mometto8f1606adc504
diff --git a/src/metabase/models/interface.clj b/src/metabase/models/interface.clj
index 862c9cfc7cb..68a48692572 100644
--- a/src/metabase/models/interface.clj
+++ b/src/metabase/models/interface.clj
@@ -264,6 +264,23 @@
(throw (ex-info (format "Must be a namespaced keyword under :%s, got: %s" qualified-ns value) {:status-code 400
:value value}))))
+(defn transform-validator-with-fixes
+ "Like [[transform-validator]], but applies `fixes` (a fn of old-value->new-value) before validation on both read
+ and write. Use this when an enum has been renamed and old values may still exist in the database or in
+ serialization exports."
+ [tf assert-fn fixes]
+ (-> tf
+ (update :out (fn [f]
+ (fn [x]
+ (let [out (fixes (f x))]
+ (assert-fn out)
+ out))))
+ (update :in (fn [f]
+ (fn [x]
+ (let [x (fixes x)]
+ (assert-fn x)
+ (f x)))))))
+
(defn transform-validator
"Given a transform, returns a transform that call `assert-fn` on the \"out\" value.
@@ -273,18 +290,7 @@
(when-not (-> x namespace some?)
(throw (ex-info \"Value is not namespaced\")))))"
[tf assert-fn]
- (-> tf
- ;; deserialization
- (update :out (fn [f]
- (fn [x]
- (let [out (f x)]
- (assert-fn out)
- out))))
- ;; serialization
- (update :in (fn [f]
- (fn [x]
- (assert-fn x)
- (f x))))))
+ (transform-validator-with-fixes tf assert-fn identity))
(def encrypted-json-in
"Serialize encrypted json."
diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.clj
index dfbd7472c59..f33cc097576 100644
--- a/src/metabase/warehouse_schema/models/table.clj
+++ b/src/metabase/warehouse_schema/models/table.clj
@@ -110,25 +110,21 @@
:status-code 400}))))
(some-> value name))})
-(def ^:private transform-data-layer
- {:in (:in mi/transform-keyword)
- :out (comp (some-fn data-layers
- ;; some databases may retain prior medallion values
- ;; this is due to race conditions with migrations (migration runs can contend with running writes)
- ;; e.g. we saw the prior values retained for the sync of a new search index table in stats.
- ;; we might later have a second migration and risk removing this map or find a better way
- ;; to make breaking enum migrations.
- {:copper :hidden
- :bronze :final
- :silver :final
- :gold :final}
- identity)
- (:out mi/transform-keyword))})
+(def ^:private legacy-data-layer->current
+ "Map old medallion data_layer values to current values.
+ Used to handle values from pre-v59 databases or serialization exports."
+ {:copper :hidden
+ :bronze :final
+ :silver :final
+ :gold :final})
(t2/deftransforms :model/Table
{:entity_type mi/transform-keyword
:visibility_type mi/transform-keyword
- :data_layer (mi/transform-validator transform-data-layer (partial mi/assert-optional-enum data-layers))
+ :data_layer (mi/transform-validator-with-fixes
+ mi/transform-keyword
+ (partial mi/assert-optional-enum data-layers)
+ (some-fn legacy-data-layer->current identity))
:field_order mi/transform-keyword
:data_source (mi/transform-validator mi/transform-keyword (partial mi/assert-optional-enum data-sources))
;; Warning: by using a transform to handle unexpected enum values, serialization becomes lossy
diff --git a/test/metabase/warehouse_schema/models/table_test.clj b/test/metabase/warehouse_schema/models/table_test.clj
index 0a1505689e9..5cbd7658c6e 100644
--- a/test/metabase/warehouse_schema/models/table_test.clj
+++ b/test/metabase/warehouse_schema/models/table_test.clj
@@ -717,3 +717,20 @@
(let [descendants (serdes/descendants "Table" table-id {})]
(is (contains? descendants ["Segment" active-seg-id]))
(is (contains? descendants ["Segment" archived-seg-id])))))))
+
+(deftest data-layer-legacy-serdes-round-trip-test
+ (testing "Importing a v58 serialization export with legacy medallion data_layer values maps them to current values"
+ (mt/with-temp [:model/Database {db-id :id} {:name "Test DB"}
+ :model/Table {table-id :id} {:db_id db-id}]
+ (let [table (t2/select-one :model/Table :id table-id)
+ extracted (serdes/extract-one "Table" {} table)]
+ (doseq [[legacy-value expected] {"copper" "hidden"
+ "bronze" "final"
+ "silver" "final"
+ "gold" "final"}]
+ (testing (format "legacy data_layer %s -> %s" legacy-value expected)
+ ;; simulate a v58 export by substituting the legacy value
+ (let [ingested (assoc extracted :data_layer legacy-value)]
+ (serdes/load-one! ingested table)
+ (is (= (keyword expected)
+ (t2/select-one-fn :data_layer :model/Table :id table-id))))))))))