Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix serialization import for metrics with dimensions (#83017)

Timofey Kachalovb26dbd55389f
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
index f0a8c144c40..5820ea4f0a1 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
@@ -933,6 +933,50 @@
:breakout [[:field {} (mt/id :orders :user_id)]]}]}}
(t2/select-one :model/Card :name "Metric Consuming Question Card"))))))))))))
+(deftest metric-dimensions-round-trip-test
+ (testing "a metric's curated dimensions and mappings survive export to YAML files and import"
+ (ts/with-random-dump-dir [dump-dir "serdesv2-"]
+ (ts/with-dbs [source-db dest-db]
+ (ts/with-db source-db
+ (let [dim-id "11111111-1111-4111-8111-111111111111"]
+ (mt/with-temp
+ [:model/Collection {coll-id :id} {:name "Collection"}
+ :model/Card _metric {:name "Metric With Dimensions"
+ :collection_id coll-id
+ :type :metric
+ :dataset_query (mt/mbql-query orders {:aggregation [[:count]]})
+ :dimensions [{:id dim-id
+ :name "CATEGORY"
+ :display-name "Category"
+ :effective-type :type/Text
+ :status :status/active
+ :sources [{:type :field
+ :field-id (mt/id :products :category)}]}]
+ :dimension_mappings [{:type :table
+ :table-id (mt/id :products)
+ :dimension-id dim-id
+ :target [:field
+ {:lib/uuid "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"
+ :source-field (mt/id :orders :product_id)}
+ (mt/id :products :category)]}]}]
+ (let [extraction (serdes/with-cache (into [] (extract/extract {})))]
+ (storage/store! (seq extraction) (storage.files/file-writer dump-dir)))
+ (ts/with-db dest-db
+ (is (serdes/with-cache (serdes.load/load-metabase! (ingest/ingest-yaml dump-dir)))
+ "successful")
+ (is (=? {:dimensions [{:id dim-id
+ :effective-type :type/Text
+ :status :status/active
+ :sources [{:type :field
+ :field-id (mt/id :products :category)}]}]
+ :dimension_mappings [{:type :table
+ :table-id (mt/id :products)
+ :dimension-id dim-id
+ :target [:field
+ {:source-field (mt/id :orders :product_id)}
+ (mt/id :products :category)]}]}
+ (t2/select-one :model/Card :name "Metric With Dimensions")))))))))))
+
(deftest gui-question-joined-to-native-source-card-survives-roundtrip-test
(testing "GUI question joining a native source-card should still run after serdes export+import (GHY-3801)"
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/metric_dimensions_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/metric_dimensions_test.clj
index 3a37b18a1b2..e63d62eaf5f 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/metric_dimensions_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/metric_dimensions_test.clj
@@ -2,6 +2,7 @@
(:require
[clojure.test :refer :all]
[medley.core :as m]
+ [metabase-enterprise.serialization.dump :as dump]
[metabase-enterprise.serialization.test-util :as ts]
[metabase-enterprise.serialization.v2.extract :as serdes.extract]
[metabase-enterprise.serialization.v2.ingest :as serdes.ingest]
@@ -13,6 +14,7 @@
[metabase.models.serialization :as serdes]
[metabase.search.core :as search]
[metabase.test :as mt]
+ [metabase.util.yaml :as yaml]
[metabase.warehouses.models.database :as models.database]
[toucan2.core :as t2]))
@@ -39,6 +41,13 @@
(ingest-errors [_]
[]))))
+(defn- through-yaml
+ "What a file-based import reads back: YAML carries no keywords, so keyword values come back as strings."
+ [entity]
+ (-> (dump/yaml-content entity)
+ (yaml/parse-string {:key-fn serdes.ingest/parse-key})
+ serdes.ingest/read-timestamps))
+
(defn- by-model [entities model-name]
(filter #(-> % :serdes/meta last :model (= model-name)) entities))
@@ -159,6 +168,16 @@
(serdes.load/load-metabase! (ingestion-in-memory serialized))
(assert-remapped-import! source))))))
+(deftest metric-v2-dimensions-yaml-round-trip-test
+ (testing "curated metric dimensions and mappings import from YAML, where their types are strings"
+ (ts/with-dbs [source-db dest-db]
+ (let [source (ts/with-db source-db (create-source-metric!))
+ serialized (ts/with-db source-db (mapv through-yaml (serdes.extract/extract {})))]
+ (ts/with-db dest-db
+ (prepare-destination!)
+ (serdes.load/load-metabase! (ingestion-in-memory serialized))
+ (assert-remapped-import! source))))))
+
(deftest metric-v2-dimensions-deletion-propagates-test
(testing "deleting a curated dimension on the source deletes it on the destination after import"
(ts/with-dbs [source-db dest-db]
diff --git a/src/metabase/metrics/transforms.clj b/src/metabase/metrics/transforms.clj
index 59da3b1880b..14b34865072 100644
--- a/src/metabase/metrics/transforms.clj
+++ b/src/metabase/metrics/transforms.clj
@@ -109,10 +109,11 @@
(mapv export-dimension dimensions)))
(defn import-dimensions
- "Inverse of [[export-dimensions]]."
+ "Inverse of [[export-dimensions]]. Normalized like [[transform-dimensions]] output, since a file-based import reads
+ keyword values back as strings."
[dimensions]
(when (some? dimensions)
- (mapv import-dimension dimensions)))
+ (mapv (comp normalize-dimension import-dimension) dimensions)))
(defn export-dimension-mappings
"Serialize a curated entity's `:dimension_mappings`: convert `:table-id` and the Field IDs inside each mapping's
@@ -126,13 +127,15 @@
mappings)))
(defn import-dimension-mappings
- "Inverse of [[export-dimension-mappings]]."
+ "Inverse of [[export-dimension-mappings]]. Normalized like [[transform-dimension-mappings]] output, for the same
+ reason as [[import-dimensions]]."
[mappings]
(when (some? mappings)
(mapv (fn [mapping]
- (cond-> mapping
- (vector? (:table-id mapping)) (update :table-id serdes/*import-table-fk*)
- (:target mapping) (update :target serdes/import-mbql)))
+ (normalize-dimension-mapping
+ (cond-> mapping
+ (vector? (:table-id mapping)) (update :table-id serdes/*import-table-fk*)
+ (:target mapping) (update :target serdes/import-mbql))))
mappings)))
(defn dimension-mappings-deps