Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Serdes for cards, segments, measures, transforms - allow deserializing only with minimal required properties without computed ones (#71464)

Alexander Polyankinb7e1d1eff8aa
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
index e67c71b803e..24abeeef4ea 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
@@ -357,6 +357,22 @@
:database (:id @db1d)}
(:definition @seg1d))))))))))
+(defn- pmbql-query
+ "Create a simple pMBQL query for the given database and table."
+ [db-id table-id]
+ (let [metadata-provider (lib-be/application-database-metadata-provider db-id)
+ table (lib.metadata/table metadata-provider table-id)]
+ (lib/query metadata-provider table)))
+
+(defn- pmbql-segment-definition
+ "Create a simple pMBQL segment definition with a filter."
+ [db-id table-id field-id]
+ (let [metadata-provider (lib-be/application-database-metadata-provider db-id)
+ table (lib.metadata/table metadata-provider table-id)
+ query (lib/query metadata-provider table)
+ field (lib.metadata/field metadata-provider field-id)]
+ (lib/filter query (lib/< field 18))))
+
(defn- pmbql-measure-definition
"Create an MBQL5 measure definition with a sum aggregation."
[db-id table-id field-id]
@@ -1855,3 +1871,213 @@
Exception
#"source-only-db|not found|Failed"
(serdes.load/load-metabase! (ingestion-in-memory @serialized))))))))))
+
+(deftest segment-minimal-required-properties-test
+ (testing "Segment deserialized with only: entity_id, name, definition, creator_id"
+ (let [serialized (atom nil)]
+ (ts/with-dbs [source-db dest-db]
+ (ts/with-db source-db
+ (let [db (ts/create! :model/Database :name "my-db")
+ table (ts/create! :model/Table :name "customers" :db_id (:id db))
+ field (ts/create! :model/Field :name "age" :table_id (:id table))
+ user (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")
+ _segment (ts/create! :model/Segment :table_id (:id table) :name "Minors"
+ :definition (pmbql-segment-definition (:id db) (:id table) (:id field))
+ :creator_id (:id user))]
+ (reset! serialized (into [] (serdes.extract/extract {})))))
+
+ (let [minimal (mapv (fn [entity]
+ (if (= "Segment" (-> entity :serdes/meta last :model))
+ (select-keys entity [:serdes/meta :entity_id :name :definition :creator_id])
+ entity))
+ @serialized)]
+ (ts/with-db dest-db
+ (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")
+ (serdes.load/load-metabase! (ingestion-in-memory minimal))
+ (let [segment (t2/select-one :model/Segment :name "Minors")
+ table (t2/select-one :model/Table :name "customers")]
+ (is (some? segment))
+ (is (= (:id table) (:table_id segment))))))))))
+
+(deftest measure-minimal-required-properties-test
+ (testing "Measure deserialized with only: entity_id, name, definition, creator_id"
+ (let [serialized (atom nil)]
+ (ts/with-dbs [source-db dest-db]
+ (ts/with-db source-db
+ (let [db (ts/create! :model/Database :name "my-db")
+ table (ts/create! :model/Table :name "sales" :db_id (:id db))
+ field (ts/create! :model/Field :name "amount" :table_id (:id table))
+ user (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")
+ _measure (ts/create! :model/Measure :table_id (:id table) :name "Total Sales"
+ :definition (pmbql-measure-definition (:id db) (:id table) (:id field))
+ :creator_id (:id user))]
+ (reset! serialized (into [] (serdes.extract/extract {})))))
+
+ (let [minimal (mapv (fn [entity]
+ (if (= "Measure" (-> entity :serdes/meta last :model))
+ (select-keys entity [:serdes/meta :entity_id :name :definition :creator_id])
+ entity))
+ @serialized)]
+ (ts/with-db dest-db
+ (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")
+ (serdes.load/load-metabase! (ingestion-in-memory minimal))
+ (let [measure (t2/select-one :model/Measure :name "Total Sales")
+ table (t2/select-one :model/Table :name "sales")]
+ (is (some? measure))
+ (is (= (:id table) (:table_id measure))))))))))
+
+(deftest card-minimal-required-properties-test
+ (testing "Card deserialized with only: entity_id, name, display, dataset_query, visualization_settings, creator_id"
+ (let [serialized (atom nil)]
+ (ts/with-dbs [source-db dest-db]
+ (ts/with-db source-db
+ (let [db (ts/create! :model/Database :name "my-db")
+ table (ts/create! :model/Table :name "customers" :db_id (:id db))
+ user (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")
+ _card (ts/create! :model/Card
+ :collection_id nil
+ :creator_id (:id user)
+ :name "Example Card"
+ :dataset_query (pmbql-query (:id db) (:id table))
+ :display :line)]
+ (reset! serialized (into [] (serdes.extract/extract {})))))
+
+ (let [minimal (mapv (fn [entity]
+ (if (= "Card" (-> entity :serdes/meta last :model))
+ (select-keys entity [:serdes/meta :entity_id :name :display :dataset_query
+ :visualization_settings :creator_id])
+ entity))
+ @serialized)]
+ (ts/with-db dest-db
+ (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")
+ (serdes.load/load-metabase! (ingestion-in-memory minimal))
+ (let [card (t2/select-one :model/Card :name "Example Card")
+ db (t2/select-one :model/Database :name "my-db")
+ table (t2/select-one :model/Table :name "customers")]
+ (is (some? card))
+ (is (= (:id db) (:database_id card)))
+ (is (= (:id table) (:table_id card)))
+ (is (nil? (:source_card_id card)))
+ (is (= :query (:query_type card))))))))))
+
+(deftest transform-minimal-required-properties-test
+ (testing "Transform deserialized with only: serdes/meta, entity_id, name, source, target"
+ (mt/with-premium-features #{:transforms-basic}
+ (let [serialized (atom nil)]
+ (ts/with-dbs [source-db dest-db]
+ (ts/with-db source-db
+ (t2/delete! :model/TransformTag)
+ (let [db (ts/create! :model/Database :name "my-db")
+ table (ts/create! :model/Table :name "customers" :db_id (:id db))
+ coll (ts/create! :model/Collection :name "Transform Collection" :namespace :transforms)
+ _transform (ts/create! :model/Transform
+ :name "Test Transform"
+ :description "A test transform"
+ :collection_id (:id coll)
+ :source {:query (pmbql-query (:id db) (:id table))
+ :type "query"}
+ :target {:database (:id db)
+ :type "table"
+ :schema "public"
+ :name "target_table"})]
+ (reset! serialized (into [] (serdes.extract/extract {})))))
+
+ (let [minimal (mapv (fn [entity]
+ (if (= "Transform" (-> entity :serdes/meta last :model))
+ (select-keys entity [:serdes/meta :entity_id :name
+ :source :target])
+ entity))
+ @serialized)]
+ (ts/with-db dest-db
+ (t2/delete! :model/TransformTag)
+ (serdes.load/load-metabase! (ingestion-in-memory minimal))
+ (let [transform (t2/select-one :model/Transform :name "Test Transform")
+ db (t2/select-one :model/Database :name "my-db")]
+ (is (some? transform))
+ (is (= (:id db) (:source_database_id transform)))))))))))
+
+(deftest dashboard-minimal-required-properties-test
+ (testing "Dashboard deserialized with only: serdes/meta, entity_id, name, creator_id"
+ (let [serialized (atom nil)]
+ (ts/with-dbs [source-db dest-db]
+ (ts/with-db source-db
+ (let [_dash (ts/create! :model/Dashboard :name "Test Dashboard")]
+ (reset! serialized (into [] (serdes.extract/extract {})))))
+
+ (let [minimal (mapv (fn [entity]
+ (if (= "Dashboard" (-> entity :serdes/meta last :model))
+ (select-keys entity [:serdes/meta :entity_id :name :creator_id])
+ entity))
+ @serialized)]
+ (ts/with-db dest-db
+ (serdes.load/load-metabase! (ingestion-in-memory minimal))
+ (let [dashboard (t2/select-one :model/Dashboard :name "Test Dashboard")]
+ (is (some? dashboard)))))))))
+
+(deftest dashboard-with-dashcard-minimal-required-properties-test
+ (testing "DashboardCard minimal required properties: serdes/meta, entity_id, row, col, size_x, size_y"
+ (let [serialized (atom nil)]
+ (ts/with-dbs [source-db dest-db]
+ (ts/with-db source-db
+ (let [dash (ts/create! :model/Dashboard :name "Test Dashboard")
+ _dc (ts/create! :model/DashboardCard :dashboard_id (:id dash))]
+ (reset! serialized (into [] (serdes.extract/extract {})))))
+
+ (let [minimal (mapv (fn [entity]
+ (let [model (-> entity :serdes/meta last :model)]
+ (case model
+ "Dashboard" (-> (select-keys entity [:serdes/meta :entity_id :name :creator_id
+ :dashcards])
+ (update :dashcards
+ (fn [dcs]
+ (mapv #(select-keys % [:serdes/meta :entity_id
+ :row :col :size_x :size_y])
+ dcs))))
+ entity)))
+ @serialized)]
+ (ts/with-db dest-db
+ (serdes.load/load-metabase! (ingestion-in-memory minimal))
+ (let [dashboard (t2/select-one :model/Dashboard :name "Test Dashboard")
+ dashcards (t2/select :model/DashboardCard :dashboard_id (:id dashboard))]
+ (is (some? dashboard))
+ (is (= 1 (count dashcards))))))))))
+
+(deftest dashcard-series-minimal-required-properties-test
+ (testing "DashboardCardSeries minimal required properties: card_id, position"
+ (let [serialized (atom nil)]
+ (ts/with-dbs [source-db dest-db]
+ (ts/with-db source-db
+ (let [dash (ts/create! :model/Dashboard :name "Test Dashboard")
+ card (ts/create! :model/Card :name "Main Card")
+ series-card (ts/create! :model/Card :name "Series Card")
+ dc (ts/create! :model/DashboardCard :dashboard_id (:id dash) :card_id (:id card))
+ _series (ts/create! :model/DashboardCardSeries :dashboardcard_id (:id dc)
+ :card_id (:id series-card) :position 0)]
+ (reset! serialized (into [] (serdes.extract/extract {})))))
+
+ (let [minimal (mapv (fn [entity]
+ (let [model (-> entity :serdes/meta last :model)]
+ (case model
+ "Dashboard" (-> (select-keys entity [:serdes/meta :entity_id :name :creator_id
+ :dashcards])
+ (update :dashcards
+ (fn [dcs]
+ (mapv (fn [dc]
+ (-> (select-keys dc [:serdes/meta :entity_id
+ :row :col :size_x :size_y
+ :card_id :series])
+ (update :series
+ (fn [ss]
+ (mapv #(select-keys % [:card_id :position])
+ ss)))))
+ dcs))))
+ entity)))
+ @serialized)]
+ (ts/with-db dest-db
+ (serdes.load/load-metabase! (ingestion-in-memory minimal))
+ (let [dashboard (t2/select-one :model/Dashboard :name "Test Dashboard")
+ dashcards (t2/select :model/DashboardCard :dashboard_id (:id dashboard))
+ series (t2/select :model/DashboardCardSeries :dashboardcard_id (:id (first dashcards)))]
+ (is (some? dashboard))
+ (is (= 1 (count dashcards)))
+ (is (= 1 (count series))))))))))
diff --git a/src/metabase/measures/models/measure.clj b/src/metabase/measures/models/measure.clj
index 618cccef1ff..43a2e7a4b9d 100644
--- a/src/metabase/measures/models/measure.clj
+++ b/src/metabase/measures/models/measure.clj
@@ -2,7 +2,7 @@
"A Measure is a saved MBQL 'macro', expanding to an `:aggregation` clause. It is tied to a table and contains
exactly one aggregation expression."
(:require
- [clojure.set :as set]
+ [medley.core :as m]
[metabase.api.common :as api]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
@@ -130,9 +130,10 @@
(validate-mbql5-definition definition)
(when (seq definition)
(lib/check-measure-overwrite nil definition))
- measure)
+ (cond-> measure
+ (seq definition) (m/assoc-some :table_id (lib/primary-source-table-id definition))))
-(t2/define-before-update :model/Measure [{:keys [id] :as measure}]
+(t2/define-before-update :model/Measure [{:keys [id definition] :as measure}]
;; throw an Exception if someone tries to update creator_id
(when (contains? (t2/changes measure) :creator_id)
(throw (UnsupportedOperationException. (tru "You cannot update the creator_id of a Measure."))))
@@ -140,7 +141,11 @@
(when-let [def-change (:definition (t2/changes measure))]
(validate-mbql5-definition def-change)
(lib/check-measure-overwrite id def-change))
- measure)
+ (if (and (contains? (t2/changes measure) :definition)
+ (seq definition))
+ (m/assoc-some measure
+ :table_id (lib/primary-source-table-id definition))
+ measure))
(defmethod mi/perms-objects-set :model/Measure
[measure read-or-write]
@@ -186,8 +191,8 @@
[:name (serdes/hydrated-hash :table) :created_at])
(defmethod serdes/dependencies "Measure" [{:keys [definition table_id]}]
- (set/union #{(serdes/table->path table_id)}
- (serdes/mbql-deps definition)))
+ (cond-> (serdes/mbql-deps definition)
+ table_id (conj (serdes/table->path table_id))))
(defmethod serdes/storage-path "Measure" [measure _ctx]
(let [{:keys [id label]} (-> measure serdes/path last)]
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index 2fdbe7080d0..9ca8e71a6e3 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -350,13 +350,7 @@
[{query :dataset_query, :as card} :- ::queries.schema/card]
(merge
card
- ;; mega HACK FIXME -- don't update this stuff when doing deserialization because it might differ from what's in the
- ;; YAML file and break tests like [[metabase-enterprise.serialization.v2.e2e.yaml-test/e2e-storage-ingestion-test]].
- ;; The root cause of this issue is that we're generating Cards that have a different Database ID or Table ID from
- ;; what's actually in their query -- we need to fix [[metabase.test.generate]], but I'm not sure how to do that
- (when (and (seq query)
- (map? query)
- (not mi/*deserializing?*))
+ (when (and (seq query) (map? query))
(merge
;; This used to be conditional on not nilling source-card-id, changed due to #68080.
{:source_card_id (source-card-id query)}
@@ -1330,7 +1324,7 @@
(concat
(mapcat serdes/mbql-deps parameter_mappings)
(serdes/parameters-deps parameters)
- [[{:model "Database" :id database_id}]]
+ (when database_id [[{:model "Database" :id database_id}]])
(when table_id #{(serdes/table->path table_id)})
(when source_card_id #{[{:model "Card" :id source_card_id}]})
(when collection_id #{[{:model "Collection" :id collection_id}]})
diff --git a/src/metabase/segments/models/segment.clj b/src/metabase/segments/models/segment.clj
index adaac856795..aeb6c999fe6 100644
--- a/src/metabase/segments/models/segment.clj
+++ b/src/metabase/segments/models/segment.clj
@@ -2,7 +2,7 @@
"A Segment is a saved MBQL 'macro', expanding to a `:filter` subclause. It is passed in as a `:filter` subclause but is
replaced by the `expand-macros` middleware with the appropriate clauses."
(:require
- [clojure.set :as set]
+ [medley.core :as m]
[metabase.api.common :as api]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
@@ -151,16 +151,18 @@
(defn- migrated-segment-definition
[{:keys [definition], table-id :table_id}]
- (let [database-id (t2/select-one-fn :db_id :model/Table :id table-id)]
- (normalize-segment-definition definition table-id database-id)))
+ (when (some? definition)
+ (let [database-id (when table-id
+ (t2/select-one-fn :db_id :model/Table :id table-id))]
+ (normalize-segment-definition definition table-id database-id))))
(t2/define-before-insert :model/Segment
- [{:keys [definition] :as segment}]
- (let [segment (cond-> segment
- (some? definition) (assoc :definition (migrated-segment-definition segment)))]
- (when (seq (:definition segment))
- (lib/check-segment-overwrite nil (:definition segment)))
- segment))
+ [segment]
+ (let [definition (migrated-segment-definition segment)]
+ (when (seq definition)
+ (lib/check-segment-overwrite nil definition))
+ (cond-> (assoc segment :definition definition)
+ (seq definition) (m/assoc-some :table_id (lib/primary-source-table-id definition)))))
(t2/define-before-update :model/Segment [{:keys [id] :as segment}]
;; throw an Exception if someone tries to update creator_id
@@ -168,10 +170,11 @@
(throw (UnsupportedOperationException. (tru "You cannot update the creator_id of a Segment."))))
;; normalize and check for cycles if definition is being updated
(if-let [def-change (:definition (t2/changes segment))]
- (let [normalized-def (migrated-segment-definition (assoc segment :definition def-change))]
- (when (seq normalized-def)
- (lib/check-segment-overwrite id normalized-def))
- (assoc segment :definition normalized-def))
+ (let [definition (migrated-segment-definition (assoc segment :definition def-change))]
+ (when (seq definition)
+ (lib/check-segment-overwrite id definition))
+ (cond-> (assoc segment :definition definition)
+ (seq definition) (m/assoc-some :table_id (lib/primary-source-table-id definition))))
segment))
(defmethod mi/perms-objects-set :model/Segment
@@ -215,8 +218,8 @@
[:name (serdes/hydrated-hash :table) :created_at])
(defmethod serdes/dependencies "Segment" [{:keys [definition table_id]}]
- (set/union #{(serdes/table->path table_id)}
- (serdes/mbql-deps definition)))
+ (cond-> (serdes/mbql-deps definition)
+ table_id (conj (serdes/table->path table_id))))
(defmethod serdes/storage-path "Segment" [segment _ctx]
(let [{:keys [id label]} (-> segment serdes/path last)]
diff --git a/test/metabase/measures/models/measure_test.clj b/test/metabase/measures/models/measure_test.clj
index 821ee5cbbc0..8841dea5049 100644
--- a/test/metabase/measures/models/measure_test.clj
+++ b/test/metabase/measures/models/measure_test.clj
@@ -43,10 +43,9 @@
(deftest identity-hash-test
(testing "Measure hashes are composed of the measure name and table identity-hash"
- (let [now #t "2022-09-01T12:34:56Z"]
- (mt/with-temp [:model/Database db {:name "field-db" :engine :h2}
- :model/Table table {:schema "PUBLIC" :name "widget" :db_id (:id db)}
- :model/Measure measure {:name "total sales" :table_id (:id table) :created_at now
+ (let [now #t "2022-09-01T12:34:56Z"
+ table (t2/select-one :model/Table (mt/id :venues))]
+ (mt/with-temp [:model/Measure measure {:name "total sales" :table_id (:id table) :created_at now
:definition (measure-definition (lib/count))}]
(is (= (serdes/raw-hash ["total sales" (serdes/identity-hash table) (:created_at measure)])
(serdes/identity-hash measure)))))))