The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Serdes for cards, segments, measures, transforms - allow deserializing only with minimal required properties without computed ones (#71464)
Alexander Polyankinb7e1d1eff8aa
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.cljindex e67c71b803e..24abeeef4ea 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj@@ -357,6 +357,22 @@:database (:id @db1d)}(:definition @seg1d))))))))))+(defn- pmbql-query+ "Create a simple pMBQL query for the given database and table."+ [db-id table-id]+ (let [metadata-provider (lib-be/application-database-metadata-provider db-id)+ table (lib.metadata/table metadata-provider table-id)]+ (lib/query metadata-provider table)))++(defn- pmbql-segment-definition+ "Create a simple pMBQL segment definition with a filter."+ [db-id table-id field-id]+ (let [metadata-provider (lib-be/application-database-metadata-provider db-id)+ table (lib.metadata/table metadata-provider table-id)+ query (lib/query metadata-provider table)+ field (lib.metadata/field metadata-provider field-id)]+ (lib/filter query (lib/< field 18))))+(defn- pmbql-measure-definition"Create an MBQL5 measure definition with a sum aggregation."[db-id table-id field-id]@@ -1855,3 +1871,213 @@Exception#"source-only-db|not found|Failed"(serdes.load/load-metabase! (ingestion-in-memory @serialized))))))))))++(deftest segment-minimal-required-properties-test+ (testing "Segment deserialized with only: entity_id, name, definition, creator_id"+ (let [serialized (atom nil)]+ (ts/with-dbs [source-db dest-db]+ (ts/with-db source-db+ (let [db (ts/create! :model/Database :name "my-db")+ table (ts/create! :model/Table :name "customers" :db_id (:id db))+ field (ts/create! :model/Field :name "age" :table_id (:id table))+ user (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")+ _segment (ts/create! :model/Segment :table_id (:id table) :name "Minors"+ :definition (pmbql-segment-definition (:id db) (:id table) (:id field))+ :creator_id (:id user))]+ (reset! serialized (into [] (serdes.extract/extract {})))))++ (let [minimal (mapv (fn [entity]+ (if (= "Segment" (-> entity :serdes/meta last :model))+ (select-keys entity [:serdes/meta :entity_id :name :definition :creator_id])+ entity))+ @serialized)]+ (ts/with-db dest-db+ (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")+ (serdes.load/load-metabase! (ingestion-in-memory minimal))+ (let [segment (t2/select-one :model/Segment :name "Minors")+ table (t2/select-one :model/Table :name "customers")]+ (is (some? segment))+ (is (= (:id table) (:table_id segment))))))))))++(deftest measure-minimal-required-properties-test+ (testing "Measure deserialized with only: entity_id, name, definition, creator_id"+ (let [serialized (atom nil)]+ (ts/with-dbs [source-db dest-db]+ (ts/with-db source-db+ (let [db (ts/create! :model/Database :name "my-db")+ table (ts/create! :model/Table :name "sales" :db_id (:id db))+ field (ts/create! :model/Field :name "amount" :table_id (:id table))+ user (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")+ _measure (ts/create! :model/Measure :table_id (:id table) :name "Total Sales"+ :definition (pmbql-measure-definition (:id db) (:id table) (:id field))+ :creator_id (:id user))]+ (reset! serialized (into [] (serdes.extract/extract {})))))++ (let [minimal (mapv (fn [entity]+ (if (= "Measure" (-> entity :serdes/meta last :model))+ (select-keys entity [:serdes/meta :entity_id :name :definition :creator_id])+ entity))+ @serialized)]+ (ts/with-db dest-db+ (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")+ (serdes.load/load-metabase! (ingestion-in-memory minimal))+ (let [measure (t2/select-one :model/Measure :name "Total Sales")+ table (t2/select-one :model/Table :name "sales")]+ (is (some? measure))+ (is (= (:id table) (:table_id measure))))))))))++(deftest card-minimal-required-properties-test+ (testing "Card deserialized with only: entity_id, name, display, dataset_query, visualization_settings, creator_id"+ (let [serialized (atom nil)]+ (ts/with-dbs [source-db dest-db]+ (ts/with-db source-db+ (let [db (ts/create! :model/Database :name "my-db")+ table (ts/create! :model/Table :name "customers" :db_id (:id db))+ user (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")+ _card (ts/create! :model/Card+ :collection_id nil+ :creator_id (:id user)+ :name "Example Card"+ :dataset_query (pmbql-query (:id db) (:id table))+ :display :line)]+ (reset! serialized (into [] (serdes.extract/extract {})))))++ (let [minimal (mapv (fn [entity]+ (if (= "Card" (-> entity :serdes/meta last :model))+ (select-keys entity [:serdes/meta :entity_id :name :display :dataset_query+ :visualization_settings :creator_id])+ entity))+ @serialized)]+ (ts/with-db dest-db+ (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")+ (serdes.load/load-metabase! (ingestion-in-memory minimal))+ (let [card (t2/select-one :model/Card :name "Example Card")+ db (t2/select-one :model/Database :name "my-db")+ table (t2/select-one :model/Table :name "customers")]+ (is (some? card))+ (is (= (:id db) (:database_id card)))+ (is (= (:id table) (:table_id card)))+ (is (nil? (:source_card_id card)))+ (is (= :query (:query_type card))))))))))++(deftest transform-minimal-required-properties-test+ (testing "Transform deserialized with only: serdes/meta, entity_id, name, source, target"+ (mt/with-premium-features #{:transforms-basic}+ (let [serialized (atom nil)]+ (ts/with-dbs [source-db dest-db]+ (ts/with-db source-db+ (t2/delete! :model/TransformTag)+ (let [db (ts/create! :model/Database :name "my-db")+ table (ts/create! :model/Table :name "customers" :db_id (:id db))+ coll (ts/create! :model/Collection :name "Transform Collection" :namespace :transforms)+ _transform (ts/create! :model/Transform+ :name "Test Transform"+ :description "A test transform"+ :collection_id (:id coll)+ :source {:query (pmbql-query (:id db) (:id table))+ :type "query"}+ :target {:database (:id db)+ :type "table"+ :schema "public"+ :name "target_table"})]+ (reset! serialized (into [] (serdes.extract/extract {})))))++ (let [minimal (mapv (fn [entity]+ (if (= "Transform" (-> entity :serdes/meta last :model))+ (select-keys entity [:serdes/meta :entity_id :name+ :source :target])+ entity))+ @serialized)]+ (ts/with-db dest-db+ (t2/delete! :model/TransformTag)+ (serdes.load/load-metabase! (ingestion-in-memory minimal))+ (let [transform (t2/select-one :model/Transform :name "Test Transform")+ db (t2/select-one :model/Database :name "my-db")]+ (is (some? transform))+ (is (= (:id db) (:source_database_id transform)))))))))))++(deftest dashboard-minimal-required-properties-test+ (testing "Dashboard deserialized with only: serdes/meta, entity_id, name, creator_id"+ (let [serialized (atom nil)]+ (ts/with-dbs [source-db dest-db]+ (ts/with-db source-db+ (let [_dash (ts/create! :model/Dashboard :name "Test Dashboard")]+ (reset! serialized (into [] (serdes.extract/extract {})))))++ (let [minimal (mapv (fn [entity]+ (if (= "Dashboard" (-> entity :serdes/meta last :model))+ (select-keys entity [:serdes/meta :entity_id :name :creator_id])+ entity))+ @serialized)]+ (ts/with-db dest-db+ (serdes.load/load-metabase! (ingestion-in-memory minimal))+ (let [dashboard (t2/select-one :model/Dashboard :name "Test Dashboard")]+ (is (some? dashboard)))))))))++(deftest dashboard-with-dashcard-minimal-required-properties-test+ (testing "DashboardCard minimal required properties: serdes/meta, entity_id, row, col, size_x, size_y"+ (let [serialized (atom nil)]+ (ts/with-dbs [source-db dest-db]+ (ts/with-db source-db+ (let [dash (ts/create! :model/Dashboard :name "Test Dashboard")+ _dc (ts/create! :model/DashboardCard :dashboard_id (:id dash))]+ (reset! serialized (into [] (serdes.extract/extract {})))))++ (let [minimal (mapv (fn [entity]+ (let [model (-> entity :serdes/meta last :model)]+ (case model+ "Dashboard" (-> (select-keys entity [:serdes/meta :entity_id :name :creator_id+ :dashcards])+ (update :dashcards+ (fn [dcs]+ (mapv #(select-keys % [:serdes/meta :entity_id+ :row :col :size_x :size_y])+ dcs))))+ entity)))+ @serialized)]+ (ts/with-db dest-db+ (serdes.load/load-metabase! (ingestion-in-memory minimal))+ (let [dashboard (t2/select-one :model/Dashboard :name "Test Dashboard")+ dashcards (t2/select :model/DashboardCard :dashboard_id (:id dashboard))]+ (is (some? dashboard))+ (is (= 1 (count dashcards))))))))))++(deftest dashcard-series-minimal-required-properties-test+ (testing "DashboardCardSeries minimal required properties: card_id, position"+ (let [serialized (atom nil)]+ (ts/with-dbs [source-db dest-db]+ (ts/with-db source-db+ (let [dash (ts/create! :model/Dashboard :name "Test Dashboard")+ card (ts/create! :model/Card :name "Main Card")+ series-card (ts/create! :model/Card :name "Series Card")+ dc (ts/create! :model/DashboardCard :dashboard_id (:id dash) :card_id (:id card))+ _series (ts/create! :model/DashboardCardSeries :dashboardcard_id (:id dc)+ :card_id (:id series-card) :position 0)]+ (reset! serialized (into [] (serdes.extract/extract {})))))++ (let [minimal (mapv (fn [entity]+ (let [model (-> entity :serdes/meta last :model)]+ (case model+ "Dashboard" (-> (select-keys entity [:serdes/meta :entity_id :name :creator_id+ :dashcards])+ (update :dashcards+ (fn [dcs]+ (mapv (fn [dc]+ (-> (select-keys dc [:serdes/meta :entity_id+ :row :col :size_x :size_y+ :card_id :series])+ (update :series+ (fn [ss]+ (mapv #(select-keys % [:card_id :position])+ ss)))))+ dcs))))+ entity)))+ @serialized)]+ (ts/with-db dest-db+ (serdes.load/load-metabase! (ingestion-in-memory minimal))+ (let [dashboard (t2/select-one :model/Dashboard :name "Test Dashboard")+ dashcards (t2/select :model/DashboardCard :dashboard_id (:id dashboard))+ series (t2/select :model/DashboardCardSeries :dashboardcard_id (:id (first dashcards)))]+ (is (some? dashboard))+ (is (= 1 (count dashcards)))+ (is (= 1 (count series))))))))))diff --git a/src/metabase/measures/models/measure.clj b/src/metabase/measures/models/measure.cljindex 618cccef1ff..43a2e7a4b9d 100644--- a/src/metabase/measures/models/measure.clj+++ b/src/metabase/measures/models/measure.clj@@ -2,7 +2,7 @@"A Measure is a saved MBQL 'macro', expanding to an `:aggregation` clause. It is tied to a table and containsexactly one aggregation expression."(:require- [clojure.set :as set]+ [medley.core :as m][metabase.api.common :as api][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib]@@ -130,9 +130,10 @@(validate-mbql5-definition definition)(when (seq definition)(lib/check-measure-overwrite nil definition))- measure)+ (cond-> measure+ (seq definition) (m/assoc-some :table_id (lib/primary-source-table-id definition))))-(t2/define-before-update :model/Measure [{:keys [id] :as measure}]+(t2/define-before-update :model/Measure [{:keys [id definition] :as measure}];; throw an Exception if someone tries to update creator_id(when (contains? (t2/changes measure) :creator_id)(throw (UnsupportedOperationException. (tru "You cannot update the creator_id of a Measure."))))@@ -140,7 +141,11 @@(when-let [def-change (:definition (t2/changes measure))](validate-mbql5-definition def-change)(lib/check-measure-overwrite id def-change))- measure)+ (if (and (contains? (t2/changes measure) :definition)+ (seq definition))+ (m/assoc-some measure+ :table_id (lib/primary-source-table-id definition))+ measure))(defmethod mi/perms-objects-set :model/Measure[measure read-or-write]@@ -186,8 +191,8 @@[:name (serdes/hydrated-hash :table) :created_at])(defmethod serdes/dependencies "Measure" [{:keys [definition table_id]}]- (set/union #{(serdes/table->path table_id)}- (serdes/mbql-deps definition)))+ (cond-> (serdes/mbql-deps definition)+ table_id (conj (serdes/table->path table_id))))(defmethod serdes/storage-path "Measure" [measure _ctx](let [{:keys [id label]} (-> measure serdes/path last)]diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex 2fdbe7080d0..9ca8e71a6e3 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -350,13 +350,7 @@[{query :dataset_query, :as card} :- ::queries.schema/card](mergecard- ;; mega HACK FIXME -- don't update this stuff when doing deserialization because it might differ from what's in the- ;; YAML file and break tests like [[metabase-enterprise.serialization.v2.e2e.yaml-test/e2e-storage-ingestion-test]].- ;; The root cause of this issue is that we're generating Cards that have a different Database ID or Table ID from- ;; what's actually in their query -- we need to fix [[metabase.test.generate]], but I'm not sure how to do that- (when (and (seq query)- (map? query)- (not mi/*deserializing?*))+ (when (and (seq query) (map? query))(merge;; This used to be conditional on not nilling source-card-id, changed due to #68080.{:source_card_id (source-card-id query)}@@ -1330,7 +1324,7 @@(concat(mapcat serdes/mbql-deps parameter_mappings)(serdes/parameters-deps parameters)- [[{:model "Database" :id database_id}]]+ (when database_id [[{:model "Database" :id database_id}]])(when table_id #{(serdes/table->path table_id)})(when source_card_id #{[{:model "Card" :id source_card_id}]})(when collection_id #{[{:model "Collection" :id collection_id}]})diff --git a/src/metabase/segments/models/segment.clj b/src/metabase/segments/models/segment.cljindex adaac856795..aeb6c999fe6 100644--- a/src/metabase/segments/models/segment.clj+++ b/src/metabase/segments/models/segment.clj@@ -2,7 +2,7 @@"A Segment is a saved MBQL 'macro', expanding to a `:filter` subclause. It is passed in as a `:filter` subclause but isreplaced by the `expand-macros` middleware with the appropriate clauses."(:require- [clojure.set :as set]+ [medley.core :as m][metabase.api.common :as api][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib]@@ -151,16 +151,18 @@(defn- migrated-segment-definition[{:keys [definition], table-id :table_id}]- (let [database-id (t2/select-one-fn :db_id :model/Table :id table-id)]- (normalize-segment-definition definition table-id database-id)))+ (when (some? definition)+ (let [database-id (when table-id+ (t2/select-one-fn :db_id :model/Table :id table-id))]+ (normalize-segment-definition definition table-id database-id))))(t2/define-before-insert :model/Segment- [{:keys [definition] :as segment}]- (let [segment (cond-> segment- (some? definition) (assoc :definition (migrated-segment-definition segment)))]- (when (seq (:definition segment))- (lib/check-segment-overwrite nil (:definition segment)))- segment))+ [segment]+ (let [definition (migrated-segment-definition segment)]+ (when (seq definition)+ (lib/check-segment-overwrite nil definition))+ (cond-> (assoc segment :definition definition)+ (seq definition) (m/assoc-some :table_id (lib/primary-source-table-id definition)))))(t2/define-before-update :model/Segment [{:keys [id] :as segment}];; throw an Exception if someone tries to update creator_id@@ -168,10 +170,11 @@(throw (UnsupportedOperationException. (tru "You cannot update the creator_id of a Segment."))));; normalize and check for cycles if definition is being updated(if-let [def-change (:definition (t2/changes segment))]- (let [normalized-def (migrated-segment-definition (assoc segment :definition def-change))]- (when (seq normalized-def)- (lib/check-segment-overwrite id normalized-def))- (assoc segment :definition normalized-def))+ (let [definition (migrated-segment-definition (assoc segment :definition def-change))]+ (when (seq definition)+ (lib/check-segment-overwrite id definition))+ (cond-> (assoc segment :definition definition)+ (seq definition) (m/assoc-some :table_id (lib/primary-source-table-id definition))))segment))(defmethod mi/perms-objects-set :model/Segment@@ -215,8 +218,8 @@[:name (serdes/hydrated-hash :table) :created_at])(defmethod serdes/dependencies "Segment" [{:keys [definition table_id]}]- (set/union #{(serdes/table->path table_id)}- (serdes/mbql-deps definition)))+ (cond-> (serdes/mbql-deps definition)+ table_id (conj (serdes/table->path table_id))))(defmethod serdes/storage-path "Segment" [segment _ctx](let [{:keys [id label]} (-> segment serdes/path last)]diff --git a/test/metabase/measures/models/measure_test.clj b/test/metabase/measures/models/measure_test.cljindex 821ee5cbbc0..8841dea5049 100644--- a/test/metabase/measures/models/measure_test.clj+++ b/test/metabase/measures/models/measure_test.clj@@ -43,10 +43,9 @@(deftest identity-hash-test(testing "Measure hashes are composed of the measure name and table identity-hash"- (let [now #t "2022-09-01T12:34:56Z"]- (mt/with-temp [:model/Database db {:name "field-db" :engine :h2}- :model/Table table {:schema "PUBLIC" :name "widget" :db_id (:id db)}- :model/Measure measure {:name "total sales" :table_id (:id table) :created_at now+ (let [now #t "2022-09-01T12:34:56Z"+ table (t2/select-one :model/Table (mt/id :venues))]+ (mt/with-temp [:model/Measure measure {:name "total sales" :table_id (:id table) :created_at now:definition (measure-definition (lib/count))}](is (= (serdes/raw-hash ["total sales" (serdes/identity-hash table) (:created_at measure)])(serdes/identity-hash measure)))))))