The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Serdes fixes for python transforms (#71735)
Alexander Polyankin9ada59a8043f
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.cljindex 0d0032ee57f..dca780cb2a1 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj@@ -2297,6 +2297,25 @@:schema "public":name "target_table"}}+ ;; Python transform with source-tables+ :model/Transform+ {python-transform-id :id+ python-transform-eid :entity_id}+ {:name "Python ETL"+ :description "A python transform"+ :entity_id "pythonEtlPythonEtlxxx"+ :collection_id coll-id+ :source_database_id db-id+ :source {:type "python"+ :source-database db-id+ :source-tables [{:alias "orders"+ :table_id table-id}]+ :body "df = ctx.source.orders"}+ :target {:database db-id+ :type "table"+ :schema "public"+ :name "target_table"}}+;; Create tag associations with specific positions:model/TransformTransformTag{}@@ -2349,8 +2368,20 @@(is (contains? deps [{:model "TransformTag" :id custom-tag-eid}]))(is (contains? deps [{:model "TransformTag" :id daily-tag-eid}])))))- (testing "transform is extracted"- (is (= #{transform-eid}+ (testing "python transform source-tables export"+ (let [ser (serdes/extract-one "Transform" {} (t2/hydrate (t2/select-one :model/Transform :id python-transform-id) :tags))]+ (is (=? {:source {:type :python+ :source-database "My Database"+ :source-tables [{:alias "orders"+ :table_id ["My Database" nil "Schemaless Table"]+ :database_id "My Database"+ :schema nil+ :table "Schemaless Table"}]+ :body "df = ctx.source.orders"}}+ ser))))++ (testing "transforms are extracted"+ (is (= #{transform-eid python-transform-eid}(ids-by-model "Transform" (extract/extract {}))))))))))(deftest transform-job-extraction-testdiff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.cljindex e8e57db6dfc..a8c138453d7 100644--- a/src/metabase/models/serialization.clj+++ b/src/metabase/models/serialization.clj@@ -1024,6 +1024,20 @@;; Need to break a circular dependency here.(:id ((resolve 'metabase.users.models.user/serdes-synthesize-user!) {:email email :is_active false})))))+;;; ## Databases++(defn ^:dynamic ^::cache *export-database-fk*+ "Given a numeric database ID, return its name as a portable reference.+ [[*import-database-fk*]] is the inverse."+ [id]+ (*export-fk-keyed* id :model/Database :name))++(defn ^:dynamic ^::cache *import-database-fk*+ "Given a portable database name, resolve it back to a numeric ID.+ [[*export-database-fk*]] is the inverse."+ [db-name]+ (*import-fk-keyed* db-name :model/Database :name))+;;; ## Tables(mu/defn ^:dynamic ^::cache *export-table-fk*@@ -1453,7 +1467,7 @@(merge entity{:id (case model"table" (*export-table-fk* id)- "database" (*export-fk-keyed* id :model/Database :name)+ "database" (*export-database-fk* id)(*export-fk* id (link-card-model->toucan-model model)))}))))(defn- json-export-mbql@@ -1626,7 +1640,7 @@(merge entity{:id (case model"table" (*import-table-fk* id)- "database" (*import-fk-keyed* id :model/Database :name)+ "database" (*import-database-fk* id)(*import-fk* id (link-card-model->toucan-model model)))}))))(defn- import-visualizations [entity]@@ -1773,9 +1787,10 @@(defn fk "Export Foreign Key" [model & [field-name]](cond;; this `::fk` is used in tests to determine that foreign keys are handled- (= model :model/User) {::fk true :export *export-user* :import *import-user*}- (= model :model/Table) {::fk true :export *export-table-fk* :import *import-table-fk*}- (= model :model/Field) {::fk true :export *export-field-fk* :import *import-field-fk*}+ (= model :model/User) {::fk true :export *export-user* :import *import-user*}+ (= model :model/Database) {::fk true :export *export-database-fk* :import *import-database-fk*}+ (= model :model/Table) {::fk true :export *export-table-fk* :import *import-table-fk*}+ (= model :model/Field) {::fk true :export *export-field-fk* :import *import-field-fk*}field-name {::fk true:export #(*export-fk-keyed* % model field-name):import #(*import-fk-keyed* % model field-name)}diff --git a/src/metabase/models/transforms/transform.clj b/src/metabase/models/transforms/transform.cljindex fbc98b696b3..d98a25a13f2 100644--- a/src/metabase/models/transforms/transform.clj+++ b/src/metabase/models/transforms/transform.clj@@ -344,6 +344,18 @@[_transform][:name :created_at])+(defn- import-maybe-int-database-fk+ "Import a database reference back to an ID. Tolerates raw numeric IDs from older exports+ where source-tables database_id values were serialized without conversion."+ [v]+ (if (pos-int? v) v (serdes/*import-database-fk* v)))++(defn- import-maybe-int-table-fk+ "Import a table reference back to an ID. Tolerates raw numeric IDs from older exports+ where source-tables table_id values were serialized without conversion."+ [v]+ (if (pos-int? v) v (serdes/*import-table-fk* v)))+(defmethod serdes/make-spec "Transform"[_model-name opts]{:copy [:name :description :entity_id :owner_email]@@ -353,15 +365,28 @@:owner_user_id (serdes/fk :model/User):collection_id (serdes/fk :model/Collection):source_database_id (serdes/fk :model/Database :name)- :source {:export #(update % :query serdes/export-mbql)+ :source {:export (fn [source]+ (-> source+ (m/update-existing :query serdes/export-mbql)+ (m/update-existing :source-database serdes/*export-database-fk*)+ (m/update-existing :source-tables+ (fn [entries]+ (->> (transforms-base.u/normalize-source-tables entries)+ (mapv (fn [entry]+ (-> entry+ (m/update-existing :table_id serdes/*export-table-fk*)+ (m/update-existing :database_id serdes/*export-database-fk*))))))))):import (fn [source](-> source+ (m/update-existing :query serdes/import-mbql)+ (m/update-existing :source-database import-maybe-int-database-fk)(m/update-existing :source-tables- (fn [st]- (if (map? st)- (transforms-base.u/source-tables-map->vec st)- st)))- (m/update-existing :query serdes/import-mbql)))}+ (fn [entries]+ (->> (cond-> entries (map? entries) transforms-base.u/source-tables-map->vec)+ (mapv (fn [entry]+ (-> entry+ (m/update-existing :table_id import-maybe-int-table-fk)+ (m/update-existing :database_id import-maybe-int-database-fk)))))))))}:target {:export #(serdes/export-mbql (dissoc % :table_id)):import serdes/import-mbql}:tags (serdes/nested :model/TransformTransformTag :transform_id opts)}})@@ -376,7 +401,10 @@[[{:model "Database" :id source_database_id}]])(for [{tag-id :tag_id} tags][{:model "TransformTag" :id tag-id}])- (serdes/mbql-deps source))))+ (serdes/mbql-deps source)+ (for [{:keys [table_id]} (:source-tables source)+ :when (vector? table_id)]+ (serdes/table->path table_id)))))(defmethod serdes/storage-path "Transform" [transform ctx](serdes/storage-default-collection-path transform ctx "transforms"))diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.cljindex 24526d4ace8..1f869b97458 100644--- a/src/metabase/warehouses/models/database.clj+++ b/src/metabase/warehouses/models/database.clj@@ -666,7 +666,7 @@::serdes/skip)):import identity}:creator_id (serdes/fk :model/User)- :router_database_id (serdes/fk :model/Database)+ :router_database_id (serdes/fk :model/Database :name):initial_sync_status {:export identity :import (constantly "complete")}}:defaults {:auto_run_queries true:is_attached_dwh falsediff --git a/test_resources/serialization_baseline/collections/transforms/python_etl.yaml b/test_resources/serialization_baseline/collections/transforms/python_etl.yamlnew file mode 100644index 00000000000..4af199c53f6--- /dev/null+++ b/test_resources/serialization_baseline/collections/transforms/python_etl.yaml@@ -0,0 +1,29 @@+created_at: '2025-10-06T20:41:45.536727Z'+creator_id: [email protected]+description: A python transform+entity_id: pythonEtlPythonEtlxxx+name: Python ETL+source:+ body: df = ctx.source.orders+ source-database: My Database+ source-tables:+ - alias: orders+ database_id: My Database+ schema: null+ table: Schemaless Table+ table_id:+ - My Database+ - null+ - Schemaless Table+ type: python+source_database_id: My Database+tags: []+target:+ database: My Database+ name: target_table+ schema: public+ type: table+serdes/meta:+- id: pythonEtlPythonEtlxxx+ label: python_etl+ model: Transform