Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Serdes fixes for python transforms (#71735)

Alexander Polyankin9ada59a8043f
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
index 0d0032ee57f..dca780cb2a1 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
@@ -2297,6 +2297,25 @@
:schema "public"
:name "target_table"}}
+ ;; Python transform with source-tables
+ :model/Transform
+ {python-transform-id :id
+ python-transform-eid :entity_id}
+ {:name "Python ETL"
+ :description "A python transform"
+ :entity_id "pythonEtlPythonEtlxxx"
+ :collection_id coll-id
+ :source_database_id db-id
+ :source {:type "python"
+ :source-database db-id
+ :source-tables [{:alias "orders"
+ :table_id table-id}]
+ :body "df = ctx.source.orders"}
+ :target {:database db-id
+ :type "table"
+ :schema "public"
+ :name "target_table"}}
+
;; Create tag associations with specific positions
:model/TransformTransformTag
{}
@@ -2349,8 +2368,20 @@
(is (contains? deps [{:model "TransformTag" :id custom-tag-eid}]))
(is (contains? deps [{:model "TransformTag" :id daily-tag-eid}])))))
- (testing "transform is extracted"
- (is (= #{transform-eid}
+ (testing "python transform source-tables export"
+ (let [ser (serdes/extract-one "Transform" {} (t2/hydrate (t2/select-one :model/Transform :id python-transform-id) :tags))]
+ (is (=? {:source {:type :python
+ :source-database "My Database"
+ :source-tables [{:alias "orders"
+ :table_id ["My Database" nil "Schemaless Table"]
+ :database_id "My Database"
+ :schema nil
+ :table "Schemaless Table"}]
+ :body "df = ctx.source.orders"}}
+ ser))))
+
+ (testing "transforms are extracted"
+ (is (= #{transform-eid python-transform-eid}
(ids-by-model "Transform" (extract/extract {}))))))))))
(deftest transform-job-extraction-test
diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.clj
index e8e57db6dfc..a8c138453d7 100644
--- a/src/metabase/models/serialization.clj
+++ b/src/metabase/models/serialization.clj
@@ -1024,6 +1024,20 @@
;; Need to break a circular dependency here.
(:id ((resolve 'metabase.users.models.user/serdes-synthesize-user!) {:email email :is_active false})))))
+;;; ## Databases
+
+(defn ^:dynamic ^::cache *export-database-fk*
+ "Given a numeric database ID, return its name as a portable reference.
+ [[*import-database-fk*]] is the inverse."
+ [id]
+ (*export-fk-keyed* id :model/Database :name))
+
+(defn ^:dynamic ^::cache *import-database-fk*
+ "Given a portable database name, resolve it back to a numeric ID.
+ [[*export-database-fk*]] is the inverse."
+ [db-name]
+ (*import-fk-keyed* db-name :model/Database :name))
+
;;; ## Tables
(mu/defn ^:dynamic ^::cache *export-table-fk*
@@ -1453,7 +1467,7 @@
(merge entity
{:id (case model
"table" (*export-table-fk* id)
- "database" (*export-fk-keyed* id :model/Database :name)
+ "database" (*export-database-fk* id)
(*export-fk* id (link-card-model->toucan-model model)))}))))
(defn- json-export-mbql
@@ -1626,7 +1640,7 @@
(merge entity
{:id (case model
"table" (*import-table-fk* id)
- "database" (*import-fk-keyed* id :model/Database :name)
+ "database" (*import-database-fk* id)
(*import-fk* id (link-card-model->toucan-model model)))}))))
(defn- import-visualizations [entity]
@@ -1773,9 +1787,10 @@
(defn fk "Export Foreign Key" [model & [field-name]]
(cond
;; this `::fk` is used in tests to determine that foreign keys are handled
- (= model :model/User) {::fk true :export *export-user* :import *import-user*}
- (= model :model/Table) {::fk true :export *export-table-fk* :import *import-table-fk*}
- (= model :model/Field) {::fk true :export *export-field-fk* :import *import-field-fk*}
+ (= model :model/User) {::fk true :export *export-user* :import *import-user*}
+ (= model :model/Database) {::fk true :export *export-database-fk* :import *import-database-fk*}
+ (= model :model/Table) {::fk true :export *export-table-fk* :import *import-table-fk*}
+ (= model :model/Field) {::fk true :export *export-field-fk* :import *import-field-fk*}
field-name {::fk true
:export #(*export-fk-keyed* % model field-name)
:import #(*import-fk-keyed* % model field-name)}
diff --git a/src/metabase/models/transforms/transform.clj b/src/metabase/models/transforms/transform.clj
index fbc98b696b3..d98a25a13f2 100644
--- a/src/metabase/models/transforms/transform.clj
+++ b/src/metabase/models/transforms/transform.clj
@@ -344,6 +344,18 @@
[_transform]
[:name :created_at])
+(defn- import-maybe-int-database-fk
+ "Import a database reference back to an ID. Tolerates raw numeric IDs from older exports
+ where source-tables database_id values were serialized without conversion."
+ [v]
+ (if (pos-int? v) v (serdes/*import-database-fk* v)))
+
+(defn- import-maybe-int-table-fk
+ "Import a table reference back to an ID. Tolerates raw numeric IDs from older exports
+ where source-tables table_id values were serialized without conversion."
+ [v]
+ (if (pos-int? v) v (serdes/*import-table-fk* v)))
+
(defmethod serdes/make-spec "Transform"
[_model-name opts]
{:copy [:name :description :entity_id :owner_email]
@@ -353,15 +365,28 @@
:owner_user_id (serdes/fk :model/User)
:collection_id (serdes/fk :model/Collection)
:source_database_id (serdes/fk :model/Database :name)
- :source {:export #(update % :query serdes/export-mbql)
+ :source {:export (fn [source]
+ (-> source
+ (m/update-existing :query serdes/export-mbql)
+ (m/update-existing :source-database serdes/*export-database-fk*)
+ (m/update-existing :source-tables
+ (fn [entries]
+ (->> (transforms-base.u/normalize-source-tables entries)
+ (mapv (fn [entry]
+ (-> entry
+ (m/update-existing :table_id serdes/*export-table-fk*)
+ (m/update-existing :database_id serdes/*export-database-fk*)))))))))
:import (fn [source]
(-> source
+ (m/update-existing :query serdes/import-mbql)
+ (m/update-existing :source-database import-maybe-int-database-fk)
(m/update-existing :source-tables
- (fn [st]
- (if (map? st)
- (transforms-base.u/source-tables-map->vec st)
- st)))
- (m/update-existing :query serdes/import-mbql)))}
+ (fn [entries]
+ (->> (cond-> entries (map? entries) transforms-base.u/source-tables-map->vec)
+ (mapv (fn [entry]
+ (-> entry
+ (m/update-existing :table_id import-maybe-int-table-fk)
+ (m/update-existing :database_id import-maybe-int-database-fk)))))))))}
:target {:export #(serdes/export-mbql (dissoc % :table_id))
:import serdes/import-mbql}
:tags (serdes/nested :model/TransformTransformTag :transform_id opts)}})
@@ -376,7 +401,10 @@
[[{:model "Database" :id source_database_id}]])
(for [{tag-id :tag_id} tags]
[{:model "TransformTag" :id tag-id}])
- (serdes/mbql-deps source))))
+ (serdes/mbql-deps source)
+ (for [{:keys [table_id]} (:source-tables source)
+ :when (vector? table_id)]
+ (serdes/table->path table_id)))))
(defmethod serdes/storage-path "Transform" [transform ctx]
(serdes/storage-default-collection-path transform ctx "transforms"))
diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.clj
index 24526d4ace8..1f869b97458 100644
--- a/src/metabase/warehouses/models/database.clj
+++ b/src/metabase/warehouses/models/database.clj
@@ -666,7 +666,7 @@
::serdes/skip))
:import identity}
:creator_id (serdes/fk :model/User)
- :router_database_id (serdes/fk :model/Database)
+ :router_database_id (serdes/fk :model/Database :name)
:initial_sync_status {:export identity :import (constantly "complete")}}
:defaults {:auto_run_queries true
:is_attached_dwh false
diff --git a/test_resources/serialization_baseline/collections/transforms/python_etl.yaml b/test_resources/serialization_baseline/collections/transforms/python_etl.yaml
new file mode 100644
index 00000000000..4af199c53f6
--- /dev/null
+++ b/test_resources/serialization_baseline/collections/transforms/python_etl.yaml
@@ -0,0 +1,29 @@
+created_at: '2025-10-06T20:41:45.536727Z'
+creator_id: [email protected]
+description: A python transform
+entity_id: pythonEtlPythonEtlxxx
+name: Python ETL
+source:
+ body: df = ctx.source.orders
+ source-database: My Database
+ source-tables:
+ - alias: orders
+ database_id: My Database
+ schema: null
+ table: Schemaless Table
+ table_id:
+ - My Database
+ - null
+ - Schemaless Table
+ type: python
+source_database_id: My Database
+tags: []
+target:
+ database: My Database
+ name: target_table
+ schema: public
+ type: table
+serdes/meta:
+- id: pythonEtlPythonEtlxxx
+ label: python_etl
+ model: Transform