Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add transforms.target_database_id column (#66953)

Oleksandr Solovyov4c9fe80dc5ce
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj
index 9643ffcc860..6d79c3bb8c0 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj
@@ -54,20 +54,22 @@
#{:transforms})
(t2/define-before-insert :model/Transform
- [{:keys [source collection_id] :as transform}]
+ [{:keys [source target collection_id] :as transform}]
(collection/check-collection-namespace :model/Transform collection_id)
(when collection_id
(collection/check-allowed-content :model/Transform collection_id))
- (assoc transform :source_type (transforms.util/transform-source-type source)))
+ (assoc transform
+ :source_type (transforms.util/transform-source-type source)
+ :target_db_id (or (:database target) (:target_db_id transform))))
(t2/define-before-update :model/Transform
- [{:keys [source] :as transform}]
+ [{:keys [source target] :as transform}]
(when-let [new-collection (:collection_id (t2/changes transform))]
(collection/check-collection-namespace :model/Transform new-collection)
(collection/check-allowed-content :model/Transform new-collection))
- (if source
- (assoc transform :source_type (transforms.util/transform-source-type source))
- transform))
+ (cond-> transform
+ source (assoc :source_type (transforms.util/transform-source-type source))
+ (:database target) (assoc :target_db_id (:database target))))
(t2/define-after-select :model/Transform
[{:keys [source] :as transform}]
@@ -232,8 +234,8 @@
(defmethod serdes/make-spec "Transform"
[_model-name opts]
- {:copy [:name :description :entity_id]
- :skip [:dependency_analysis_version :source_type]
+ {:copy [:name :description :entity_id]
+ :skip [:dependency_analysis_version :source_type :target_db_id]
:transform {:created_at (serdes/date)
:creator_id (serdes/fk :model/User)
:collection_id (serdes/fk :model/Collection)
@@ -267,15 +269,6 @@
(when query-text
(subs query-text 0 (min (count query-text) search/max-searchable-value-length)))))
-(defn- extract-transform-db-id
- "Return the database ID from transform source; else nil."
- [{:keys [source]}]
- (let [parsed-source (transform-source-out source)]
- (case (:type parsed-source)
- :query (get-in parsed-source [:query :database])
- :python (parsed-source :source-database)
- nil)))
-
;;; ------------------------------------------------- Search ---------------------------------------------------
(search.spec/define-spec "transform"
@@ -289,8 +282,7 @@
:view-count false
:native-query {:fn maybe-extract-transform-query-text
:fields [:source :source_type]}
- :database-id {:fn extract-transform-db-id
- :fields [:source]}}
+ :database-id :target_db_id}
:search-terms [:name :description]
:render-terms {:transform-name :name
:transform-id :id}})
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj
index 17ce1b442ef..4bccf38e3e8 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj
@@ -20,7 +20,8 @@
(defmethod transforms.i/target-db-id :query
[transform]
- (-> transform :source :query :database))
+ (or (get-in transform [:target :database])
+ (:target_db_id transform)))
(mr/def ::transform-details
[:map
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj
index 4f12c7cb12a..b427bd71fd0 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj
@@ -5,7 +5,8 @@
(defmethod transforms.i/target-db-id :python
[transform]
- (-> transform :target :database))
+ (or (get-in transform [:target :database])
+ (:target_db_id transform)))
(defmethod transforms.i/source-db-id :python
[transform]
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
index 8d74c2baaf1..0a78e800c81 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
@@ -61,7 +61,8 @@
:query query}
:target {:type "table"
:schema schema
- :name table-name}})
+ :name table-name
+ :database (mt/id)}})
transform-id (:id response)
crowberto-id (mt/user->id :crowberto)
creator-id (t2/select-one-fn :creator_id :model/Transform transform-id)]
@@ -88,7 +89,8 @@
:query mbql-query}
:target {:type "table"
:schema schema
- :name table-name}})]
+ :name table-name
+ :database (mt/id)}})]
(is (= "mbql" (:source_type response))))))
(testing "Native query transforms are detected as :native"
@@ -100,7 +102,8 @@
:query (lib/native-query (mt/metadata-provider) "SELECT 1")}
:target {:type "table"
:schema schema
- :name table-name}})]
+ :name table-name
+ :database (mt/id)}})]
(is (= "native" (:source_type response))))))
(testing "Python transforms are detected as :python"
@@ -132,7 +135,8 @@
:query native-query}
:target {:type "table"
:schema schema
- :name table-name}})]
+ :name table-name
+ :database (mt/id)}})]
(is (= "native" (:source_type created)))
(testing "Type automatically changes to mbql when updating to an MBQL query"
@@ -155,7 +159,8 @@
:query query}
:target {:type "table"
:schema schema
- :name "test_transform"}})]
+ :name "test_transform"
+ :database (mt/id)}})]
(is (= "error-premium-feature-not-available" (:status response)))))))
(testing "Creating a query transform with :transforms feature succeeds"
@@ -170,7 +175,8 @@
:query query}
:target {:type "table"
:schema schema
- :name table-name}})]
+ :name table-name
+ :database (mt/id)}})]
(is (some? (:id response))))))))))
(deftest update-transform-feature-flag-test
@@ -186,7 +192,8 @@
:query query}
:target {:type "table"
:schema schema
- :name table-name}}
+ :name table-name
+ :database (mt/id)}}
created (mt/user-http-request :crowberto :post 200 "ee/transform" transform-payload)]
;; Now test update without feature flag
(mt/with-premium-features #{}
@@ -208,7 +215,8 @@
:query query}
:target {:type "table"
:schema schema
- :name table-name}}
+ :name table-name
+ :database (mt/id)}}
created (mt/user-http-request :crowberto :post 200 "ee/transform" transform-payload)]
;; Now test run without feature flag
(mt/with-premium-features #{}
@@ -231,7 +239,8 @@
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
- :name table-name}}
+ :name table-name
+ :database (mt/id)}}
_ (mt/user-http-request :crowberto :post 200 "ee/transform" body)
list-resp (mt/user-http-request :crowberto :get 200 "ee/transform")
crowberto-id (mt/user->id :crowberto)]
@@ -280,7 +289,8 @@
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
- :name table-name}}
+ :name table-name
+ :database (mt/id)}}
resp (mt/user-http-request :crowberto :post 200 "ee/transform" body)
get-resp (mt/user-http-request :crowberto :get 200 (format "ee/transform/%s" (:id resp)))
crowberto-id (mt/user->id :crowberto)]
@@ -296,7 +306,8 @@
:name transform-name
:target {:schema "public"
:name "orders_2"
- :type "table"}})
+ :type "table"
+ :database (mt/id)}})
(deftest get-transform-dependencies-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
@@ -331,14 +342,16 @@
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
- :name table-name}})
+ :name table-name
+ :database (mt/id)}})
transform {:name "Gadget Products 2"
:description "Desc"
:source {:type "query"
:query query2}
:target {:type "table"
:schema (get-test-schema)
- :name table-name}}
+ :name table-name
+ :database (mt/id)}}
put-resp (mt/user-http-request :crowberto :put 200 (format "ee/transform/%s" (:id resp))
transform)
crowberto-id (mt/user->id :crowberto)]
@@ -360,7 +373,8 @@
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
- :name table1-name}}
+ :name table1-name
+ :database (mt/id)}}
resp (mt/user-http-request :crowberto :post 200 "ee/transform"
original)
updated {:name "Doohickey Products"
@@ -369,7 +383,8 @@
:query query2}
:target {:type "table"
:schema (get-test-schema)
- :name table2-name}}]
+ :name table2-name
+ :database (mt/id)}}]
(is (=? (-> updated
(m/dissoc-in [:source :query :lib/metadata]))
(-> (mt/user-http-request :crowberto :put 200 (format "ee/transform/%s" (:id resp)) updated)
@@ -387,7 +402,8 @@
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
- :name table-name}})]
+ :name table-name
+ :database (mt/id)}})]
(mt/user-http-request :crowberto :delete 204 (format "ee/transform/%s" (:id resp)))
(mt/user-http-request :crowberto :get 404 (format "ee/transform/%s" (:id resp)))))))))
@@ -402,7 +418,8 @@
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
- :name table-name}})]
+ :name table-name
+ :database (mt/id)}})]
(mt/user-http-request :crowberto :delete 204 (format "ee/transform/%s/table" (:id resp)))))))))
(defn- test-run
@@ -482,10 +499,12 @@
(let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
(with-transform-cleanup! [{table1-name :name :as target1} {:type "table"
:schema schema
- :name "gadget_products"}
+ :name "gadget_products"
+ :database (mt/id)}
{table2-name :name :as target2} {:type "table"
:schema schema
- :name "doohickey_products"}]
+ :name "doohickey_products"
+ :database (mt/id)}]
(let [query2 (make-query "Doohickey")
original {:name "Gadget Products"
:source {:type "query"
@@ -853,7 +872,7 @@
(mt/user-http-request :crowberto :post 400 "ee/transform"
{:name "Gadget Products"
:source {:type "query" :query query}
- :target {:type "table" :schema schema :name table-name}}))))))))))
+ :target {:type "table" :schema schema :name table-name :database (mt/id)}}))))))))))
(deftest update-transform-with-routing-fails-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
@@ -869,7 +888,7 @@
:user_attribute "db_name"}
:model/Transform transform {:name "Gadget Products"
:source {:type "query" :query query}
- :target {:type "table" :schema schema :name table-name}}]
+ :target {:type "table" :schema schema :name table-name :database (mt/id)}}]
(is (= "Transforms are not supported on databases with DB routing enabled."
(mt/user-http-request :crowberto :put 400 (format "ee/transform/%s" (:id transform))
(assoc transform :name "Gadget Products 2")))))))))))
@@ -901,7 +920,8 @@
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
- :name table-name}}
+ :name table-name
+ :database (mt/id)}}
transform-id (test-transform-revisions :post "ee/transform" gadget-req 1)
widget-req {:name "Widget Products"
:description "The widget products"
@@ -910,7 +930,8 @@
:tag_ids [4]
:target {:type "table"
:schema (get-test-schema)
- :name table-name}}]
+ :name table-name
+ :database (mt/id)}}]
(test-transform-revisions :put (str "ee/transform/" transform-id) widget-req 2)))))))
(defmethod driver/database-supports? [::driver/driver ::extract-columns-from-query]
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/models_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/models_test.clj
index 3c501aa8660..35368a13add 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/models_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/models_test.clj
@@ -185,7 +185,7 @@
:target {:type "table"
:schema "public"
:name "test_table_with_schema"
- :db_id db-id}}
+ :database db-id}}
:model/Transform transform2 {:name "Transform with NULL schema"
:source {:type "query"
:query {:database db-id
@@ -195,7 +195,7 @@
:target {:type "table"
:schema nil
:name "test_table_null_schema"
- :db_id db-id}}]
+ :database db-id}}]
(testing "Hydrates table with non-NULL schema"
(let [hydrated (t2/hydrate transform1 :table-with-db-and-fields)]
@@ -246,7 +246,7 @@
:target {:type "table"
:schema "public"
:name "test_table"
- :db_id db-id}}
+ :database db-id}}
:model/User {user2-id :id} user2-data
:model/Transform transform2 {:name "Second transform"
:creator_id user2-id
@@ -258,7 +258,7 @@
:target {:type "table"
:schema "public"
:name "test_table2"
- :db_id db-id}}]
+ :database db-id}}]
(testing "Hydrates creator with user details"
(is (=? (assoc user1-data :id user1-id)
(:creator (t2/hydrate transform1 :creator)))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
index cbe370e31ad..1dd0f3e0995 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
@@ -9,16 +9,18 @@
[toucan2.core :as t2]))
(defn- make-transform [query & [name schema]]
- (let [name (or name (mt/random-name))
+ (let [name (or name (mt/random-name))
default-schema (when (get-method driver.sql/default-schema driver/*driver*)
(driver.sql/default-schema driver/*driver*))
- schema (or schema default-schema "public")]
- {:source {:type :query
- :query query}
- :name (str "transform_" name)
- :target {:schema schema
- :name name
- :type :table}}))
+ schema (or schema default-schema "public")]
+ {:source {:type :query
+ :query query}
+ :name (str "transform_" name)
+ :target {:schema schema
+ :name name
+ :type :table}
+ :target_db_id (or (:database query)
+ (mt/id))}))
(defn- make-python-transform [source-tables & [name schema target-db]]
(let [name (or name (mt/random-name))
@@ -33,7 +35,8 @@
:target {:database target-db
:schema schema
:name name
- :type "table"}}))
+ :type "table"}
+ :target_db_id target-db}))
(defn- transform-deps-for-db [transform]
(mt/with-metadata-provider (mt/id)
diff --git a/resources/migrations/058_update_migrations.yaml b/resources/migrations/058_update_migrations.yaml
index 1ce7b8282e9..0c7e1067e4d 100644
--- a/resources/migrations/058_update_migrations.yaml
+++ b/resources/migrations/058_update_migrations.yaml
@@ -1235,7 +1235,81 @@ databaseChangeLog:
path: instance_analytics_views/tables/v1/h2-tables.sql
relativeToChangelogFile: true
-# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
+ - changeSet:
+ id: v58.2025-12-15T12:16:15
+ author: piranha
+ comment: Add target_db_id column to transform table
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - columnExists:
+ tableName: transform
+ columnName: target_db_id
+ changes:
+ - addColumn:
+ tableName: transform
+ columns:
+ - column:
+ name: target_db_id
+ type: int
+ remarks: Target database ID for the transform output
+ constraints:
+ nullable: true
+ rollback:
+ - dropColumn:
+ tableName: transform
+ columnName: target_db_id
+
+ - changeSet:
+ id: v58.2025-12-15T12:16:16
+ author: piranha
+ comment: Backfill target_db_id column for existing transforms
+ changes:
+ - customChange:
+ class: "metabase.app_db.custom_migrations.BackfillTransformTargetDbId"
+
+ - changeSet:
+ id: v58.2025-12-15T12:16:17
+ author: piranha
+ comment: Make transform.target_db_id column NOT NULL
+ changes:
+ - addNotNullConstraint:
+ tableName: transform
+ columnName: target_db_id
+ columnDataType: int
+ rollback:
+ - dropNotNullConstraint:
+ tableName: transform
+ columnName: target_db_id
+ columnDataType: int
+
+ - changeSet:
+ id: v58.2025-12-15T12:16:18
+ author: piranha
+ comment: Add index on transform.target_db_id
+ changes:
+ - createIndex:
+ tableName: transform
+ indexName: idx_transform_target_db_id
+ columns:
+ - column:
+ name: target_db_id
+ rollback:
+ - dropIndex:
+ tableName: transform
+ indexName: idx_transform_target_db_id
+
+ - changeSet:
+ id: v58.2025-12-15T12:16:19
+ author: piranha
+ comment: Add foreign key constraint for transform.target_db_id
+ changes:
+ - addForeignKeyConstraint:
+ baseTableName: transform
+ baseColumnNames: target_db_id
+ referencedTableName: metabase_database
+ referencedColumnNames: id
+ constraintName: fk_transform_ref_target_db_id
########################################################################################################################
#
diff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.clj
index bdaf6cd2944..21d0812549f 100644
--- a/src/metabase/app_db/custom_migrations.clj
+++ b/src/metabase/app_db/custom_migrations.clj
@@ -1823,3 +1823,13 @@
(define-migration MoveExistingAtSymbolUserAttributes
(reserve-at-symbol-user-attributes/migrate!))
+
+(define-migration BackfillTransformTargetDbId
+ (let [update! (fn [{:keys [id target]}]
+ (let [target-map (json/decode target)
+ db-id (get target-map "database")]
+ (when db-id
+ (t2/update! :transform id {:target_db_id db-id}))))]
+ (run! update! (t2/reducible-query {:select [:id :target]
+ :from [:transform]
+ :where [:= :target_db_id nil]}))))
diff --git a/test/metabase/app_db/custom_migrations_test.clj b/test/metabase/app_db/custom_migrations_test.clj
index 6b0dc5c480e..ab2a32d962f 100644
--- a/test/metabase/app_db/custom_migrations_test.clj
+++ b/test/metabase/app_db/custom_migrations_test.clj
@@ -2725,3 +2725,37 @@
(json/decode (t2/select-one-fn :login_attributes :core_user :id user-id))))
(is (= {"_@foo" "bang"}
(json/decode (t2/select-one-fn :login_attributes :core_user :id other-user-id))))))))
+
+(deftest backfill-transform-target-database-id-test
+ (testing "Migration v58.2025-12-15T12:16:16: backfill target_db_id for existing transforms"
+ (impl/test-migrations ["v58.2025-12-15T12:16:16"] [migrate!]
+ (let [user-id (first (t2/insert-returning-pks! :core_user {:first_name "Test"
+ :last_name "User"
+ :password "password"
+ :date_joined :%now}))
+ db-id (first (t2/insert-returning-pks! :metabase_database {:name "test-db"
+ :engine "h2"
+ :details "{}"
+ :created_at :%now
+ :updated_at :%now}))
+ xf-id (first (t2/insert-returning-pks! :transform
+ {:name "Query transform"
+ :source (json/encode {:type "query"
+ :query {:database db-id
+ :type "native"
+ :native {:query "SELECT 1"}}})
+ :source_type "native"
+ :target (json/encode {:type "table"
+ :schema "public"
+ :name "output_table"
+ :database db-id})
+ :creator_id user-id
+ :created_at :%now
+ :updated_at :%now}))]
+ (testing "before migration, target_db_id is nil"
+ (is (nil? (:target_db_id (t2/select-one :transform :id xf-id)))))
+ (migrate!)
+ (testing "after migration, target_db_id is populated"
+ (is (= db-id (:target_db_id (t2/select-one :transform :id xf-id)))
+ "Query transform target_db_id should equal target.database"))))))
diff --git a/test/metabase/test/util.clj b/test/metabase/test/util.clj
index 6ffb00cb86e..5184ebb6661 100644
--- a/test/metabase/test/util.clj
+++ b/test/metabase/test/util.clj
@@ -333,7 +333,9 @@
:source {:type "query"
:query (lib/native-query (data/metadata-provider) "SELECT 1 as num")}
:target {:type "table"
- :name (str "test_table_" (u/generate-nano-id))}})
+ :name (str "test_table_" (u/generate-nano-id))
+ :database (data/id)}
+ :target_db_id (data/id)})
:model/TransformJob
(fn [_]