Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Revert "Add transforms.target_database_id column (#66953)" (#67630)

Chris Truterb222e32bdfe2
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj
index 69ad10d762d..70d3e5b3bc8 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj
@@ -61,22 +61,20 @@
#{:transforms})
(t2/define-before-insert :model/Transform
- [{:keys [source target collection_id] :as transform}]
+ [{:keys [source collection_id] :as transform}]
(collection/check-collection-namespace :model/Transform collection_id)
(when collection_id
(collection/check-allowed-content :model/Transform collection_id))
- (assoc transform
- :source_type (transforms.util/transform-source-type source)
- :target_db_id (or (:database target) (:target_db_id transform))))
+ (assoc transform :source_type (transforms.util/transform-source-type source)))
(t2/define-before-update :model/Transform
- [{:keys [source target] :as transform}]
+ [{:keys [source] :as transform}]
(when-let [new-collection (:collection_id (t2/changes transform))]
(collection/check-collection-namespace :model/Transform new-collection)
(collection/check-allowed-content :model/Transform new-collection))
- (cond-> transform
- source (assoc :source_type (transforms.util/transform-source-type source))
- (:database target) (assoc :target_db_id (:database target))))
+ (if source
+ (assoc transform :source_type (transforms.util/transform-source-type source))
+ transform))
(t2/define-after-select :model/Transform
[{:keys [source] :as transform}]
@@ -241,8 +239,8 @@
(defmethod serdes/make-spec "Transform"
[_model-name opts]
- {:copy [:name :description :entity_id]
- :skip [:dependency_analysis_version :source_type :target_db_id]
+ {:copy [:name :description :entity_id]
+ :skip [:dependency_analysis_version :source_type]
:transform {:created_at (serdes/date)
:creator_id (serdes/fk :model/User)
:collection_id (serdes/fk :model/Collection)
@@ -276,6 +274,15 @@
(when query-text
(subs query-text 0 (min (count query-text) search/max-searchable-value-length)))))
+(defn- extract-transform-db-id
+ "Return the database ID from transform source; else nil."
+ [{:keys [source]}]
+ (let [parsed-source (transform-source-out source)]
+ (case (:type parsed-source)
+ :query (get-in parsed-source [:query :database])
+ :python (parsed-source :source-database)
+ nil)))
+
;;; ------------------------------------------------- Search ---------------------------------------------------
(search.spec/define-spec "transform"
@@ -289,7 +296,8 @@
:view-count false
:native-query {:fn maybe-extract-transform-query-text
:fields [:source :source_type]}
- :database-id :target_db_id}
+ :database-id {:fn extract-transform-db-id
+ :fields [:source]}}
:search-terms [:name :description]
:render-terms {:transform-name :name
:transform-id :id}})
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj
index 4bccf38e3e8..17ce1b442ef 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj
@@ -20,8 +20,7 @@
(defmethod transforms.i/target-db-id :query
[transform]
- (or (get-in transform [:target :database])
- (:target_db_id transform)))
+ (-> transform :source :query :database))
(mr/def ::transform-details
[:map
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj
index 95ea6214d50..c16898ead94 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj
@@ -5,8 +5,7 @@
(defmethod transforms.i/target-db-id :python
[transform]
- (or (get-in transform [:target :database])
- (:target_db_id transform)))
+ (-> transform :target :database))
(defmethod transforms.i/source-db-id :python
[transform]
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
index 0a78e800c81..8d74c2baaf1 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
@@ -61,8 +61,7 @@
:query query}
:target {:type "table"
:schema schema
- :name table-name
- :database (mt/id)}})
+ :name table-name}})
transform-id (:id response)
crowberto-id (mt/user->id :crowberto)
creator-id (t2/select-one-fn :creator_id :model/Transform transform-id)]
@@ -89,8 +88,7 @@
:query mbql-query}
:target {:type "table"
:schema schema
- :name table-name
- :database (mt/id)}})]
+ :name table-name}})]
(is (= "mbql" (:source_type response))))))
(testing "Native query transforms are detected as :native"
@@ -102,8 +100,7 @@
:query (lib/native-query (mt/metadata-provider) "SELECT 1")}
:target {:type "table"
:schema schema
- :name table-name
- :database (mt/id)}})]
+ :name table-name}})]
(is (= "native" (:source_type response))))))
(testing "Python transforms are detected as :python"
@@ -135,8 +132,7 @@
:query native-query}
:target {:type "table"
:schema schema
- :name table-name
- :database (mt/id)}})]
+ :name table-name}})]
(is (= "native" (:source_type created)))
(testing "Type automatically changes to mbql when updating to an MBQL query"
@@ -159,8 +155,7 @@
:query query}
:target {:type "table"
:schema schema
- :name "test_transform"
- :database (mt/id)}})]
+ :name "test_transform"}})]
(is (= "error-premium-feature-not-available" (:status response)))))))
(testing "Creating a query transform with :transforms feature succeeds"
@@ -175,8 +170,7 @@
:query query}
:target {:type "table"
:schema schema
- :name table-name
- :database (mt/id)}})]
+ :name table-name}})]
(is (some? (:id response))))))))))
(deftest update-transform-feature-flag-test
@@ -192,8 +186,7 @@
:query query}
:target {:type "table"
:schema schema
- :name table-name
- :database (mt/id)}}
+ :name table-name}}
created (mt/user-http-request :crowberto :post 200 "ee/transform" transform-payload)]
;; Now test update without feature flag
(mt/with-premium-features #{}
@@ -215,8 +208,7 @@
:query query}
:target {:type "table"
:schema schema
- :name table-name
- :database (mt/id)}}
+ :name table-name}}
created (mt/user-http-request :crowberto :post 200 "ee/transform" transform-payload)]
;; Now test run without feature flag
(mt/with-premium-features #{}
@@ -239,8 +231,7 @@
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
- :name table-name
- :database (mt/id)}}
+ :name table-name}}
_ (mt/user-http-request :crowberto :post 200 "ee/transform" body)
list-resp (mt/user-http-request :crowberto :get 200 "ee/transform")
crowberto-id (mt/user->id :crowberto)]
@@ -289,8 +280,7 @@
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
- :name table-name
- :database (mt/id)}}
+ :name table-name}}
resp (mt/user-http-request :crowberto :post 200 "ee/transform" body)
get-resp (mt/user-http-request :crowberto :get 200 (format "ee/transform/%s" (:id resp)))
crowberto-id (mt/user->id :crowberto)]
@@ -306,8 +296,7 @@
:name transform-name
:target {:schema "public"
:name "orders_2"
- :type "table"
- :database (mt/id)}})
+ :type "table"}})
(deftest get-transform-dependencies-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
@@ -342,16 +331,14 @@
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
- :name table-name
- :database (mt/id)}})
+ :name table-name}})
transform {:name "Gadget Products 2"
:description "Desc"
:source {:type "query"
:query query2}
:target {:type "table"
:schema (get-test-schema)
- :name table-name
- :database (mt/id)}}
+ :name table-name}}
put-resp (mt/user-http-request :crowberto :put 200 (format "ee/transform/%s" (:id resp))
transform)
crowberto-id (mt/user->id :crowberto)]
@@ -373,8 +360,7 @@
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
- :name table1-name
- :database (mt/id)}}
+ :name table1-name}}
resp (mt/user-http-request :crowberto :post 200 "ee/transform"
original)
updated {:name "Doohickey Products"
@@ -383,8 +369,7 @@
:query query2}
:target {:type "table"
:schema (get-test-schema)
- :name table2-name
- :database (mt/id)}}]
+ :name table2-name}}]
(is (=? (-> updated
(m/dissoc-in [:source :query :lib/metadata]))
(-> (mt/user-http-request :crowberto :put 200 (format "ee/transform/%s" (:id resp)) updated)
@@ -402,8 +387,7 @@
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
- :name table-name
- :database (mt/id)}})]
+ :name table-name}})]
(mt/user-http-request :crowberto :delete 204 (format "ee/transform/%s" (:id resp)))
(mt/user-http-request :crowberto :get 404 (format "ee/transform/%s" (:id resp)))))))))
@@ -418,8 +402,7 @@
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
- :name table-name
- :database (mt/id)}})]
+ :name table-name}})]
(mt/user-http-request :crowberto :delete 204 (format "ee/transform/%s/table" (:id resp)))))))))
(defn- test-run
@@ -499,12 +482,10 @@
(let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))]
(with-transform-cleanup! [{table1-name :name :as target1} {:type "table"
:schema schema
- :name "gadget_products"
- :database (mt/id)}
+ :name "gadget_products"}
{table2-name :name :as target2} {:type "table"
:schema schema
- :name "doohickey_products"
- :database (mt/id)}]
+ :name "doohickey_products"}]
(let [query2 (make-query "Doohickey")
original {:name "Gadget Products"
:source {:type "query"
@@ -872,7 +853,7 @@
(mt/user-http-request :crowberto :post 400 "ee/transform"
{:name "Gadget Products"
:source {:type "query" :query query}
- :target {:type "table" :schema schema :name table-name :database (mt/id)}}))))))))))
+ :target {:type "table" :schema schema :name table-name}}))))))))))
(deftest update-transform-with-routing-fails-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
@@ -888,7 +869,7 @@
:user_attribute "db_name"}
:model/Transform transform {:name "Gadget Products"
:source {:type "query" :query query}
- :target {:type "table" :schema schema :name table-name :database (mt/id)}}]
+ :target {:type "table" :schema schema :name table-name}}]
(is (= "Transforms are not supported on databases with DB routing enabled."
(mt/user-http-request :crowberto :put 400 (format "ee/transform/%s" (:id transform))
(assoc transform :name "Gadget Products 2")))))))))))
@@ -920,8 +901,7 @@
:query (make-query "Gadget")}
:target {:type "table"
:schema (get-test-schema)
- :name table-name
- :database (mt/id)}}
+ :name table-name}}
transform-id (test-transform-revisions :post "ee/transform" gadget-req 1)
widget-req {:name "Widget Products"
:description "The widget products"
@@ -930,8 +910,7 @@
:tag_ids [4]
:target {:type "table"
:schema (get-test-schema)
- :name table-name
- :database (mt/id)}}]
+ :name table-name}}]
(test-transform-revisions :put (str "ee/transform/" transform-id) widget-req 2)))))))
(defmethod driver/database-supports? [::driver/driver ::extract-columns-from-query]
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/models_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/models_test.clj
index 273453db6cc..53375b97f0e 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/models_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/models_test.clj
@@ -185,7 +185,7 @@
:target {:type "table"
:schema "public"
:name "test_table_with_schema"
- :database db-id}}
+ :db_id db-id}}
:model/Transform transform2 {:name "Transform with NULL schema"
:source {:type "query"
:query {:database db-id
@@ -195,7 +195,7 @@
:target {:type "table"
:schema nil
:name "test_table_null_schema"
- :database db-id}}]
+ :db_id db-id}}]
(testing "Hydrates table with non-NULL schema"
(let [hydrated (t2/hydrate transform1 :table-with-db-and-fields)]
@@ -246,7 +246,7 @@
:target {:type "table"
:schema "public"
:name "test_table"
- :database db-id}}
+ :db_id db-id}}
:model/User {user2-id :id} user2-data
:model/Transform transform2 {:name "Second transform"
:creator_id user2-id
@@ -258,7 +258,7 @@
:target {:type "table"
:schema "public"
:name "test_table2"
- :database db-id}}]
+ :db_id db-id}}]
(testing "Hydrates creator with user details"
(is (=? (assoc user1-data :id user1-id)
(:creator (t2/hydrate transform1 :creator)))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
index 1dd0f3e0995..cbe370e31ad 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
@@ -9,18 +9,16 @@
[toucan2.core :as t2]))
(defn- make-transform [query & [name schema]]
- (let [name (or name (mt/random-name))
+ (let [name (or name (mt/random-name))
default-schema (when (get-method driver.sql/default-schema driver/*driver*)
(driver.sql/default-schema driver/*driver*))
- schema (or schema default-schema "public")]
- {:source {:type :query
- :query query}
- :name (str "transform_" name)
- :target {:schema schema
- :name name
- :type :table}
- :target_db_id (or (:database query)
- (mt/id))}))
+ schema (or schema default-schema "public")]
+ {:source {:type :query
+ :query query}
+ :name (str "transform_" name)
+ :target {:schema schema
+ :name name
+ :type :table}}))
(defn- make-python-transform [source-tables & [name schema target-db]]
(let [name (or name (mt/random-name))
@@ -35,8 +33,7 @@
:target {:database target-db
:schema schema
:name name
- :type "table"}
- :target_db_id target-db}))
+ :type "table"}}))
(defn- transform-deps-for-db [transform]
(mt/with-metadata-provider (mt/id)
diff --git a/resources/migrations/058_update_migrations.yaml b/resources/migrations/058_update_migrations.yaml
index e4b12d43671..32e48684323 100644
--- a/resources/migrations/058_update_migrations.yaml
+++ b/resources/migrations/058_update_migrations.yaml
@@ -1235,81 +1235,7 @@ databaseChangeLog:
path: instance_analytics_views/tables/v1/h2-tables.sql
relativeToChangelogFile: true
- - changeSet:
- id: v58.2025-12-15T12:16:15
- author: piranha
- comment: Add target_db_id column to transform table
- preConditions:
- - onFail: MARK_RAN
- - not:
- - columnExists:
- tableName: transform
- columnName: target_db_id
- changes:
- - addColumn:
- tableName: transform
- columns:
- - column:
- name: target_db_id
- type: int
- remarks: Target database ID for the transform output
- constraints:
- nullable: true
- rollback:
- - dropColumn:
- tableName: transform
- columnName: target_db_id
-
- - changeSet:
- id: v58.2025-12-15T12:16:16
- author: piranha
- comment: Backfill target_db_id column for existing transforms
- changes:
- - customChange:
- class: "metabase.app_db.custom_migrations.BackfillTransformTargetDbId"
-
- - changeSet:
- id: v58.2025-12-15T12:16:17
- author: piranha
- comment: Make transform.target_db_id column NOT NULL
- changes:
- - addNotNullConstraint:
- tableName: transform
- columnName: target_db_id
- columnDataType: int
- rollback:
- - dropNotNullConstraint:
- tableName: transform
- columnName: target_db_id
- columnDataType: int
-
- - changeSet:
- id: v58.2025-12-15T12:16:18
- author: piranha
- comment: Add index on transform.target_db_id
- changes:
- - createIndex:
- tableName: transform
- indexName: idx_transform_target_db_id
- columns:
- - column:
- name: target_db_id
- rollback:
- - dropIndex:
- tableName: transform
- indexName: idx_transform_target_db_id
-
- - changeSet:
- id: v58.2025-12-15T12:16:19
- author: piranha
- comment: Add foreign key constraint for transform.target_db_id
- changes:
- - addForeignKeyConstraint:
- baseTableName: transform
- baseColumnNames: target_db_id
- referencedTableName: metabase_database
- referencedColumnNames: id
- constraintName: fk_transform_ref_target_db_id
+# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
- changeSet:
id: v58.2025-12-18T16:14:03
diff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.clj
index 21d0812549f..bdaf6cd2944 100644
--- a/src/metabase/app_db/custom_migrations.clj
+++ b/src/metabase/app_db/custom_migrations.clj
@@ -1823,13 +1823,3 @@
(define-migration MoveExistingAtSymbolUserAttributes
(reserve-at-symbol-user-attributes/migrate!))
-
-(define-migration BackfillTransformTargetDbId
- (let [update! (fn [{:keys [id target]}]
- (let [target-map (json/decode target)
- db-id (get target-map "database")]
- (when db-id
- (t2/update! :transform id {:target_db_id db-id}))))]
- (run! update! (t2/reducible-query {:select [:id :target]
- :from [:transform]
- :where [:= :target_db_id nil]}))))
diff --git a/test/metabase/app_db/custom_migrations_test.clj b/test/metabase/app_db/custom_migrations_test.clj
index ab2a32d962f..6b0dc5c480e 100644
--- a/test/metabase/app_db/custom_migrations_test.clj
+++ b/test/metabase/app_db/custom_migrations_test.clj
@@ -2725,37 +2725,3 @@
(json/decode (t2/select-one-fn :login_attributes :core_user :id user-id))))
(is (= {"_@foo" "bang"}
(json/decode (t2/select-one-fn :login_attributes :core_user :id other-user-id))))))))
-
-(deftest backfill-transform-target-database-id-test
- (testing "Migration v58.2025-12-15T12:16:16: backfill target_db_id for existing transforms"
- (impl/test-migrations ["v58.2025-12-15T12:16:16"] [migrate!]
- (let [user-id (first (t2/insert-returning-pks! :core_user {:first_name "Test"
- :last_name "User"
- :password "password"
- :date_joined :%now}))
- db-id (first (t2/insert-returning-pks! :metabase_database {:name "test-db"
- :engine "h2"
- :details "{}"
- :created_at :%now
- :updated_at :%now}))
- xf-id (first (t2/insert-returning-pks! :transform
- {:name "Query transform"
- :source (json/encode {:type "query"
- :query {:database db-id
- :type "native"
- :native {:query "SELECT 1"}}})
- :source_type "native"
- :target (json/encode {:type "table"
- :schema "public"
- :name "output_table"
- :database db-id})
- :creator_id user-id
- :created_at :%now
- :updated_at :%now}))]
- (testing "before migration, target_db_id is nil"
- (is (nil? (:target_db_id (t2/select-one :transform :id xf-id)))))
- (migrate!)
- (testing "after migration, target_db_id is populated"
- (is (= db-id (:target_db_id (t2/select-one :transform :id xf-id)))
- "Query transform target_db_id should equal target.database"))))))
diff --git a/test/metabase/test/util.clj b/test/metabase/test/util.clj
index 5184ebb6661..6ffb00cb86e 100644
--- a/test/metabase/test/util.clj
+++ b/test/metabase/test/util.clj
@@ -333,9 +333,7 @@
:source {:type "query"
:query (lib/native-query (data/metadata-provider) "SELECT 1 as num")}
:target {:type "table"
- :name (str "test_table_" (u/generate-nano-id))
- :database (data/id)}
- :target_db_id (data/id)})
+ :name (str "test_table_" (u/generate-nano-id))}})
:model/TransformJob
(fn [_]