The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Revert "Add transforms.target_database_id column (#66953)" (#67630)
Chris Truterb222e32bdfe2
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.cljindex 69ad10d762d..70d3e5b3bc8 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj@@ -61,22 +61,20 @@#{:transforms})(t2/define-before-insert :model/Transform- [{:keys [source target collection_id] :as transform}]+ [{:keys [source collection_id] :as transform}](collection/check-collection-namespace :model/Transform collection_id)(when collection_id(collection/check-allowed-content :model/Transform collection_id))- (assoc transform- :source_type (transforms.util/transform-source-type source)- :target_db_id (or (:database target) (:target_db_id transform))))+ (assoc transform :source_type (transforms.util/transform-source-type source)))(t2/define-before-update :model/Transform- [{:keys [source target] :as transform}]+ [{:keys [source] :as transform}](when-let [new-collection (:collection_id (t2/changes transform))](collection/check-collection-namespace :model/Transform new-collection)(collection/check-allowed-content :model/Transform new-collection))- (cond-> transform- source (assoc :source_type (transforms.util/transform-source-type source))- (:database target) (assoc :target_db_id (:database target))))+ (if source+ (assoc transform :source_type (transforms.util/transform-source-type source))+ transform))(t2/define-after-select :model/Transform[{:keys [source] :as transform}]@@ -241,8 +239,8 @@(defmethod serdes/make-spec "Transform"[_model-name opts]- {:copy [:name :description :entity_id]- :skip [:dependency_analysis_version :source_type :target_db_id]+ {:copy [:name :description :entity_id]+ :skip [:dependency_analysis_version :source_type]:transform {:created_at (serdes/date):creator_id (serdes/fk :model/User):collection_id (serdes/fk :model/Collection)@@ -276,6 +274,15 @@(when query-text(subs query-text 0 (min (count query-text) search/max-searchable-value-length)))))+(defn- extract-transform-db-id+ "Return the database ID from transform source; else nil."+ [{:keys [source]}]+ (let [parsed-source (transform-source-out source)]+ (case (:type parsed-source)+ :query (get-in parsed-source [:query :database])+ :python (parsed-source :source-database)+ nil)))+;;; ------------------------------------------------- Search ---------------------------------------------------(search.spec/define-spec "transform"@@ -289,7 +296,8 @@:view-count false:native-query {:fn maybe-extract-transform-query-text:fields [:source :source_type]}- :database-id :target_db_id}+ :database-id {:fn extract-transform-db-id+ :fields [:source]}}:search-terms [:name :description]:render-terms {:transform-name :name:transform-id :id}})diff --git a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.cljindex 4bccf38e3e8..17ce1b442ef 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj@@ -20,8 +20,7 @@(defmethod transforms.i/target-db-id :query[transform]- (or (get-in transform [:target :database])- (:target_db_id transform)))+ (-> transform :source :query :database))(mr/def ::transform-details[:mapdiff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/impl.cljindex 95ea6214d50..c16898ead94 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj@@ -5,8 +5,7 @@(defmethod transforms.i/target-db-id :python[transform]- (or (get-in transform [:target :database])- (:target_db_id transform)))+ (-> transform :target :database))(defmethod transforms.i/source-db-id :python[transform]diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.cljindex 0a78e800c81..8d74c2baaf1 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj@@ -61,8 +61,7 @@:query query}:target {:type "table":schema schema- :name table-name- :database (mt/id)}})+ :name table-name}})transform-id (:id response)crowberto-id (mt/user->id :crowberto)creator-id (t2/select-one-fn :creator_id :model/Transform transform-id)]@@ -89,8 +88,7 @@:query mbql-query}:target {:type "table":schema schema- :name table-name- :database (mt/id)}})]+ :name table-name}})](is (= "mbql" (:source_type response))))))(testing "Native query transforms are detected as :native"@@ -102,8 +100,7 @@:query (lib/native-query (mt/metadata-provider) "SELECT 1")}:target {:type "table":schema schema- :name table-name- :database (mt/id)}})]+ :name table-name}})](is (= "native" (:source_type response))))))(testing "Python transforms are detected as :python"@@ -135,8 +132,7 @@:query native-query}:target {:type "table":schema schema- :name table-name- :database (mt/id)}})]+ :name table-name}})](is (= "native" (:source_type created)))(testing "Type automatically changes to mbql when updating to an MBQL query"@@ -159,8 +155,7 @@:query query}:target {:type "table":schema schema- :name "test_transform"- :database (mt/id)}})]+ :name "test_transform"}})](is (= "error-premium-feature-not-available" (:status response)))))))(testing "Creating a query transform with :transforms feature succeeds"@@ -175,8 +170,7 @@:query query}:target {:type "table":schema schema- :name table-name- :database (mt/id)}})]+ :name table-name}})](is (some? (:id response))))))))))(deftest update-transform-feature-flag-test@@ -192,8 +186,7 @@:query query}:target {:type "table":schema schema- :name table-name- :database (mt/id)}}+ :name table-name}}created (mt/user-http-request :crowberto :post 200 "ee/transform" transform-payload)];; Now test update without feature flag(mt/with-premium-features #{}@@ -215,8 +208,7 @@:query query}:target {:type "table":schema schema- :name table-name- :database (mt/id)}}+ :name table-name}}created (mt/user-http-request :crowberto :post 200 "ee/transform" transform-payload)];; Now test run without feature flag(mt/with-premium-features #{}@@ -239,8 +231,7 @@:query (make-query "Gadget")}:target {:type "table":schema (get-test-schema)- :name table-name- :database (mt/id)}}+ :name table-name}}_ (mt/user-http-request :crowberto :post 200 "ee/transform" body)list-resp (mt/user-http-request :crowberto :get 200 "ee/transform")crowberto-id (mt/user->id :crowberto)]@@ -289,8 +280,7 @@:query (make-query "Gadget")}:target {:type "table":schema (get-test-schema)- :name table-name- :database (mt/id)}}+ :name table-name}}resp (mt/user-http-request :crowberto :post 200 "ee/transform" body)get-resp (mt/user-http-request :crowberto :get 200 (format "ee/transform/%s" (:id resp)))crowberto-id (mt/user->id :crowberto)]@@ -306,8 +296,7 @@:name transform-name:target {:schema "public":name "orders_2"- :type "table"- :database (mt/id)}})+ :type "table"}})(deftest get-transform-dependencies-test(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)@@ -342,16 +331,14 @@:query (make-query "Gadget")}:target {:type "table":schema (get-test-schema)- :name table-name- :database (mt/id)}})+ :name table-name}})transform {:name "Gadget Products 2":description "Desc":source {:type "query":query query2}:target {:type "table":schema (get-test-schema)- :name table-name- :database (mt/id)}}+ :name table-name}}put-resp (mt/user-http-request :crowberto :put 200 (format "ee/transform/%s" (:id resp))transform)crowberto-id (mt/user->id :crowberto)]@@ -373,8 +360,7 @@:query (make-query "Gadget")}:target {:type "table":schema (get-test-schema)- :name table1-name- :database (mt/id)}}+ :name table1-name}}resp (mt/user-http-request :crowberto :post 200 "ee/transform"original)updated {:name "Doohickey Products"@@ -383,8 +369,7 @@:query query2}:target {:type "table":schema (get-test-schema)- :name table2-name- :database (mt/id)}}]+ :name table2-name}}](is (=? (-> updated(m/dissoc-in [:source :query :lib/metadata]))(-> (mt/user-http-request :crowberto :put 200 (format "ee/transform/%s" (:id resp)) updated)@@ -402,8 +387,7 @@:query (make-query "Gadget")}:target {:type "table":schema (get-test-schema)- :name table-name- :database (mt/id)}})]+ :name table-name}})](mt/user-http-request :crowberto :delete 204 (format "ee/transform/%s" (:id resp)))(mt/user-http-request :crowberto :get 404 (format "ee/transform/%s" (:id resp)))))))))@@ -418,8 +402,7 @@:query (make-query "Gadget")}:target {:type "table":schema (get-test-schema)- :name table-name- :database (mt/id)}})]+ :name table-name}})](mt/user-http-request :crowberto :delete 204 (format "ee/transform/%s/table" (:id resp)))))))))(defn- test-run@@ -499,12 +482,10 @@(let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))](with-transform-cleanup! [{table1-name :name :as target1} {:type "table":schema schema- :name "gadget_products"- :database (mt/id)}+ :name "gadget_products"}{table2-name :name :as target2} {:type "table":schema schema- :name "doohickey_products"- :database (mt/id)}]+ :name "doohickey_products"}](let [query2 (make-query "Doohickey")original {:name "Gadget Products":source {:type "query"@@ -872,7 +853,7 @@(mt/user-http-request :crowberto :post 400 "ee/transform"{:name "Gadget Products":source {:type "query" :query query}- :target {:type "table" :schema schema :name table-name :database (mt/id)}}))))))))))+ :target {:type "table" :schema schema :name table-name}}))))))))))(deftest update-transform-with-routing-fails-test(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)@@ -888,7 +869,7 @@:user_attribute "db_name"}:model/Transform transform {:name "Gadget Products":source {:type "query" :query query}- :target {:type "table" :schema schema :name table-name :database (mt/id)}}]+ :target {:type "table" :schema schema :name table-name}}](is (= "Transforms are not supported on databases with DB routing enabled."(mt/user-http-request :crowberto :put 400 (format "ee/transform/%s" (:id transform))(assoc transform :name "Gadget Products 2")))))))))))@@ -920,8 +901,7 @@:query (make-query "Gadget")}:target {:type "table":schema (get-test-schema)- :name table-name- :database (mt/id)}}+ :name table-name}}transform-id (test-transform-revisions :post "ee/transform" gadget-req 1)widget-req {:name "Widget Products":description "The widget products"@@ -930,8 +910,7 @@:tag_ids [4]:target {:type "table":schema (get-test-schema)- :name table-name- :database (mt/id)}}]+ :name table-name}}](test-transform-revisions :put (str "ee/transform/" transform-id) widget-req 2)))))))(defmethod driver/database-supports? [::driver/driver ::extract-columns-from-query]diff --git a/enterprise/backend/test/metabase_enterprise/transforms/models_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/models_test.cljindex 273453db6cc..53375b97f0e 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/models_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/models_test.clj@@ -185,7 +185,7 @@:target {:type "table":schema "public":name "test_table_with_schema"- :database db-id}}+ :db_id db-id}}:model/Transform transform2 {:name "Transform with NULL schema":source {:type "query":query {:database db-id@@ -195,7 +195,7 @@:target {:type "table":schema nil:name "test_table_null_schema"- :database db-id}}]+ :db_id db-id}}](testing "Hydrates table with non-NULL schema"(let [hydrated (t2/hydrate transform1 :table-with-db-and-fields)]@@ -246,7 +246,7 @@:target {:type "table":schema "public":name "test_table"- :database db-id}}+ :db_id db-id}}:model/User {user2-id :id} user2-data:model/Transform transform2 {:name "Second transform":creator_id user2-id@@ -258,7 +258,7 @@:target {:type "table":schema "public":name "test_table2"- :database db-id}}]+ :db_id db-id}}](testing "Hydrates creator with user details"(is (=? (assoc user1-data :id user1-id)(:creator (t2/hydrate transform1 :creator)))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.cljindex 1dd0f3e0995..cbe370e31ad 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj@@ -9,18 +9,16 @@[toucan2.core :as t2]))(defn- make-transform [query & [name schema]]- (let [name (or name (mt/random-name))+ (let [name (or name (mt/random-name))default-schema (when (get-method driver.sql/default-schema driver/*driver*)(driver.sql/default-schema driver/*driver*))- schema (or schema default-schema "public")]- {:source {:type :query- :query query}- :name (str "transform_" name)- :target {:schema schema- :name name- :type :table}- :target_db_id (or (:database query)- (mt/id))}))+ schema (or schema default-schema "public")]+ {:source {:type :query+ :query query}+ :name (str "transform_" name)+ :target {:schema schema+ :name name+ :type :table}}))(defn- make-python-transform [source-tables & [name schema target-db]](let [name (or name (mt/random-name))@@ -35,8 +33,7 @@:target {:database target-db:schema schema:name name- :type "table"}- :target_db_id target-db}))+ :type "table"}}))(defn- transform-deps-for-db [transform](mt/with-metadata-provider (mt/id)diff --git a/resources/migrations/058_update_migrations.yaml b/resources/migrations/058_update_migrations.yamlindex e4b12d43671..32e48684323 100644--- a/resources/migrations/058_update_migrations.yaml+++ b/resources/migrations/058_update_migrations.yaml@@ -1235,81 +1235,7 @@ databaseChangeLog:path: instance_analytics_views/tables/v1/h2-tables.sqlrelativeToChangelogFile: true- - changeSet:- id: v58.2025-12-15T12:16:15- author: piranha- comment: Add target_db_id column to transform table- preConditions:- - onFail: MARK_RAN- - not:- - columnExists:- tableName: transform- columnName: target_db_id- changes:- - addColumn:- tableName: transform- columns:- - column:- name: target_db_id- type: int- remarks: Target database ID for the transform output- constraints:- nullable: true- rollback:- - dropColumn:- tableName: transform- columnName: target_db_id-- - changeSet:- id: v58.2025-12-15T12:16:16- author: piranha- comment: Backfill target_db_id column for existing transforms- changes:- - customChange:- class: "metabase.app_db.custom_migrations.BackfillTransformTargetDbId"-- - changeSet:- id: v58.2025-12-15T12:16:17- author: piranha- comment: Make transform.target_db_id column NOT NULL- changes:- - addNotNullConstraint:- tableName: transform- columnName: target_db_id- columnDataType: int- rollback:- - dropNotNullConstraint:- tableName: transform- columnName: target_db_id- columnDataType: int-- - changeSet:- id: v58.2025-12-15T12:16:18- author: piranha- comment: Add index on transform.target_db_id- changes:- - createIndex:- tableName: transform- indexName: idx_transform_target_db_id- columns:- - column:- name: target_db_id- rollback:- - dropIndex:- tableName: transform- indexName: idx_transform_target_db_id-- - changeSet:- id: v58.2025-12-15T12:16:19- author: piranha- comment: Add foreign key constraint for transform.target_db_id- changes:- - addForeignKeyConstraint:- baseTableName: transform- baseColumnNames: target_db_id- referencedTableName: metabase_database- referencedColumnNames: id- constraintName: fk_transform_ref_target_db_id+# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<- changeSet:id: v58.2025-12-18T16:14:03diff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.cljindex 21d0812549f..bdaf6cd2944 100644--- a/src/metabase/app_db/custom_migrations.clj+++ b/src/metabase/app_db/custom_migrations.clj@@ -1823,13 +1823,3 @@(define-migration MoveExistingAtSymbolUserAttributes(reserve-at-symbol-user-attributes/migrate!))--(define-migration BackfillTransformTargetDbId- (let [update! (fn [{:keys [id target]}]- (let [target-map (json/decode target)- db-id (get target-map "database")]- (when db-id- (t2/update! :transform id {:target_db_id db-id}))))]- (run! update! (t2/reducible-query {:select [:id :target]- :from [:transform]- :where [:= :target_db_id nil]}))))diff --git a/test/metabase/app_db/custom_migrations_test.clj b/test/metabase/app_db/custom_migrations_test.cljindex ab2a32d962f..6b0dc5c480e 100644--- a/test/metabase/app_db/custom_migrations_test.clj+++ b/test/metabase/app_db/custom_migrations_test.clj@@ -2725,37 +2725,3 @@(json/decode (t2/select-one-fn :login_attributes :core_user :id user-id))))(is (= {"_@foo" "bang"}(json/decode (t2/select-one-fn :login_attributes :core_user :id other-user-id))))))))--(deftest backfill-transform-target-database-id-test- (testing "Migration v58.2025-12-15T12:16:16: backfill target_db_id for existing transforms"- (impl/test-migrations ["v58.2025-12-15T12:16:16"] [migrate!]- (let [user-id (first (t2/insert-returning-pks! :core_user {:first_name "Test"- :last_name "User"- :email "[email protected]"- :password "password"- :date_joined :%now}))- db-id (first (t2/insert-returning-pks! :metabase_database {:name "test-db"- :engine "h2"- :details "{}"- :created_at :%now- :updated_at :%now}))- xf-id (first (t2/insert-returning-pks! :transform- {:name "Query transform"- :source (json/encode {:type "query"- :query {:database db-id- :type "native"- :native {:query "SELECT 1"}}})- :source_type "native"- :target (json/encode {:type "table"- :schema "public"- :name "output_table"- :database db-id})- :creator_id user-id- :created_at :%now- :updated_at :%now}))]- (testing "before migration, target_db_id is nil"- (is (nil? (:target_db_id (t2/select-one :transform :id xf-id)))))- (migrate!)- (testing "after migration, target_db_id is populated"- (is (= db-id (:target_db_id (t2/select-one :transform :id xf-id)))- "Query transform target_db_id should equal target.database"))))))diff --git a/test/metabase/test/util.clj b/test/metabase/test/util.cljindex 5184ebb6661..6ffb00cb86e 100644--- a/test/metabase/test/util.clj+++ b/test/metabase/test/util.clj@@ -333,9 +333,7 @@:source {:type "query":query (lib/native-query (data/metadata-provider) "SELECT 1 as num")}:target {:type "table"- :name (str "test_table_" (u/generate-nano-id))- :database (data/id)}- :target_db_id (data/id)})+ :name (str "test_table_" (u/generate-nano-id))}}):model/TransformJob(fn [_]