The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add transforms.target_database_id column (#66953)
Oleksandr Solovyov4c9fe80dc5ce
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.cljindex 9643ffcc860..6d79c3bb8c0 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj@@ -54,20 +54,22 @@#{:transforms})(t2/define-before-insert :model/Transform- [{:keys [source collection_id] :as transform}]+ [{:keys [source target collection_id] :as transform}](collection/check-collection-namespace :model/Transform collection_id)(when collection_id(collection/check-allowed-content :model/Transform collection_id))- (assoc transform :source_type (transforms.util/transform-source-type source)))+ (assoc transform+ :source_type (transforms.util/transform-source-type source)+ :target_db_id (or (:database target) (:target_db_id transform))))(t2/define-before-update :model/Transform- [{:keys [source] :as transform}]+ [{:keys [source target] :as transform}](when-let [new-collection (:collection_id (t2/changes transform))](collection/check-collection-namespace :model/Transform new-collection)(collection/check-allowed-content :model/Transform new-collection))- (if source- (assoc transform :source_type (transforms.util/transform-source-type source))- transform))+ (cond-> transform+ source (assoc :source_type (transforms.util/transform-source-type source))+ (:database target) (assoc :target_db_id (:database target))))(t2/define-after-select :model/Transform[{:keys [source] :as transform}]@@ -232,8 +234,8 @@(defmethod serdes/make-spec "Transform"[_model-name opts]- {:copy [:name :description :entity_id]- :skip [:dependency_analysis_version :source_type]+ {:copy [:name :description :entity_id]+ :skip [:dependency_analysis_version :source_type :target_db_id]:transform {:created_at (serdes/date):creator_id (serdes/fk :model/User):collection_id (serdes/fk :model/Collection)@@ -267,15 +269,6 @@(when query-text(subs query-text 0 (min (count query-text) search/max-searchable-value-length)))))-(defn- extract-transform-db-id- "Return the database ID from transform source; else nil."- [{:keys [source]}]- (let [parsed-source (transform-source-out source)]- (case (:type parsed-source)- :query (get-in parsed-source [:query :database])- :python (parsed-source :source-database)- nil)))-;;; ------------------------------------------------- Search ---------------------------------------------------(search.spec/define-spec "transform"@@ -289,8 +282,7 @@:view-count false:native-query {:fn maybe-extract-transform-query-text:fields [:source :source_type]}- :database-id {:fn extract-transform-db-id- :fields [:source]}}+ :database-id :target_db_id}:search-terms [:name :description]:render-terms {:transform-name :name:transform-id :id}})diff --git a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.cljindex 17ce1b442ef..4bccf38e3e8 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj@@ -20,7 +20,8 @@(defmethod transforms.i/target-db-id :query[transform]- (-> transform :source :query :database))+ (or (get-in transform [:target :database])+ (:target_db_id transform)))(mr/def ::transform-details[:mapdiff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/impl.cljindex 4f12c7cb12a..b427bd71fd0 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj@@ -5,7 +5,8 @@(defmethod transforms.i/target-db-id :python[transform]- (-> transform :target :database))+ (or (get-in transform [:target :database])+ (:target_db_id transform)))(defmethod transforms.i/source-db-id :python[transform]diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.cljindex 8d74c2baaf1..0a78e800c81 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj@@ -61,7 +61,8 @@:query query}:target {:type "table":schema schema- :name table-name}})+ :name table-name+ :database (mt/id)}})transform-id (:id response)crowberto-id (mt/user->id :crowberto)creator-id (t2/select-one-fn :creator_id :model/Transform transform-id)]@@ -88,7 +89,8 @@:query mbql-query}:target {:type "table":schema schema- :name table-name}})]+ :name table-name+ :database (mt/id)}})](is (= "mbql" (:source_type response))))))(testing "Native query transforms are detected as :native"@@ -100,7 +102,8 @@:query (lib/native-query (mt/metadata-provider) "SELECT 1")}:target {:type "table":schema schema- :name table-name}})]+ :name table-name+ :database (mt/id)}})](is (= "native" (:source_type response))))))(testing "Python transforms are detected as :python"@@ -132,7 +135,8 @@:query native-query}:target {:type "table":schema schema- :name table-name}})]+ :name table-name+ :database (mt/id)}})](is (= "native" (:source_type created)))(testing "Type automatically changes to mbql when updating to an MBQL query"@@ -155,7 +159,8 @@:query query}:target {:type "table":schema schema- :name "test_transform"}})]+ :name "test_transform"+ :database (mt/id)}})](is (= "error-premium-feature-not-available" (:status response)))))))(testing "Creating a query transform with :transforms feature succeeds"@@ -170,7 +175,8 @@:query query}:target {:type "table":schema schema- :name table-name}})]+ :name table-name+ :database (mt/id)}})](is (some? (:id response))))))))))(deftest update-transform-feature-flag-test@@ -186,7 +192,8 @@:query query}:target {:type "table":schema schema- :name table-name}}+ :name table-name+ :database (mt/id)}}created (mt/user-http-request :crowberto :post 200 "ee/transform" transform-payload)];; Now test update without feature flag(mt/with-premium-features #{}@@ -208,7 +215,8 @@:query query}:target {:type "table":schema schema- :name table-name}}+ :name table-name+ :database (mt/id)}}created (mt/user-http-request :crowberto :post 200 "ee/transform" transform-payload)];; Now test run without feature flag(mt/with-premium-features #{}@@ -231,7 +239,8 @@:query (make-query "Gadget")}:target {:type "table":schema (get-test-schema)- :name table-name}}+ :name table-name+ :database (mt/id)}}_ (mt/user-http-request :crowberto :post 200 "ee/transform" body)list-resp (mt/user-http-request :crowberto :get 200 "ee/transform")crowberto-id (mt/user->id :crowberto)]@@ -280,7 +289,8 @@:query (make-query "Gadget")}:target {:type "table":schema (get-test-schema)- :name table-name}}+ :name table-name+ :database (mt/id)}}resp (mt/user-http-request :crowberto :post 200 "ee/transform" body)get-resp (mt/user-http-request :crowberto :get 200 (format "ee/transform/%s" (:id resp)))crowberto-id (mt/user->id :crowberto)]@@ -296,7 +306,8 @@:name transform-name:target {:schema "public":name "orders_2"- :type "table"}})+ :type "table"+ :database (mt/id)}})(deftest get-transform-dependencies-test(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)@@ -331,14 +342,16 @@:query (make-query "Gadget")}:target {:type "table":schema (get-test-schema)- :name table-name}})+ :name table-name+ :database (mt/id)}})transform {:name "Gadget Products 2":description "Desc":source {:type "query":query query2}:target {:type "table":schema (get-test-schema)- :name table-name}}+ :name table-name+ :database (mt/id)}}put-resp (mt/user-http-request :crowberto :put 200 (format "ee/transform/%s" (:id resp))transform)crowberto-id (mt/user->id :crowberto)]@@ -360,7 +373,8 @@:query (make-query "Gadget")}:target {:type "table":schema (get-test-schema)- :name table1-name}}+ :name table1-name+ :database (mt/id)}}resp (mt/user-http-request :crowberto :post 200 "ee/transform"original)updated {:name "Doohickey Products"@@ -369,7 +383,8 @@:query query2}:target {:type "table":schema (get-test-schema)- :name table2-name}}]+ :name table2-name+ :database (mt/id)}}](is (=? (-> updated(m/dissoc-in [:source :query :lib/metadata]))(-> (mt/user-http-request :crowberto :put 200 (format "ee/transform/%s" (:id resp)) updated)@@ -387,7 +402,8 @@:query (make-query "Gadget")}:target {:type "table":schema (get-test-schema)- :name table-name}})]+ :name table-name+ :database (mt/id)}})](mt/user-http-request :crowberto :delete 204 (format "ee/transform/%s" (:id resp)))(mt/user-http-request :crowberto :get 404 (format "ee/transform/%s" (:id resp)))))))))@@ -402,7 +418,8 @@:query (make-query "Gadget")}:target {:type "table":schema (get-test-schema)- :name table-name}})]+ :name table-name+ :database (mt/id)}})](mt/user-http-request :crowberto :delete 204 (format "ee/transform/%s/table" (:id resp)))))))))(defn- test-run@@ -482,10 +499,12 @@(let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))](with-transform-cleanup! [{table1-name :name :as target1} {:type "table":schema schema- :name "gadget_products"}+ :name "gadget_products"+ :database (mt/id)}{table2-name :name :as target2} {:type "table":schema schema- :name "doohickey_products"}]+ :name "doohickey_products"+ :database (mt/id)}](let [query2 (make-query "Doohickey")original {:name "Gadget Products":source {:type "query"@@ -853,7 +872,7 @@(mt/user-http-request :crowberto :post 400 "ee/transform"{:name "Gadget Products":source {:type "query" :query query}- :target {:type "table" :schema schema :name table-name}}))))))))))+ :target {:type "table" :schema schema :name table-name :database (mt/id)}}))))))))))(deftest update-transform-with-routing-fails-test(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)@@ -869,7 +888,7 @@:user_attribute "db_name"}:model/Transform transform {:name "Gadget Products":source {:type "query" :query query}- :target {:type "table" :schema schema :name table-name}}]+ :target {:type "table" :schema schema :name table-name :database (mt/id)}}](is (= "Transforms are not supported on databases with DB routing enabled."(mt/user-http-request :crowberto :put 400 (format "ee/transform/%s" (:id transform))(assoc transform :name "Gadget Products 2")))))))))))@@ -901,7 +920,8 @@:query (make-query "Gadget")}:target {:type "table":schema (get-test-schema)- :name table-name}}+ :name table-name+ :database (mt/id)}}transform-id (test-transform-revisions :post "ee/transform" gadget-req 1)widget-req {:name "Widget Products":description "The widget products"@@ -910,7 +930,8 @@:tag_ids [4]:target {:type "table":schema (get-test-schema)- :name table-name}}]+ :name table-name+ :database (mt/id)}}](test-transform-revisions :put (str "ee/transform/" transform-id) widget-req 2)))))))(defmethod driver/database-supports? [::driver/driver ::extract-columns-from-query]diff --git a/enterprise/backend/test/metabase_enterprise/transforms/models_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/models_test.cljindex 3c501aa8660..35368a13add 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/models_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/models_test.clj@@ -185,7 +185,7 @@:target {:type "table":schema "public":name "test_table_with_schema"- :db_id db-id}}+ :database db-id}}:model/Transform transform2 {:name "Transform with NULL schema":source {:type "query":query {:database db-id@@ -195,7 +195,7 @@:target {:type "table":schema nil:name "test_table_null_schema"- :db_id db-id}}]+ :database db-id}}](testing "Hydrates table with non-NULL schema"(let [hydrated (t2/hydrate transform1 :table-with-db-and-fields)]@@ -246,7 +246,7 @@:target {:type "table":schema "public":name "test_table"- :db_id db-id}}+ :database db-id}}:model/User {user2-id :id} user2-data:model/Transform transform2 {:name "Second transform":creator_id user2-id@@ -258,7 +258,7 @@:target {:type "table":schema "public":name "test_table2"- :db_id db-id}}]+ :database db-id}}](testing "Hydrates creator with user details"(is (=? (assoc user1-data :id user1-id)(:creator (t2/hydrate transform1 :creator)))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.cljindex cbe370e31ad..1dd0f3e0995 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj@@ -9,16 +9,18 @@[toucan2.core :as t2]))(defn- make-transform [query & [name schema]]- (let [name (or name (mt/random-name))+ (let [name (or name (mt/random-name))default-schema (when (get-method driver.sql/default-schema driver/*driver*)(driver.sql/default-schema driver/*driver*))- schema (or schema default-schema "public")]- {:source {:type :query- :query query}- :name (str "transform_" name)- :target {:schema schema- :name name- :type :table}}))+ schema (or schema default-schema "public")]+ {:source {:type :query+ :query query}+ :name (str "transform_" name)+ :target {:schema schema+ :name name+ :type :table}+ :target_db_id (or (:database query)+ (mt/id))}))(defn- make-python-transform [source-tables & [name schema target-db]](let [name (or name (mt/random-name))@@ -33,7 +35,8 @@:target {:database target-db:schema schema:name name- :type "table"}}))+ :type "table"}+ :target_db_id target-db}))(defn- transform-deps-for-db [transform](mt/with-metadata-provider (mt/id)diff --git a/resources/migrations/058_update_migrations.yaml b/resources/migrations/058_update_migrations.yamlindex 1ce7b8282e9..0c7e1067e4d 100644--- a/resources/migrations/058_update_migrations.yaml+++ b/resources/migrations/058_update_migrations.yaml@@ -1235,7 +1235,81 @@ databaseChangeLog:path: instance_analytics_views/tables/v1/h2-tables.sqlrelativeToChangelogFile: true-# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<+ - changeSet:+ id: v58.2025-12-15T12:16:15+ author: piranha+ comment: Add target_db_id column to transform table+ preConditions:+ - onFail: MARK_RAN+ - not:+ - columnExists:+ tableName: transform+ columnName: target_db_id+ changes:+ - addColumn:+ tableName: transform+ columns:+ - column:+ name: target_db_id+ type: int+ remarks: Target database ID for the transform output+ constraints:+ nullable: true+ rollback:+ - dropColumn:+ tableName: transform+ columnName: target_db_id++ - changeSet:+ id: v58.2025-12-15T12:16:16+ author: piranha+ comment: Backfill target_db_id column for existing transforms+ changes:+ - customChange:+ class: "metabase.app_db.custom_migrations.BackfillTransformTargetDbId"++ - changeSet:+ id: v58.2025-12-15T12:16:17+ author: piranha+ comment: Make transform.target_db_id column NOT NULL+ changes:+ - addNotNullConstraint:+ tableName: transform+ columnName: target_db_id+ columnDataType: int+ rollback:+ - dropNotNullConstraint:+ tableName: transform+ columnName: target_db_id+ columnDataType: int++ - changeSet:+ id: v58.2025-12-15T12:16:18+ author: piranha+ comment: Add index on transform.target_db_id+ changes:+ - createIndex:+ tableName: transform+ indexName: idx_transform_target_db_id+ columns:+ - column:+ name: target_db_id+ rollback:+ - dropIndex:+ tableName: transform+ indexName: idx_transform_target_db_id++ - changeSet:+ id: v58.2025-12-15T12:16:19+ author: piranha+ comment: Add foreign key constraint for transform.target_db_id+ changes:+ - addForeignKeyConstraint:+ baseTableName: transform+ baseColumnNames: target_db_id+ referencedTableName: metabase_database+ referencedColumnNames: id+ constraintName: fk_transform_ref_target_db_id#########################################################################################################################diff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.cljindex bdaf6cd2944..21d0812549f 100644--- a/src/metabase/app_db/custom_migrations.clj+++ b/src/metabase/app_db/custom_migrations.clj@@ -1823,3 +1823,13 @@(define-migration MoveExistingAtSymbolUserAttributes(reserve-at-symbol-user-attributes/migrate!))++(define-migration BackfillTransformTargetDbId+ (let [update! (fn [{:keys [id target]}]+ (let [target-map (json/decode target)+ db-id (get target-map "database")]+ (when db-id+ (t2/update! :transform id {:target_db_id db-id}))))]+ (run! update! (t2/reducible-query {:select [:id :target]+ :from [:transform]+ :where [:= :target_db_id nil]}))))diff --git a/test/metabase/app_db/custom_migrations_test.clj b/test/metabase/app_db/custom_migrations_test.cljindex 6b0dc5c480e..ab2a32d962f 100644--- a/test/metabase/app_db/custom_migrations_test.clj+++ b/test/metabase/app_db/custom_migrations_test.clj@@ -2725,3 +2725,37 @@(json/decode (t2/select-one-fn :login_attributes :core_user :id user-id))))(is (= {"_@foo" "bang"}(json/decode (t2/select-one-fn :login_attributes :core_user :id other-user-id))))))))++(deftest backfill-transform-target-database-id-test+ (testing "Migration v58.2025-12-15T12:16:16: backfill target_db_id for existing transforms"+ (impl/test-migrations ["v58.2025-12-15T12:16:16"] [migrate!]+ (let [user-id (first (t2/insert-returning-pks! :core_user {:first_name "Test"+ :last_name "User"+ :email "[email protected]"+ :password "password"+ :date_joined :%now}))+ db-id (first (t2/insert-returning-pks! :metabase_database {:name "test-db"+ :engine "h2"+ :details "{}"+ :created_at :%now+ :updated_at :%now}))+ xf-id (first (t2/insert-returning-pks! :transform+ {:name "Query transform"+ :source (json/encode {:type "query"+ :query {:database db-id+ :type "native"+ :native {:query "SELECT 1"}}})+ :source_type "native"+ :target (json/encode {:type "table"+ :schema "public"+ :name "output_table"+ :database db-id})+ :creator_id user-id+ :created_at :%now+ :updated_at :%now}))]+ (testing "before migration, target_db_id is nil"+ (is (nil? (:target_db_id (t2/select-one :transform :id xf-id)))))+ (migrate!)+ (testing "after migration, target_db_id is populated"+ (is (= db-id (:target_db_id (t2/select-one :transform :id xf-id)))+ "Query transform target_db_id should equal target.database"))))))diff --git a/test/metabase/test/util.clj b/test/metabase/test/util.cljindex 6ffb00cb86e..5184ebb6661 100644--- a/test/metabase/test/util.clj+++ b/test/metabase/test/util.clj@@ -333,7 +333,9 @@:source {:type "query":query (lib/native-query (data/metadata-provider) "SELECT 1 as num")}:target {:type "table"- :name (str "test_table_" (u/generate-nano-id))}})+ :name (str "test_table_" (u/generate-nano-id))+ :database (data/id)}+ :target_db_id (data/id)}):model/TransformJob(fn [_]