The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Support transform upserts in Workspace (#69283)
Chris Truterb1212ac2635c
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj b/enterprise/backend/src/metabase_enterprise/workspaces/api.cljindex dc0c181ead9..c444563ac68 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/api.clj@@ -788,44 +788,60 @@:schema isolated_schema:name isolated_table})))-(api.macros/defendpoint :post "/:ws-id/transform"- :- WorkspaceTransform- "Add another transform to the Changeset. This could be a fork of an existing global transform, or something new."- {:access :workspace}- [{:keys [ws-id]} :- [:map [:ws-id ::ws.t/appdb-id]]- _query-params- body :- [:map #_{:closed true}- [:name :string]- [:description {:optional true} [:maybe :string]]- [:source ::transform-source]- ;; Not sure why this schema is giving trouble- #_[:target ::transform-target]]]+(defn- create-workspace-transform!+ "Shared logic for creating a new workspace transform.+ Validates the transform, checks constraints, and inserts it into the changeset.+ Returns the created transform with isolated target info attached.++ Options:+ - `:ref-id` - Optional custom ref_id for the transform (defaults to auto-generated)"+ [ws-id body & {:keys [ref-id]}]+ (doseq [field [:name :source :target]]+ (api/check-400 (get body field) (str (name field) " is required when creating a new transform")))+ ;; Check premium feature requirements(api/check (transforms.util/check-feature-enabled body)[402 (deferred-tru "Premium features required for this transform type are not enabled.")])(t2/with-transaction [_tx](let [workspace (u/prog1 (api/check-404 (t2/select-one :model/Workspace :id ws-id))- (api/check-400 (not= :archived (:base_status <>)) "Cannot create transforms in an archived workspace"))- ;; TODO (Chris 2026-02-02) -- We use 400 here, but 403 in the validation route. Consistency would be nice.- _ (api/check-400 (not (internal-target-conflict? ws-id (:target body)))- (deferred-tru "Another transform in this workspace already targets that table"))+ (api/check-400 (not= :archived (:base_status <>))+ "Cannot create transforms in an archived workspace"))global-id (:global_id body (:id body));; Verify transform source is allowed in workspaces (not MBQL, no card references, etc.)_ (let [source-type (transforms/transform-source-type (:source body))reason (checkout-disabled-reason {:source_type source-type :source (:source body)})](api/check-400 (nil? reason)(case reason- "mbql" (deferred-tru "MBQL transforms cannot be added to workspaces.")- "card-reference" (deferred-tru "Transforms that reference other questions cannot be added to workspaces.")+ "mbql"+ (deferred-tru "MBQL transforms cannot be added to workspaces.")+ "card-reference"+ (deferred-tru "Transforms referencing questions cannot be added to workspaces.")(deferred-tru "This transform cannot be added to a workspace: {0}." reason))));; For uninitialized workspaces, preserve the target database from the request body- ;; (add-to-changeset! will reinitialize the workspace with it if different from provisional);; For initialized workspaces, ensure the target database matches the workspace databasebody (-> body (dissoc :global_id)(cond-> (not= :uninitialized (:db_status workspace))(update :target assoc :database (:database_id workspace))))- transform (ws.common/add-to-changeset! api/*current-user-id* workspace :transform global-id body)]+ ;; Check for internal target conflict AFTER adding database to target+ _ (api/check-400 (not (internal-target-conflict? ws-id (:target body)))+ (deferred-tru "Another transform in this workspace already targets that table"))+ transform (ws.common/add-to-changeset! api/*current-user-id* workspace :transform global-id body+ :ref-id ref-id)](attach-isolated-target (select-malli-keys WorkspaceTransform workspace-transform-alias transform)))))+(api.macros/defendpoint :post "/:ws-id/transform"+ :- WorkspaceTransform+ "Add another transform to the Changeset. This could be a fork of an existing global transform, or something new."+ {:access :workspace}+ [{:keys [ws-id]} :- [:map [:ws-id ::ws.t/appdb-id]]+ _query-params+ body :- [:map #_{:closed true}+ [:name :string]+ [:description {:optional true} [:maybe :string]]+ [:source ::transform-source]+ ;; Not sure why this schema is giving trouble+ #_[:target ::transform-target]]]+ (create-workspace-transform! ws-id body))+;; TODO (Sanya 2025-12-12) -- Confirm precisely which fields are needed by the FE(def ^:private WorkspaceTransformListing"Schema for a transform in a workspace"@@ -867,7 +883,9 @@(fetch-ws-transform ws-id tx-id))(api.macros/defendpoint :put "/:ws-id/transform/:tx-id" :- WorkspaceTransform- "Update a transform in a workspace."+ "Update or create a transform in a workspace.+ If the transform exists, updates it. If it doesn't exist, creates a new transform with the provided ref_id.+ For creation, name, source, and target are required."{:access :workspace}[{:keys [ws-id tx-id]} :- [:map [:ws-id ::ws.t/appdb-id] [:tx-id ::ws.t/ref-id]]_query-params@@ -876,23 +894,35 @@[:description {:optional true} [:maybe :string]][:source {:optional true} ::transform-source][:target {:optional true} ::transform-target]]]- (t2/with-transaction [_tx]- (let [existing (api/check-404 (t2/select-one :model/WorkspaceTransform :workspace_id ws-id :ref_id tx-id))- ;; Merge only :database and :schema from existing target to preserve them when not explicitly provided.- ;; Other fields are NOT merged, allowing them to be removed by omitting from the request.- base (select-keys (:target existing) [:database :schema])- merged-body (cond-> body- (:target body) (update :target #(merge base %)))- source-or-target-changed? (or (:source body) (:target body))]- (t2/update! :model/WorkspaceTransform {:workspace_id ws-id :ref_id tx-id} merged-body)- ;; If source or target changed, increment versions for re-analysis- (when source-or-target-changed?- (ws.impl/increment-analysis-version! ws-id tx-id)- ;; It's unfortunate that we are using an extra SQL statement for this, rather than doing it in the earlier- ;; statement that set the [[body]]. Combining them would be tricky, as we still need the model transforms to- ;; serialize the fields in the body, but the increment requires using an SQL expression.- (ws.impl/increment-graph-version! ws-id)))- (fetch-ws-transform ws-id tx-id)))+ ;; We use explicit check-then-branch rather than app-db/update-or-insert! because:+ ;; 1. WorkspaceTransform has a compound primary key which that helper doesn't support+ ;; 2. Update and insert have very different logic (update merges + increments versions,+ ;; insert requires full validation, workspace initialization, status transitions)+ (let [existing (t2/select-one :model/WorkspaceTransform :ref_id tx-id :workspace_id ws-id)]+ (if existing+ ;; UPDATE path+ (t2/with-transaction [_tx]+ (let [;; Merge only :database and :schema from existing target to preserve them when not explicitly provided.+ ;; Other fields are NOT merged, allowing them to be removed by omitting from the request.+ base (select-keys (:target existing) [:database :schema])+ merged-body (cond-> body+ (:target body) (update :target #(merge base %)))+ source-or-target-changed? (or (:source body) (:target body))]+ ;; If target is changing, check for conflicts with other transforms (excluding this one)+ (when (:target body)+ (api/check-400 (not (internal-target-conflict? ws-id (:target merged-body) tx-id))+ (deferred-tru "Another transform in this workspace already targets that table")))+ (t2/update! :model/WorkspaceTransform {:workspace_id ws-id :ref_id tx-id} merged-body)+ ;; If source or target changed, increment versions for re-analysis+ (when source-or-target-changed?+ (ws.impl/increment-analysis-version! ws-id tx-id)+ ;; It's unfortunate that we are using an extra SQL statement for this, rather than doing it in the earlier+ ;; statement that set the [[body]]. Combining them would be tricky, as we still need the model transforms to+ ;; serialize the fields in the body, but the increment requires using an SQL expression.+ (ws.impl/increment-graph-version! ws-id)))+ (fetch-ws-transform ws-id tx-id))+ ;; INSERT path (upsert creates new transform)+ (create-workspace-transform! ws-id body :ref-id tx-id))))(api.macros/defendpoint :post "/:ws-id/transform/:tx-id/archive" :- :nil"Mark the given transform to be archived when the workspace is merged.diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/common.clj b/enterprise/backend/src/metabase_enterprise/workspaces/common.cljindex 3a953c8d8aa..9a7d4eddeb2 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/common.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/common.clj@@ -149,8 +149,11 @@(defn add-to-changeset!"Add the given transform to the workspace changeset.If workspace db_status is uninitialized, initializes it with the transform's target database.- If workspace base_status is empty, transitions it to active."- [creator-id workspace entity-type global-id body]+ If workspace base_status is empty, transitions it to active.++ Options:+ - `:ref-id` - Optional custom ref_id (e.g., for upsert operations)."+ [creator-id workspace entity-type global-id body & {:keys [ref-id]}](ws.u/assert-transform! entity-type);; Initialize workspace if uninitialized (outside transaction so async task can see committed data)(let [workspace (if (= :uninitialized (:db_status workspace))@@ -164,7 +167,7 @@body (assoc-in body [:target :database] workspace-db-id)transform (ws.u/insert-returning-ws-tx!(assoc (select-keys body [:name :description :source :target])- :ref_id (ws.u/generate-ref-id)+ :ref_id (or ref-id (ws.u/generate-ref-id)):creator_id creator-id:global_id global-id:workspace_id workspace-id))]diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.cljindex f79f5532a37..fd3c2741ce4 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj@@ -1050,12 +1050,195 @@{:name "Updated Name":description "Updated description"})))(is (= "Updated Name" (t2/select-one-fn :name :model/WorkspaceTransform :workspace_id (:id workspace1) :ref_id (:ref_id transform)))))- (testing "returns 404 if transform not in workspace"- (is (= "Not found."- (mt/user-http-request :crowberto :put 404+ (testing "returns 400 if transform not in workspace (upsert requires all fields)"+ ;; With upsert behavior, PUT to non-existent transform attempts to create,+ ;; so missing required fields returns 400 instead of 404+ (is (= "source is required when creating a new transform"+ (mt/user-http-request :crowberto :put 400(ws-url (:id workspace2) "/transform" (:ref_id transform)){:name "Should Fail"}))))))))+(deftest ^:synchronized put-workspace-transform-target-conflict-test+ (testing "PUT /api/ee/workspace/:id/transform/:txid returns 400 when updating target to conflict"+ (let [target-schema (t2/select-one-fn :schema :model/Table (mt/id :orders))]+ (ws.tu/with-workspaces! [workspace {:name "Workspace 1" :database_id (mt/id)}]+ ;; Create two transforms with different targets via POST API+ (let [tx1 (mt/user-http-request :crowberto :post 200 (ws-url (:id workspace) "/transform")+ {:name "Transform 1"+ :source {:type "query"+ :query (->native (mt/mbql-query orders {:aggregation [[:count]]}))}+ :target {:type "table"+ :schema target-schema+ :name "target_table_1"}})+ _tx2 (mt/user-http-request :crowberto :post 200 (ws-url (:id workspace) "/transform")+ {:name "Transform 2"+ :source {:type "query"+ :query (->native (mt/mbql-query orders {:aggregation [[:count]]}))}+ :target {:type "table"+ :schema target-schema+ :name "target_table_2"}})]+ (testing "updating target to match another transform's target returns 400"+ (is (= "Another transform in this workspace already targets that table"+ (mt/user-http-request :crowberto :put 400+ (ws-url (:id workspace) "/transform" (:ref_id tx1))+ {:target {:type "table"+ :schema target-schema+ :name "target_table_2"}}))))+ (testing "updating target to a unique table succeeds"+ (is (=? {:ref_id (:ref_id tx1)}+ (mt/user-http-request :crowberto :put 200+ (ws-url (:id workspace) "/transform" (:ref_id tx1))+ {:target {:type "table"+ :schema target-schema+ :name "target_table_3"}}))))+ (testing "updating non-target fields doesn't trigger conflict check"+ (is (=? {:ref_id (:ref_id tx1)+ :name "Renamed Transform 1"}+ (mt/user-http-request :crowberto :put 200+ (ws-url (:id workspace) "/transform" (:ref_id tx1))+ {:name "Renamed Transform 1"})))))))))++(deftest ^:synchronized upsert-workspace-transform-test+ (testing "PUT /api/ee/workspace/:id/transform/:txid (upsert behavior)"+ (let [target-schema (t2/select-one-fn :schema :model/Table (mt/id :orders))]+ (ws.tu/with-workspaces! [workspace {:name "Workspace 1" :database_id (mt/id)}]+ (let [new-ref-id (ws.u/generate-ref-id)]+ (testing "creates new transform when ref_id doesn't exist"+ (let [result (mt/user-http-request :crowberto :put 200+ (ws-url (:id workspace) "/transform" new-ref-id)+ {:name "New Transform via Upsert"+ :source {:type "query"+ :query (->native (mt/mbql-query orders {:aggregation [[:count]]}))}+ :target {:type "table"+ :schema target-schema+ :name "upsert_test_table"}})]+ (is (=? {:ref_id new-ref-id+ :name "New Transform via Upsert"+ :source {:type "query"}}+ result))+ (is (t2/exists? :model/WorkspaceTransform :workspace_id (:id workspace) :ref_id new-ref-id))))++ (testing "updates existing transform with same ref_id"+ (let [result (mt/user-http-request :crowberto :put 200+ (ws-url (:id workspace) "/transform" new-ref-id)+ {:name "Updated Transform Name"})]+ (is (=? {:ref_id new-ref-id+ :name "Updated Transform Name"}+ result))+ (is (= "Updated Transform Name"+ (t2/select-one-fn :name :model/WorkspaceTransform :workspace_id (:id workspace) :ref_id new-ref-id))))))))))++(deftest upsert-transform-missing-required-fields-test+ (testing "PUT /api/ee/workspace/:id/transform/:txid returns 400 when creating without required fields"+ (ws.tu/with-workspaces! [workspace {:name "Workspace 1" :database_id (mt/id)}]+ (let [target-schema (t2/select-one-fn :schema :model/Table (mt/id :orders))]+ (testing "missing name returns 400"+ (let [ref-id (ws.u/generate-ref-id)]+ (is (= "name is required when creating a new transform"+ (mt/user-http-request :crowberto :put 400+ (ws-url (:id workspace) "/transform" ref-id)+ {:source {:type "query"+ :query (->native (mt/mbql-query orders {:aggregation [[:count]]}))}+ :target {:type "table"+ :schema target-schema+ :name "missing_name_test"}})))))+ (testing "missing source returns 400"+ (let [ref-id (ws.u/generate-ref-id)]+ (is (= "source is required when creating a new transform"+ (mt/user-http-request :crowberto :put 400+ (ws-url (:id workspace) "/transform" ref-id)+ {:name "Test Transform"+ :target {:type "table"+ :schema target-schema+ :name "missing_source_test"}})))))+ (testing "missing target returns 400"+ (let [ref-id (ws.u/generate-ref-id)]+ (is (= "target is required when creating a new transform"+ (mt/user-http-request :crowberto :put 400+ (ws-url (:id workspace) "/transform" ref-id)+ {:name "Test Transform"+ :source {:type "query"+ :query (->native (mt/mbql-query orders {:aggregation [[:count]]}))}})))))))))++(deftest upsert-transform-archived-workspace-test+ (testing "PUT /api/ee/workspace/:id/transform/:txid returns 400 for archived workspace"+ (ws.tu/with-workspaces! [workspace {:name "Workspace 1" :database_id (mt/id)}]+ ;; Archive the workspace+ (t2/update! :model/Workspace (:id workspace) {:base_status :archived})+ (let [new-ref-id (ws.u/generate-ref-id)+ target-schema (t2/select-one-fn :schema :model/Table (mt/id :orders))]+ (is (= "Cannot create transforms in an archived workspace"+ (mt/user-http-request :crowberto :put 400+ (ws-url (:id workspace) "/transform" new-ref-id)+ {:name "New Transform"+ :source {:type "query"+ :query (->native (mt/mbql-query orders {:aggregation [[:count]]}))}+ :target {:type "table"+ :schema target-schema+ :name "archived_ws_test"}})))))))++(deftest ^:synchronized upsert-transform-internal-target-conflict-test+ (testing "PUT /api/ee/workspace/:id/transform/:txid returns 400 for internal target conflict"+ (let [target-schema (t2/select-one-fn :schema :model/Table (mt/id :orders))]+ (ws.tu/with-workspaces! [workspace {:name "Workspace 1" :database_id (mt/id)}]+ ;; Create first transform targeting a specific table via POST API+ (mt/user-http-request :crowberto :post 200 (ws-url (:id workspace) "/transform")+ {:name "Existing Transform"+ :source {:type "query"+ :query (->native (mt/mbql-query orders {:aggregation [[:count]]}))}+ :target {:type "table"+ :schema target-schema+ :name "conflict_target_table"}})+ ;; Try to upsert a new transform with the same target+ (let [new-ref-id (ws.u/generate-ref-id)]+ (is (= "Another transform in this workspace already targets that table"+ (mt/user-http-request :crowberto :put 400+ (ws-url (:id workspace) "/transform" new-ref-id)+ {:name "Conflicting Transform"+ :source {:type "query"+ :query (->native (mt/mbql-query orders {:aggregation [[:count]]}))}+ :target {:type "table"+ :schema target-schema+ :name "conflict_target_table"}}))))))))++(deftest ^:synchronized upsert-transform-initializes-workspace-test+ (testing "PUT /api/ee/workspace/:id/transform/:txid initializes uninitialized workspace"+ (let [target-schema (t2/select-one-fn :schema :model/Table (mt/id :orders))]+ (ws.tu/with-workspaces! [workspace {:name "Uninitialized WS"}]+ ;; Verify workspace starts uninitialized+ (is (= :uninitialized (:db_status (t2/select-one :model/Workspace :id (:id workspace)))))+ (let [new-ref-id (ws.u/generate-ref-id)]+ (mt/user-http-request :crowberto :put 200+ (ws-url (:id workspace) "/transform" new-ref-id)+ {:name "First Transform"+ :source {:type "query"+ :query (->native (mt/mbql-query orders {:aggregation [[:count]]}))}+ :target {:type "table"+ :database (mt/id)+ :schema target-schema+ :name "init_ws_test"}})+ ;; Verify workspace is now initialized+ (let [ws (t2/select-one :model/Workspace :id (:id workspace))]+ (is (not= :uninitialized (:db_status ws)))))))))++(deftest ^:synchronized upsert-transform-transitions-base-status-test+ (testing "PUT /api/ee/workspace/:id/transform/:txid transitions base_status from :empty to :active"+ (let [target-schema (t2/select-one-fn :schema :model/Table (mt/id :orders))]+ (ws.tu/with-workspaces! [workspace {:name "Empty WS" :database_id (mt/id)}]+ ;; Verify workspace starts with :empty base_status+ (is (= :empty (:base_status (t2/select-one :model/Workspace :id (:id workspace)))))+ (let [new-ref-id (ws.u/generate-ref-id)]+ (mt/user-http-request :crowberto :put 200+ (ws-url (:id workspace) "/transform" new-ref-id)+ {:name "First Transform"+ :source {:type "query"+ :query (->native (mt/mbql-query orders {:aggregation [[:count]]}))}+ :target {:type "table"+ :schema target-schema+ :name "status_transition_test"}})+ ;; Verify base_status transitioned to :active+ (is (= :active (:base_status (t2/select-one :model/Workspace :id (:id workspace))))))))))+(deftest delete-workspace-transform-test(testing "DELETE /api/ee/workspace/:id/transform/:txid"(ws.tu/with-workspaces! [workspace1 {:name "Workspace 1"}@@ -1995,7 +2178,7 @@:schema "public":name "card_ref_blocked_test"}}](ws.tu/with-workspaces! [ws {:name "Test Workspace" :database_id (mt/id)}]- (is (= "Transforms that reference other questions cannot be added to workspaces."+ (is (= "Transforms referencing questions cannot be added to workspaces."(mt/user-http-request :crowberto :post 400 (ws-url (:id ws) "/transform")(merge {:global_id (:id tx)}(select-keys tx [:name :source :target])))))))))