The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Create table entries for transform targets on save (#70552)
Chris Truter7ecba8f6d49b
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex df5c6b0fea3..6126b70269d 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1676,7 +1676,8 @@lib-bequery-processorsync- util}+ util+ warehouse-schema}:friends #{transformsenterprise/transforms-pythonenterprise/workspaces}}@@ -1847,7 +1848,8 @@remote-syncsearchutil- warehouses}}+ warehouses+ workspaces}}warehouse-schema-rest{:team "UX West"@@ -1921,6 +1923,11 @@warehousesenterprise/advanced-permissions}}+ workspaces+ {:team "Metabot"+ :api #{metabase.workspaces.core}+ :uses #{util}}+xrays{:team "Querying":api #{metabase.xrays.api@@ -2642,6 +2649,7 @@transformstransforms-baseutil+ warehouse-schemaenterprise/transforms-python}}enterprise/writable-connectiondiff --git a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj b/enterprise/backend/src/metabase_enterprise/workspaces/api.cljindex 53b3fc1007c..89f98883867 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/api.clj@@ -49,7 +49,8 @@[:database {:optional true} ::ws.t/appdb-id][:type [:enum "table"]][:schema {:optional true} [:maybe [:string {:min 1}]]]- [:name [:string {:min 1}]]])+ [:name [:string {:min 1}]]+ [:table_id {:optional true} [:maybe ::ws.t/appdb-id]]])(def ^:private InputTable[:mapdiff --git a/enterprise/backend/src/metabase_enterprise/workspaces/api/common.clj b/enterprise/backend/src/metabase_enterprise/workspaces/api/common.cljindex ed029872d4c..a7ade2153c4 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/api/common.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/api/common.clj@@ -26,6 +26,7 @@[metabase.util :as u][metabase.util.i18n :refer [deferred-tru]][metabase.util.malli.registry :as mr]+ [metabase.warehouse-schema.models.table :as table][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -79,7 +80,7 @@;;; ------------------------------------------------- Utilities -------------------------------------------------------(defn flag-enabled?+(defn- flag-enabled?"Coerce a flag parameter (true, false, or 1) to boolean."[v](boolean (#{1 true} v)))@@ -230,6 +231,10 @@;; Check for internal target conflict AFTER adding database to target_ (api/check-400 (not (internal-target-conflict? ws-id (:target body)))(deferred-tru "Another transform in this workspace already targets that table"))+ ;; Eagerly create global target table row so it's immediately visible+ {:keys [database schema name]} (:target body)+ _ (when (and database name)+ (table/upsert-transform-target-table! database schema name))transform (ws.common/add-to-changeset! api/*current-user-id* workspace :transform global-id body:ref-id ref-id)](-> (select-malli-keys WorkspaceTransform workspace-transform-alias transform)@@ -406,14 +411,18 @@;; Build a map of [d s t] => id for every table that has been synced since the output row was written.fallback-map (merge(ws.impl/table-ids-fallbacks :global_schema :global_table :global_table_id all-outputs)- (ws.impl/table-ids-fallbacks :isolated_schema :isolated_table :isolated_table_id all-outputs))]+ (ws.impl/table-ids-fallbacks :isolated_schema :isolated_table :isolated_table_id all-outputs))+ table-ids (into (into #{} (keep :isolated_table_id) all-outputs) (vals fallback-map))+ active-table-ids (t2/select-fn-set :id [:model/Table :id] :id [:in table-ids] :active true)]{:inputs (sort-by (juxt :db_id :schema :table) inputs):outputs (sort-by(juxt :db_id (comp (juxt :schema :table) :global))(concat;; Workspace transform outputs(for [{:keys [ref_id db_id global_schema global_table global_table_id- isolated_schema isolated_table isolated_table_id]} outputs]+ isolated_schema isolated_table isolated_table_id]} outputs+ :let [isolated_table_id (or isolated_table_id+ (get fallback-map [db_id isolated_schema isolated_table]))]]{:db_id db_id:global {:transform_id nil:schema global_schema@@ -422,10 +431,14 @@:isolated {:transform_id ref_id:schema isolated_schema:table isolated_table- :table_id (or isolated_table_id (get fallback-map [db_id isolated_schema isolated_table]))}})+ :table_id isolated_table_id+ :active (contains? active-table-ids isolated_table_id)}})+;; External transform outputs(for [{:keys [transform_id db_id global_schema global_table global_table_id- isolated_schema isolated_table isolated_table_id]} external-outputs]+ isolated_schema isolated_table isolated_table_id]} external-outputs+ :let [isolated_table_id (or isolated_table_id+ (get fallback-map [db_id isolated_schema isolated_table]))]]{:db_id db_id:global {:transform_id transform_id:schema global_schema@@ -434,7 +447,8 @@:isolated {:transform_id transform_id:schema isolated_schema:table isolated_table- :table_id (or isolated_table_id (get fallback-map [db_id isolated_schema isolated_table]))}})))}))+ :table_id isolated_table_id+ :active (contains? active-table-ids isolated_table_id)}})))}))(defn get-workspace-log"Handler body for GET /:ws-id/log."@@ -593,6 +607,7 @@(let [workspace (t2/select-one :model/Workspace :id ws-id)_ (api/check-404 workspace)_ (api/check-400 (not= :archived (:base_status workspace)) "Cannot execute archived workspace")+ _ (check-transforms-enabled! (:database_id workspace))graph (ws.impl/get-or-calculate-graph! workspace)](ws.impl/execute-workspace! workspace graph {:stale-only? (flag-enabled? stale_only)})))@@ -673,7 +688,11 @@;; If target is changing, check for conflicts with other transforms (excluding this one)(when (:target body)(api/check-400 (not (internal-target-conflict? ws-id (:target merged-body) tx-id))- (deferred-tru "Another transform in this workspace already targets that table")))+ (deferred-tru "Another transform in this workspace already targets that table"))+ ;; Eagerly create global target table row so it's immediately visible+ (let [{:keys [database schema name]} (:target merged-body)]+ (when (and database name)+ (table/upsert-transform-target-table! database schema name))))(t2/update! :model/WorkspaceTransform {:workspace_id ws-id :ref_id tx-id} merged-body);; If source or target changed, increment versions for re-analysis(when source-or-target-changed?diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj b/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.cljindex 5f4f6a732dc..68fc1720471 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj@@ -55,9 +55,6 @@[metabase.app-db.core :as app-db][metabase.driver :as driver][metabase.driver.sql :as driver.sql]- ;; TODO (Chris 2025-12-17) -- I solemnly declare that we will clean up this coupling nightmare for table normalization- ^{:clj-kondo/ignore [:metabase/modules]}- [metabase.driver.sql.normalize :as sql.normalize][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.util :as u]@@ -65,6 +62,7 @@[metabase.util.malli :as mu][metabase.util.malli.registry :as mr][metabase.util.performance :as perf]+ [metabase.warehouse-schema.models.table :as table][toucan2.core :as t2]));;; ---------------------------------------- Schemas ----------------------------------------@@ -226,21 +224,12 @@Stores both global (original) and isolated (workspace-specific) table identifiers.With epochal versioning, we always insert new rows - cleanup of old versions happens separately.Silently ignores constraint violations from concurrent inserts."- [workspace-id ref-id isolated-schema {:keys [db_id schema table]} normalize-sql transform-version]+ [workspace-id ref-id isolated-schema {:keys [db_id schema table]} transform-version]+ ;; 5 queries here: 2 × upsert-transform-target-table! (2 queries each) + 1 insert.+ ;; Worst case 9 on concurrent upsert conflicts.(let [isolated-table (ws.u/isolated-table-name schema table)- qry-table-id (fn [s t]- (or (t2/select-one-fn :id [:model/Table :id]- :db_id db_id- :schema (normalize-sql s)- :name (normalize-sql t))- ;; Turns out transforms don't normalize the metadata they create- ;; TODO (Chris 2026-01-26) -- This is getting really tangled and expensive, revisit- (t2/select-one-fn :id [:model/Table :id]- :db_id db_id- :schema s- :name t)))- global-table-id (qry-table-id schema table)- isolated-table-id (qry-table-id isolated-schema isolated-table)]+ global-table-id (table/upsert-transform-target-table! db_id schema table)+ isolated-table-id (table/upsert-transform-target-table! db_id isolated-schema isolated-table)](ws.u/ignore-constraint-violation(t2/insert! :model/WorkspaceOutput{:workspace_id workspace-id@@ -264,19 +253,22 @@"Ensure a workspace_input row exists for the given table coordinate.Returns the workspace_input id (existing or newly created)."[workspace-id {:keys [db_id schema table table_id]}]- (app-db/update-or-insert!- :model/WorkspaceInput- {:workspace_id workspace-id- :db_id db_id- :schema schema- :table table}- (fn [existing]- (cond-> {:workspace_id workspace-id- :db_id db_id- :schema schema- :table table- :table_id (:table_id existing)}- table_id (assoc :table_id table_id)))))+ (let [table_id (or table_id+ (t2/select-one-fn :id [:model/Table :id]+ :db_id db_id :schema schema :name table))]+ (app-db/update-or-insert!+ :model/WorkspaceInput+ {:workspace_id workspace-id+ :db_id db_id+ :schema schema+ :table table}+ (fn [existing]+ (cond-> {:workspace_id workspace-id+ :db_id db_id+ :schema schema+ :table table+ :table_id (:table_id existing)}+ table_id (assoc :table_id table_id))))))(defn- insert-workspace-inputs!"Insert a single workspace_input record per table, with a join entry per transform that uses that table."@@ -330,11 +322,10 @@:id [:in {:select [:database_id]:from [:workspace]:where [:= :id workspace-id]}])- normalize (partial sql.normalize/normalize-name driver)default-schema (driver.sql/default-schema driver);; Normalize external inputs so schemas are consistentnormalized-inputs (map (partial normalize-input-schema default-schema) inputs)];; Insert inputs first, then output - output row acts as "commit marker" for version check(insert-workspace-inputs! workspace-id ref-id normalized-inputs transform-version)- (insert-workspace-output! workspace-id ref-id isolated-schema output normalize transform-version)+ (insert-workspace-output! workspace-id ref-id isolated-schema output transform-version)nil))))diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/impl.clj b/enterprise/backend/src/metabase_enterprise/workspaces/impl.cljindex c377b958874..2606b8d5e98 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/impl.clj@@ -12,6 +12,7 @@[metabase.driver.sql :as driver.sql][metabase.util :as u][metabase.util.log :as log]+ [metabase.warehouse-schema.models.table :as table][toucan2.core :as t2]))(defn- query-ungranted-external-inputs@@ -566,16 +567,17 @@(when-not (t2/exists? :model/WorkspaceOutputExternal :workspace_id workspace-id :graph_version [:>= graph-version])(let [external-tx-ids (extract-external-transform-ids entities)](when (seq external-tx-ids)+ ;; There are 4N + 3 queries for N transforms, we may want to optimize...+ ;; 1 exists+ ;; + 1 select+ ;; + N × 2 upsert-transform-target-table! (2 queries each)+ ;; + 1 batch insert+ ;; Worst case 8N + 3 on concurrent upsert conflicts.(let [transforms (t2/select [:model/Transform :id :target] :id [:in external-tx-ids])rows (for [{tx-id :id, {:keys [database schema name]} :target} transforms](let [isolated-table (ws.u/isolated-table-name schema name)- ;; TODO (Chris 2026-01-26) -- 2N + 1 is really not great here...- global-table-id (t2/select-one-fn :id [:model/Table :id]- :db_id database :schema schema :name name)- isolated-table-id (t2/select-one-fn :id [:model/Table :id]- :db_id database- :schema isolated-schema- :name isolated-table)]+ global-table-id (table/upsert-transform-target-table! database schema name)+ isolated-table-id (table/upsert-transform-target-table! database isolated-schema isolated-table)]{:workspace_id workspace-id:transform_id tx-id:graph_version graph-versiondiff --git a/enterprise/backend/src/metabase_enterprise/workspaces/validation.clj b/enterprise/backend/src/metabase_enterprise/workspaces/validation.cljindex 2d27343fd84..8dfae6e59eb 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/validation.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/validation.clj@@ -77,7 +77,8 @@(defn- workspace-outputs"Get all workspace outputs with their global and isolated table info.- Includes both workspace transform outputs and enclosed external transform outputs."+ Includes both workspace transform outputs and enclosed external transform outputs.+ Enriches each output with :isolated_active from the referenced metabase_table."[workspace-id];; TODO (Chris 2026-02-02) -- rather fetch all this as an input to graph analysis, and save it in [[graph]].(let [ws-outputs (t2/select [:model/WorkspaceOutput@@ -92,8 +93,15 @@:id :transform_id :db_id:global_schema :global_table :global_table_id:isolated_schema :isolated_table :isolated_table_id]- :workspace_id workspace-id))]- (concat ws-outputs ext-outputs)))+ :workspace_id workspace-id))+ all-outputs (concat ws-outputs ext-outputs)+ ;; We could avoid this extra query by using a JOIN, but we'd need to define a derived model for this to preserve+ ;; the model transforms (e.g. char(36) trimming on ref_id), and that feels excessive for a single query.+ active-ids (t2/select-fn-set :id [:model/Table :id]+ :id [:in (keep :isolated_table_id all-outputs)]+ :active true)]+ (for [output all-outputs]+ (assoc output :isolated_active (contains? active-ids (:isolated_table_id output))))))(defn- ids-for-type [entities desired-node-type](for [{:keys [node-type id]} entities, :when (= node-type desired-node-type)] id))@@ -192,7 +200,7 @@Returns a sequence of problem maps."[output checked-out-ids enclosed-ids internal-dependents-map](let [{:keys [ref_id db_id global_schema global_table global_table_id- isolated_table_id external?]} output+ isolated_table_id isolated_active external?]} output;; For external outputs, ref_id is actually the transform_id (an int)transform-type (if external? :external-transform :workspace-transform)external-transforms (external-transform-dependents global_table_id checked-out-ids enclosed-ids)@@ -202,8 +210,8 @@table-coord {:db_id db_id :schema global_schema :table global_table}](cond- ;; Case 1: Isolated table doesn't exist yet- (nil? isolated_table_id)+ ;; Case 1: Isolated table is not active (no row, hasn't been run, or was deactivated)+ (not isolated_active)(cond;; External transforms depend on this output that doesn't existhas-external-dependents?diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.cljindex 7ea1fa7c411..30d44d7d5d0 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj@@ -1341,21 +1341,25 @@:source {:type :query:query (lib/query mp products)}:target {:schema "PUBLIC"- :name "referenced_transform_table"}}- :model/Table _ {:name "referenced_transform_table"- :db_id (mt/id)- :schema "PUBLIC"}]+ :name "referenced_transform_table"}}]+ ;; Simulate the referenced transform having been run: activate its provisional target table+ ;; so the table→transform dep is visible in the dependency graph (which filters active=true).+ (t2/update! :model/Table {:db_id (mt/id) :schema "PUBLIC" :name "referenced_transform_table"} {:active true})(events/publish-event! :event/transform-run-complete{:object {:db-id (mt/id):output-schema "PUBLIC":output-table "referenced_transform_table":transform-id referenced-transform-id}})(while (#'dependencies.backfill/backfill-dependencies!))- (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=transform&query=unreftest")]- (is (=? {:data [{:id unreffed-transform-id- :type "transform"- :data {:name "Unreferenced Transform - unreftest"}}]}- response))))))))+ (try+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=transform&query=unreftest")]+ (is (=? {:data [{:id unreffed-transform-id+ :type "transform"+ :data {:name "Unreferenced Transform - unreftest"}}]}+ response)))+ (finally+ ;; Clean up provisional table rows created by define-after-insert+ (t2/delete! :model/Table :db_id (mt/id) :name [:in ["referenced_transform_table" "unreferenced_transform_table"]]))))))))(deftest ^:sequential unreferenced-snippets-test(testing "GET /api/ee/dependencies/unreferenced - only unreferenced snippets are returned"diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.cljindex d47ccb1f3a7..897d692b86b 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj@@ -346,21 +346,22 @@(run-with-dependencies-setup!(fn [_](let [target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]- (mt/with-temp [:model/Transform {transform-id :id} {:target target}- :model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}]- (events/publish-event! :event/transform-run-complete- {:object {:db-id (mt/id)- :output-schema "Other"- :output-table :test_table- :transform-id transform-id}- :user-id api/*current-user-id*})- (is (= models.dependency/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))- (is (=? [{:from_entity_type :table- :from_entity_id table-id,- :to_entity_type :transform,- :to_entity_id transform-id}]- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))+ (mt/with-temp [:model/Transform {transform-id :id} {:target target}]+ ;; Transform after-insert creates the target table row via upsert-transform-target-table!+ (let [table-id (t2/select-one-pk :model/Table :db_id (mt/id) :schema "Other" :name "test_table")]+ (events/publish-event! :event/transform-run-complete+ {:object {:db-id (mt/id)+ :output-schema "Other"+ :output-table :test_table+ :transform-id transform-id}+ :user-id api/*current-user-id*})+ (is (= models.dependency/current-dependency-analysis-version+ (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))+ (is (=? [{:from_entity_type :table+ :from_entity_id table-id,+ :to_entity_type :transform,+ :to_entity_id transform-id}]+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))(deftest ^:sequential python-transform-update-handles-downstream-dependencies-test(testing "python transform update events handles downstream dependencies"@@ -378,35 +379,36 @@target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}](mt/with-temp [:model/Transform {transform-id :id :as transform} {:target target:source source}- :model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}+ ;; test_table is created by the Transform after-insert via upsert-transform-target-table!:model/Table {} {:schema "Other", :db_id (mt/id), :name "test_table2"}]- (testing "initial run"- (events/publish-event! :event/transform-run-complete- {:object {:db-id (mt/id)- :output-schema "Other"- :output-table :test_table- :transform-id transform-id}- :user-id api/*current-user-id*})- (is (=? [{:from_entity_type :table- :from_entity_id table-id,- :to_entity_type :transform,- :to_entity_id transform-id}]- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))- (testing "keeping target"- (events/publish-event! :event/update-transform- {:object transform- :user-id api/*current-user-id*})- (is (=? [{:from_entity_type :table- :from_entity_id table-id,- :to_entity_type :transform,- :to_entity_id transform-id}]- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))- (testing "changing target update"- (events/publish-event! :event/update-transform- {:object (assoc-in transform [:target :name] "test_table2")- :user-id api/*current-user-id*})- (is (empty?- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))+ (let [table-id (t2/select-one-pk :model/Table :db_id (mt/id) :schema "Other" :name "test_table")]+ (testing "initial run"+ (events/publish-event! :event/transform-run-complete+ {:object {:db-id (mt/id)+ :output-schema "Other"+ :output-table :test_table+ :transform-id transform-id}+ :user-id api/*current-user-id*})+ (is (=? [{:from_entity_type :table+ :from_entity_id table-id,+ :to_entity_type :transform,+ :to_entity_id transform-id}]+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))+ (testing "keeping target"+ (events/publish-event! :event/update-transform+ {:object transform+ :user-id api/*current-user-id*})+ (is (=? [{:from_entity_type :table+ :from_entity_id table-id,+ :to_entity_type :transform,+ :to_entity_id transform-id}]+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))+ (testing "changing target update"+ (events/publish-event! :event/update-transform+ {:object (assoc-in transform [:target :name] "test_table2")+ :user-id api/*current-user-id*})+ (is (empty?+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))))(deftest ^:sequential query-transform-update-handles-downstream-dependencies-test(testing "query transform update events handles downstream dependencies"@@ -416,35 +418,36 @@:type :query}target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}](mt/with-temp [:model/Transform {transform-id :id :as transform} {:target target :source source}- :model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}+ ;; test_table is created by the Transform after-insert via upsert-transform-target-table!:model/Table {} {:schema "Other", :db_id (mt/id), :name "test_table2"}]- (testing "initial run"- (events/publish-event! :event/transform-run-complete- {:object {:db-id (mt/id)- :output-schema "Other"- :output-table :test_table- :transform-id transform-id}- :user-id api/*current-user-id*})- (is (=? [{:from_entity_type :table- :from_entity_id table-id,- :to_entity_type :transform,- :to_entity_id transform-id}]- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))- (testing "keeping target"- (events/publish-event! :event/update-transform- {:object transform- :user-id api/*current-user-id*})- (is (=? [{:from_entity_type :table- :from_entity_id table-id,- :to_entity_type :transform,- :to_entity_id transform-id}]- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))- (testing "changing target update"- (events/publish-event! :event/update-transform- {:object (assoc-in transform [:target :name] "test_table2")- :user-id api/*current-user-id*})- (is (empty?- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))+ (let [table-id (t2/select-one-pk :model/Table :db_id (mt/id) :schema "Other" :name "test_table")]+ (testing "initial run"+ (events/publish-event! :event/transform-run-complete+ {:object {:db-id (mt/id)+ :output-schema "Other"+ :output-table :test_table+ :transform-id transform-id}+ :user-id api/*current-user-id*})+ (is (=? [{:from_entity_type :table+ :from_entity_id table-id,+ :to_entity_type :transform,+ :to_entity_id transform-id}]+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))+ (testing "keeping target"+ (events/publish-event! :event/update-transform+ {:object transform+ :user-id api/*current-user-id*})+ (is (=? [{:from_entity_type :table+ :from_entity_id table-id,+ :to_entity_type :transform,+ :to_entity_id transform-id}]+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))+ (testing "changing target update"+ (events/publish-event! :event/update-transform+ {:object (assoc-in transform [:target :name] "test_table2")+ :user-id api/*current-user-id*})+ (is (empty?+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))))(deftest transform-dependency-calculation-error-handling-test(testing "When transform dependency calculation throws an error, it should be logged and the version should still be updated"diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.cljindex 86d3c6fd975..50f46f7afbb 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj@@ -59,7 +59,7 @@:source {:type "query":query (mt/mbql-query nil{:source-table (str "card__" card-id)})}- :target {:type "table" :name "test_table"}+ :target {:type "table" :name (mt/random-name)}:dependency_analysis_version 0}](is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))(is (t2/exists? :model/Transform :id transform-id :dependency_analysis_version 0))diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/create_autogenerated_dashboard_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/create_autogenerated_dashboard_test.cljindex 88db5c9599f..c8c13d42245 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/create_autogenerated_dashboard_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/create_autogenerated_dashboard_test.clj@@ -32,14 +32,14 @@(testing "returns final-response? flag";; We need to test the structure without actually hitting the database;; Using mt/with-temp for table creation- (mt/with-temp [:model/Table table {:name "test_table" :db_id (mt/id)}]+ (mt/with-temp [:model/Table table {:name (mt/random-name) :db_id (mt/id)}](let [result (autogen-dashboard/create-autogenerated-dashboard{:source {:table_id (:id table)}:memory-atom (atom {:state {}})})](is (true? (:final-response? result))))))(testing "returns redirect reaction"- (mt/with-temp [:model/Table table {:name "test_table" :db_id (mt/id)}]+ (mt/with-temp [:model/Table table {:name (mt/random-name) :db_id (mt/id)}](let [result (autogen-dashboard/create-autogenerated-dashboard{:source {:table_id (:id table)}:memory-atom (atom {:state {}})})]@@ -48,7 +48,7 @@(is (str/starts-with? (:url (first (:reactions result))) "/auto/dashboard/")))))(testing "returns user-friendly message"- (mt/with-temp [:model/Table table {:name "test_table" :db_id (mt/id)}]+ (mt/with-temp [:model/Table table {:name (mt/random-name) :db_id (mt/id)}](let [result (autogen-dashboard/create-autogenerated-dashboard{:source {:table_id (:id table)}:memory-atom (atom {:state {}})})]@@ -56,7 +56,7 @@(is (str/includes? (get-in result [:structured-output :message]) "dashboard")))))(testing "returns path in structured output"- (mt/with-temp [:model/Table table {:name "test_table" :db_id (mt/id)}]+ (mt/with-temp [:model/Table table {:name (mt/random-name) :db_id (mt/id)}](let [result (autogen-dashboard/create-autogenerated-dashboard{:source {:table_id (:id table)}:memory-atom (atom {:state {}})})]@@ -66,7 +66,7 @@(deftest path-generation-test(testing "table source generates correct path"- (mt/with-temp [:model/Table table {:name "test_table" :db_id (mt/id)}]+ (mt/with-temp [:model/Table table {:name (mt/random-name) :db_id (mt/id)}](let [result (autogen-dashboard/create-autogenerated-dashboard{:source {:table_id (:id table)}:memory-atom (atom {:state {}})})]diff --git a/enterprise/backend/test/metabase_enterprise/permission_debug/impl_test.clj b/enterprise/backend/test/metabase_enterprise/permission_debug/impl_test.cljindex 9fc89743636..dc137d77170 100644--- a/enterprise/backend/test/metabase_enterprise/permission_debug/impl_test.clj+++ b/enterprise/backend/test/metabase_enterprise/permission_debug/impl_test.clj@@ -5,7 +5,8 @@[metabase.permissions.core :as perms][metabase.test :as mt][metabase.test.fixtures :as fixtures]- [metabase.util.i18n :refer [tru]]))+ [metabase.util.i18n :refer [tru]]+ [metabase.warehouse-schema.models.table :as table]))(use-fixtures :once (fixtures/initialize :db))@@ -208,6 +209,8 @@:dataset_query (mt/native-query {:query "SELECT * FROM checkins WHERE date > '2014-01-01'"})}](let [regular-user-id (mt/user->id :rasta)](perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :blocked)+ ;; GC leaked provisional table rows so they don't appear in blocked-tables+ (table/gc-transform-target-tables!)(let [result (permission-debug.impl/debug-permissions{:user-id regular-user-id:model-id (str (:id card))diff --git a/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj b/enterprise/backend/test/metabase_enterprise/search/scoring_test.cljindex 602ab7d6c76..29cca6615ca 100644--- a/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj+++ b/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj@@ -199,7 +199,7 @@:source {:type "query":query (mt/native-query {:query "SELECT 1"})}:target {:type "table"- :name "test_table"}}+ :name (mt/random-name)}}:model/RecentViews _ (recent-view c1 now)](appdb.scoring-test/with-index-contents[{:model "card" :id c1 :name "test card recent"}diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.cljindex 8b96157d4a6..599216046ef 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj@@ -98,8 +98,7 @@" print(\"out2\")"" print(\"err2\", file=sys.stderr)"" return pd.DataFrame({'x': [42, 43]})"]- table-id (t2/select-one-pk :model/Table :db_id (mt/id))- body {:source_tables [(transforms.tu/source-table-entry "test" table-id)]+ body {:source_tables [(transforms.tu/default-source-table-entry)]:code (str/join "\n" program)}{:keys [error logs output]} (mt/user-http-request :crowberto :post 200 "ee/transforms-python/test-run" body)](is (nil? error))@@ -111,7 +110,7 @@user :crowbertofeatures #{:transforms :transforms-python}}}](let [source-tables (or source-tables- [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id) :active true))])+ [(transforms.tu/default-source-table-entry)])body (merge {:source_tables source-tables, :code (str/join "\n" program)} extra-opts)](mt/with-premium-features features(mt/user-http-request-full-response user :post "ee/transforms-python/test-run" body))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.cljindex dd63ecdce62..6f095a12bda 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj@@ -5,9 +5,10 @@[metabase.transforms-base.interface :as transforms-base.i][metabase.transforms-base.ordering :as ordering][metabase.transforms.test-util :as transforms.tu]+ [metabase.warehouse-schema.models.table :as ws.table][toucan2.core :as t2]))-(defn- make-transform [query & [name schema]]+(defn- sql-tx [query & [name schema]](let [name (or name (mt/random-name))schema (or schema (transforms.tu/default-schema-or-public))]{:source {:type :query@@ -17,7 +18,7 @@:name name:type :table}}))-(defn- make-python-transform [source-tables & [name schema target-db]]+(defn- py-tx [source-tables & [name schema target-db]](let [name (or name (mt/random-name))schema (or schema (transforms.tu/default-schema-or-public))target-db (or target-db (mt/id))]@@ -37,8 +38,8 @@(deftest python-transform-basic-dependencies-test(testing "Python transforms with source-tables dependencies are extracted correctly"- (mt/with-temp [:model/Transform {t1 :id} (make-python-transform [(transforms.tu/source-table-entry "orders" (mt/id :orders))- (transforms.tu/source-table-entry "products" (mt/id :products))])]+ (mt/with-temp [:model/Transform {t1 :id} (py-tx [(transforms.tu/source-table-entry "orders" (mt/id :orders))+ (transforms.tu/source-table-entry "products" (mt/id :products))])](is (= #{{:table (mt/id :orders)}{:table (mt/id :products)}}(transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))@@ -47,12 +48,11 @@(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)(testing "Python transforms are ordered correctly based on source-tables dependencies"(let [schema (transforms.tu/default-schema-or-public)]- (mt/with-temp [:model/Table {table1 :id} {:schema schema- :name "output_1"}- :model/Field _ {:table_id table1- :name "foo"}- :model/Transform {t1 :id} (make-python-transform [(transforms.tu/source-table-entry "orders" (mt/id :orders))] "output_1")- :model/Transform {t2 :id} (make-python-transform [(transforms.tu/source-table-entry "output_1" table1)] "output_2")]+ (ws.table/gc-transform-target-tables!)+ (mt/with-temp [:model/Table {table1 :id} {:schema schema :name "output_1"}+ :model/Field _ {:table_id table1 :name "foo"}+ :model/Transform {t1 :id} (py-tx [(transforms.tu/source-table-entry "orders" (mt/id :orders))] "output_1")+ :model/Transform {t2 :id} (py-tx [(transforms.tu/source-table-entry "output_1" table1)] "output_2")](is (= {t1 #{}t2 #{t1}}(ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2]])))))))))@@ -61,18 +61,15 @@(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)(testing "Python transforms with multiple table dependencies"(let [schema (transforms.tu/default-schema-or-public)]- (mt/with-temp [:model/Table {table1 :id} {:schema schema- :name "output_1"}- :model/Field _ {:table_id table1- :name "foo"}- :model/Table {table2 :id} {:schema schema- :name "output_2"}- :model/Field _ {:table_id table2- :name "bar"}- :model/Transform {t1 :id} (make-python-transform [(transforms.tu/source-table-entry "orders" (mt/id :orders))] "output_1")- :model/Transform {t2 :id} (make-python-transform [(transforms.tu/source-table-entry "products" (mt/id :products))] "output_2")- :model/Transform {t3 :id} (make-python-transform [(transforms.tu/source-table-entry "output_1" table1)- (transforms.tu/source-table-entry "output_2" table2)] "final_output")]+ (ws.table/gc-transform-target-tables!)+ (mt/with-temp [:model/Table {table1 :id} {:schema schema :name "output_1"}+ :model/Field _ {:table_id table1 :name "foo"}+ :model/Table {table2 :id} {:schema schema :name "output_2"}+ :model/Field _ {:table_id table2 :name "bar"}+ :model/Transform {t1 :id} (py-tx [(transforms.tu/source-table-entry "orders" (mt/id :orders))] "output_1")+ :model/Transform {t2 :id} (py-tx [(transforms.tu/source-table-entry "products" (mt/id :products))] "output_2")+ :model/Transform {t3 :id} (py-tx [(transforms.tu/source-table-entry "output_1" table1)+ (transforms.tu/source-table-entry "output_2" table2)] "final_output")](is (= {t1 #{}t2 #{}t3 #{t1 t2}}@@ -82,28 +79,22 @@(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python :transforms/table)(testing "Python and query transforms are ordered together correctly"(let [schema (transforms.tu/default-schema-or-public)]- (mt/with-temp [:model/Table {table1 :id} {:schema schema- :name "sql_output"}- :model/Field _ {:table_id table1- :name "foo"}- :model/Table {table2 :id} {:schema schema- :name "python_output"}- :model/Field _ {:table_id table2- :name "bar"}+ (ws.table/gc-transform-target-tables!)+ (mt/with-temp [:model/Table {table1 :id} {:schema schema :name "sql_output"}+ :model/Field _ {:table_id table1 :name "foo"}+ :model/Table {table2 :id} {:schema schema :name "python_output"}+ :model/Field _ {:table_id table2 :name "bar"};; SQL transform that depends on orders table- :model/Transform {t1 :id} (make-transform- {:database (mt/id),- :type "query",- :query {:source-table (mt/id :orders)}}- "sql_output")+ :model/Transform {t1 :id} (sql-tx {:database (mt/id)+ :type "query"+ :query {:source-table (mt/id :orders)}}+ "sql_output");; Python transform that depends on the SQL transform's output- :model/Transform {t2 :id} (make-python-transform [(transforms.tu/source-table-entry "sql_output" table1)] "python_output")+ :model/Transform {t2 :id} (py-tx [(transforms.tu/source-table-entry "sql_output" table1)] "python_output");; Another SQL transform that depends on the Python transform's output- :model/Transform {t3 :id} (make-transform- {:database (mt/id)- :type "query"- :query {:source-table table2}}- "final_output")]+ :model/Transform {t3 :id} (sql-tx {:database (mt/id)+ :type "query"+ :query {:source-table table2}} "final_output")](is (= {t1 #{}t2 #{t1}t3 #{t2}}diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.cljindex fa62b6f6c38..76496d6afa4 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj@@ -223,7 +223,7 @@:source {:type "python":body (program->source program):source-database (mt/id)- :source-tables [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id)))]}+ :source-tables [(transforms.tu/default-source-table-entry)]}:target (assoc target :database (mt/id))})))(block-on-run [{:keys [expect-status]} target transform-id]@@ -329,7 +329,7 @@:model/Transform transform {:name "Python Transform Cross DB":source {:type "python":source-database (mt/id)- :source-tables [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id)))]+ :source-tables [(transforms.tu/default-source-table-entry)]:body "def transform():\n pass"}:target {:type "table":schema "PUBLIC"@@ -526,7 +526,7 @@(let [initial-transform {:name "Schema Change Integration Test":source {:type "python":source-database (mt/id)- :source-tables [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id)))]+ :source-tables [(transforms.tu/default-source-table-entry)]:body (str "import pandas as pd\n""\n""def transform():\n"@@ -545,7 +545,7 @@(let [updated-transform (assoc initial-transform:source {:type "python":source-database (mt/id)- :source-tables [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id)))]+ :source-tables [(transforms.tu/default-source-table-entry)]:body (str "import pandas as pd\n""\n""def transform():\n"diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.cljindex 2f54fbae89c..0e260d7f240 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj@@ -834,7 +834,7 @@:isolated {:transform_id ref-id#_#_:schema (:schema workspace):table string?- :table_id nil}}]}+ :table_id int?}}]}(mt/user-http-request :crowberto :get 200 (ws-url (:id workspace) "/table")))))(testing "and after we run the transform, id for isolated table appears"(is (=? {:status "succeeded"}@@ -1804,8 +1804,8 @@(testing "fallback populates global table_id from metabase_table"(is (= (:id table)(-> result :outputs first :global :table_id))))- (testing "isolated table_id remains nil when table doesn't exist"- (is (nil? (-> result :outputs first :isolated :table_id))))))))))+ (testing "isolated table_id is populated with transform target table"+ (is (int? (-> result :outputs first :isolated :table_id))))))))))(deftest ^:parallel tables-endpoint-fallback-isolated-table-id-test(testing "GET /api/ee/workspace/:id/table uses fallback for null isolated_table_id"diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.cljindex 7a1f59a73f3..8ac5eabea6f 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj@@ -57,7 +57,7 @@(is (=? [{:workspace_id (:id workspace):global_table output-table:global_schema nil- :global_table_id nil+ :global_table_id pos-int?:isolated_schema string?:isolated_table string?:isolated_table_id number?}]diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/pages/WorkspacePage/DataTab/DataTabSidebar.tsx b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/pages/WorkspacePage/DataTab/DataTabSidebar.tsxindex 4af6cd210cd..674f1148017 100644--- a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/pages/WorkspacePage/DataTab/DataTabSidebar.tsx+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/pages/WorkspacePage/DataTab/DataTabSidebar.tsx@@ -150,6 +150,7 @@ export const DataTabSidebar = ({hasChanges={hasChanges}transform={workspaceTransform}tableId={tableId ?? undefined}+ tableActive={table.isolated.active ?? false}isSelected={tableId === selectedTableId}isRunning={workspaceTransformdiff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/pages/WorkspacePage/DataTab/TableListItem.tsx b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/pages/WorkspacePage/DataTab/TableListItem.tsxindex d1a59b041e6..284778e20f3 100644--- a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/pages/WorkspacePage/DataTab/TableListItem.tsx+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/pages/WorkspacePage/DataTab/TableListItem.tsx@@ -27,6 +27,7 @@ type TableListItemProps = {hasChanges?: boolean;transform?: WorkspaceTransformListItem;tableId?: TableId;+ tableActive?: boolean;isSelected?: boolean;isRunning?: boolean;readOnly?: boolean;@@ -43,6 +44,7 @@ export const TableListItem = ({hasChanges = false,transform,tableId,+ tableActive = false,isSelected = false,isRunning = false,readOnly = false,@@ -59,29 +61,30 @@ export const TableListItem = ({}};+ const tableExists = !!tableId && (type !== "output" || tableActive);+const handleTableClick = () => {if (isRunning) {return;}- if (tableId && onTableClick) {+ if (tableExists && tableId && onTableClick) {onTableClick({ tableId, name, schema, transformId: transform?.ref_id });} else if (type === "output" && transform && onRunTransform && !readOnly) {onRunTransform(transform);}};- const hasResults = type === "output" && !!tableId;const canRunTransform =- type === "output" && !hasResults && transform && onRunTransform;- const isClickable = (tableId && onTableClick) || canRunTransform;- const displayTooltip = type !== "input" && !hasResults && !isRunning;+ type === "output" && !tableExists && transform && onRunTransform;+ const isClickable = (tableExists && onTableClick) || canRunTransform;+ const displayTooltip = type !== "input" && !tableExists && !isRunning;return (<BoxclassName={cx(S.root, {[S.selected]: isSelected,[S.clickable]: isClickable,- [S.noResults]: type === "output" && !tableId,+ [S.noResults]: type === "output" && !tableExists,})}aria-label={displayName}onClick={isClickable ? handleTableClick : undefined}diff --git a/frontend/src/metabase-types/api/workspace.ts b/frontend/src/metabase-types/api/workspace.tsindex 504a2acdfde..ac206b5d2a2 100644--- a/frontend/src/metabase-types/api/workspace.ts+++ b/frontend/src/metabase-types/api/workspace.ts@@ -250,6 +250,7 @@ export type WorkspaceOutputTableEntry = {schema: string;table: string;table_id: number | null;+ active?: boolean;};export type WorkspaceOutputTable = {diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.cljindex 9c22f5cd006..e16e9848a84 100644--- a/mage/src/mage/modules.clj+++ b/mage/src/mage/modules.clj@@ -137,7 +137,8 @@usersutilversion- view-log})+ view-log+ warehouse-schema})(defn- affected-modules"Set of modules that are direct or indirect dependents of `modules`, and thus are affected by changes to them.diff --git a/resources/migrations/060/20260307_table_transform_target.yaml b/resources/migrations/060/20260307_table_transform_target.yamlnew file mode 100644index 00000000000..f47fae91ff1--- /dev/null+++ b/resources/migrations/060/20260307_table_transform_target.yaml@@ -0,0 +1,35 @@+## Add transform_target column to metabase_table and backfill existing transforms++databaseChangeLog:+ - logicalFilePath: migrations/060_update_migrations.yaml+ - changeSet:+ id: v60.2026-03-07T00:00:03+ author: christruter+ comment: Add transform_target column to metabase_table+ changes:+ - addColumn:+ tableName: metabase_table+ columns:+ - column:+ name: transform_target+ type: ${boolean.type}+ defaultValueBoolean: false+ constraints:+ nullable: false+ remarks: >-+ Whether any transform is configured to target this table. Conservative: eagerly set to true+ when a transform targets this table (never stale in this direction), cleared asynchronously+ when no transforms reference it anymore (may have false positives, never false negatives).+ rollback:+ - dropColumn:+ tableName: metabase_table+ columnName: transform_target++ - changeSet:+ id: v60.2026-03-07T00:00:04+ author: christruter+ comment: Backfill transform target tables for existing transforms and workspace outputs+ changes:+ - customChange:+ class: metabase.app_db.custom_migrations.BackfillTransformTargetTables+ rollback:diff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.cljindex 7eedb67395e..7987c43ba24 100644--- a/src/metabase/app_db/custom_migrations.clj+++ b/src/metabase/app_db/custom_migrations.clj@@ -2051,3 +2051,85 @@(t2/query {:update :transform:set {:source (json-in (dissoc parsed :source-incremental-strategy))}:where [:= :id id]}))))))++(define-migration BackfillTransformTargetTables+ ;; For each transform that has a target (database, schema, name), ensure a metabase_table row exists.+ ;; If the table already exists (active or inactive), do nothing. Otherwise, insert a transform target row.+ ;;+ ;; Similarly, backfill workspace_output rows missing global_table_id or isolated_table_id.+ (let [;; Reproduces humanization/name->human-readable-name :simple (can't call library code in migrations)+ acronyms #{"id" "url" "ip" "uid" "uuid" "guid"}+ cap-word (fn [word]+ (let [lower (lower-case-en word)]+ (if (contains? acronyms lower)+ (upper-case-en word)+ (if (= word (upper-case-en word))+ (str/capitalize word)+ (str (str/capitalize (subs word 0 1)) (subs word 1))))))+ humanize (fn [s]+ (when (seq s)+ (let [result (str/join " " (for [part (str/split s #"[-_\s]+")+ :when (not (str/blank? part))]+ (cap-word part)))]+ (if (str/blank? result) s result))))+ find-table-id (fn [db-id schema table-name]+ (:id (first (t2/query {:select [:id]+ :from [:metabase_table]+ :where [:and+ [:= :db_id db-id]+ (if (some? schema)+ [:= [:lower :schema] (lower-case-en schema)]+ [:is :schema nil])+ [:= [:lower :name] (lower-case-en table-name)]]+ :limit 1}))))+ upsert-table! (fn [db-id schema table-name]+ (or (find-table-id db-id schema table-name)+ (try+ (t2/query {:insert-into :metabase_table+ :values [{:db_id db-id+ :schema schema+ :name table-name+ :display_name (humanize table-name)+ :active false+ :transform_target true+ :data_source "metabase-transform"+ :data_authority "computed"+ :initial_sync_status "complete"+ :created_at :%now+ :updated_at :%now}]})+ (find-table-id db-id schema table-name)+ (catch Exception _+ (find-table-id db-id schema table-name)))))]+ ;; 1. Backfill for transforms: create provisional tables for targets that don't exist+ (doseq [{:keys [target target_db_id]} (t2/query {:select [:target :target_db_id]+ :from [:transform]+ :where [:not= :target_db_id nil]})]+ (let [target-map (json-out target false)]+ (when-let [table-name (get target-map "name")]+ (let [schema (get target-map "schema")+ db-id target_db_id]+ (upsert-table! db-id schema table-name)))))+ ;; 2-5. Backfill workspace_output and workspace_output_external rows missing table FKs+ (let [backfill-table-fk! (fn [from-table schema-col table-col fk-col]+ (doseq [{:keys [id db_id] :as row}+ (t2/query {:select [:id :db_id schema-col table-col]+ :from [from-table]+ :where [:= fk-col nil]})]+ (when-let [table-id (upsert-table! db_id (get row schema-col) (get row table-col))]+ (t2/query {:update from-table+ :set {fk-col table-id}+ :where [:= :id id]}))))]+ (backfill-table-fk! :workspace_output :global_schema :global_table :global_table_id)+ (backfill-table-fk! :workspace_output :isolated_schema :isolated_table :isolated_table_id)+ (backfill-table-fk! :workspace_output_external :global_schema :global_table :global_table_id)+ (backfill-table-fk! :workspace_output_external :isolated_schema :isolated_table :isolated_table_id))+ ;; 6-7. Backfill workspace_input and workspace_input_external rows missing table_id+ (doseq [from-table [:workspace_input :workspace_input_external]+ :let [rows (t2/query {:select [:id :db_id :schema :table]+ :from [from-table]+ :where [:= :table_id nil]})]+ {:keys [id db_id schema table]} rows]+ (when-let [table-id (find-table-id db_id schema table)]+ (t2/query {:update from-table+ :set {:table_id table-id}+ :where [:= :id id]})))))diff --git a/src/metabase/app_db/query.clj b/src/metabase/app_db/query.cljindex 93bcde0d750..51303e02bd4 100644--- a/src/metabase/app_db/query.clj+++ b/src/metabase/app_db/query.clj@@ -187,7 +187,9 @@(merge instance select-map))](with-conflict-retry(or (apply t2/select-one model select-kvs)- (t2/insert-returning-instance! model (insert-fn))))))+ ;; Wrap INSERT in a savepoint — see comment in [[update-or-insert!]].+ (t2/with-transaction [_conn]+ (t2/insert-returning-instance! model (insert-fn)))))))(defn update-or-insert!"Update a database record, if it exists, otherwise create it.@@ -200,6 +202,9 @@Note that the generated entity must be consistent with `select-map`, if it disagrees on any keys then an exceptionwill be thrown. It is OK for the entity to omit fields from `select-map`, they will implicitly be added on.+ When called with an existing record, `update-fn` may return nil to signal that no update is needed. In this case+ the existing row is left untouched and its primary key is returned. This is useful for \"get-or-create\" patterns.+This is more general than using `UPSERT`, `MERGE` or `INSERT .. ON CONFLICT`, and it also allows one to avoidcalculating initial values that may be expensive, or require side effects.@@ -217,16 +222,25 @@pk-key (keyword (first pks))update-fn (fn [existing](let [updated (update-fn existing)]- ;; the inserted / updated values must be consistent with the select query- (assert (not (u/conflicting-keys? select-map updated))- "This should not be used to change any of the identifying values")- ;; For convenience, we allow the update-fn to omit fields in the search-map- (merge updated select-map)))]+ ;; When called with an existing record, returning nil signals "no update needed".+ (when (or updated (not existing))+ ;; the inserted / updated values must be consistent with the select query+ (assert (not (u/conflicting-keys? select-map updated))+ "This should not be used to change any of the identifying values")+ ;; For convenience, we allow the update-fn to omit fields in the search-map+ (merge updated select-map))))](with-conflict-retry(if-let [existing (apply t2/select-one model select-kvs)]- (let [pk (pk-key existing)- updated (update-fn existing)]- (t2/update! model pk updated)- ;; the private key may have been changed by the update, and this is OK.- (pk-key updated pk))- (t2/insert-returning-pk! model (update-fn nil))))))+ (let [pk (pk-key existing)]+ (if-let [updated (update-fn existing)]+ (do (t2/update! model pk updated)+ ;; the primary key may have been changed by the update, and this is OK.+ (pk-key updated pk))+ ;; update-fn returned nil — no update needed, return existing pk.+ pk))+ ;; Wrap INSERT in a savepoint so that a constraint violation inside an existing+ ;; PostgreSQL transaction doesn't abort the outer transaction. The savepoint is+ ;; rolled back on failure, keeping the outer transaction usable for the retry+ ;; in [[with-conflict-retry]].+ (t2/with-transaction [_conn]+ (t2/insert-returning-pk! model (update-fn nil)))))))diff --git a/src/metabase/models/transforms/transform.clj b/src/metabase/models/transforms/transform.cljindex 06c50b9a61b..704294dd47e 100644--- a/src/metabase/models/transforms/transform.clj+++ b/src/metabase/models/transforms/transform.clj@@ -108,11 +108,16 @@;; In practice, our serialized representation should not contain any database ids, and this should;; not be required.;; TODO (Chris 2026-02-02) -- Update tests so this workaround is unnecessary.- valid-db-id? (and target-db-id (t2/exists? :model/Database :id target-db-id))]+ valid-db-id? (and target-db-id (t2/exists? :model/Database :id target-db-id))+ target-name (get-in transform [:target :name])+ table-id (when (and valid-db-id? target-name)+ (transforms-base.u/upsert-target-table!+ target-db-id (get-in transform [:target :schema]) target-name))](when-not valid-db-id?(log/warnf "Invalid target database id (%s) ignored for new transform (%s)" target-db-id (:name transform)))(-> transform(assoc-in [:target :database] target-db-id)+ (cond-> table-id (assoc-in [:target :table_id] table-id))(assoc:source_type (transforms-base.u/transform-source-type source):target_db_id (when valid-db-id? target-db-id)@@ -123,15 +128,27 @@(when-let [new-collection (:collection_id (t2/changes transform))](collection/check-collection-namespace :model/Transform new-collection)(collection/check-allowed-content :model/Transform new-collection))- (let [changes (t2/changes transform)]+ ;; The target db is recomputed when source changes because for MBQL transforms,+ ;; the source query's :database is the source of truth for the target database.+ (let [target-changed? (or (:source (t2/changes transform)) (:target (t2/changes transform)))+ target-db-id (when target-changed?+ ;; No database existence check added here, unlike for insert.+ ;; Just allow updates for an invalid target to fail.+ (transforms-base.i/target-db-id transform))+ target-name (get-in transform [:target :name])+ table-id (when (and target-db-id target-name)+ (transforms-base.u/upsert-target-table!+ target-db-id (get-in transform [:target :schema]) target-name))](cond-> transformsource(assoc :source_type (transforms-base.u/transform-source-type source):source_database_id (or source_database_id (transforms-base.i/source-db-id transform)))- (or (:source changes) (:target changes))- ;; No database existence check added here, unlike for insert. Just allow updates for an invalid target to fail.- (assoc :target_db_id (transforms-base.i/target-db-id transform))+ target-changed?+ (assoc :target_db_id target-db-id)++ table-id+ (assoc-in [:target :table_id] table-id);; Reset checkpoint when the incremental filter field changes(let [old-field-id (get-in (t2/original transform) [:source :source-incremental-strategy :checkpoint-filter-field-id])@@ -228,6 +245,9 @@transform)(t2/define-before-delete :model/Transform [transform]+ ;; TODO (Chris 2026-03-19) -- Once we have indexed FKs pointing from both global and workspace transforms+ ;; to their target table, we should consider synchronously deleting orphaned provisional table rows+ ;; (tables that have never been physically materialized). See gc-transform-target-tables! for the batch equivalent.(events/publish-event! :event/delete-transform {:id (:id transform)})(search.core/delete! :model/Transform [(str (:id transform))])transform)@@ -345,7 +365,8 @@(transforms-base.u/source-tables-map->vec st)st)))(m/update-existing :query serdes/import-mbql)))}- :target {:export serdes/export-mbql :import serdes/import-mbql}+ :target {:export #(serdes/export-mbql (dissoc % :table_id))+ :import serdes/import-mbql}:tags (serdes/nested :model/TransformTransformTag :transform_id opts)}})(defmethod serdes/dependencies "Transform"diff --git a/src/metabase/sync/sync_metadata/tables.clj b/src/metabase/sync/sync_metadata/tables.cljindex bedb8c65dfd..7749993cd43 100644--- a/src/metabase/sync/sync_metadata/tables.clj+++ b/src/metabase/sync/sync_metadata/tables.clj@@ -307,7 +307,8 @@(defn- archive-tables!"Mark tables that have been deactivated for longer than the configured threshold as archived- and suffixes their names."+ and suffixes their names. Skips tables with `transform_target = true` (provisional transform+ output entries) since transforms still reference them by their original name."[database](let [;; we use UTC offset time for suffix, may not match db time but;; it doesn't matter much, the source of time truth is `archived_at`,@@ -320,6 +321,7 @@:db_id (u/the-id database):active false:archived_at nil+ :transform_target false:deactivated_at [:< threshold-expr])archived (atom 0)](doseq [table tables-to-archivediff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.cljindex 56a3da85774..333dab141f3 100644--- a/src/metabase/transforms_base/util.clj+++ b/src/metabase/transforms_base/util.clj@@ -28,6 +28,7 @@[metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]+ [metabase.warehouse-schema.models.table :as table][toucan2.core :as t2])(:import(java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)@@ -552,9 +553,12 @@(and (:table_id entry) (not (:table entry)))(merge (int-id->metadata (:table_id entry)) entry)- ;; Has table metadata but no table_id — look it up+ ;; Has table metadata but no table_id — look it up, upsert transform target if not found(missing-table-id? entry)- (assoc entry :table_id (ref-lookup (source-table-ref->key entry)))+ (assoc entry :table_id (or (ref-lookup (source-table-ref->key entry))+ (when (and (:database_id entry) (:table entry))+ (table/upsert-transform-target-table!+ (:database_id entry) (:schema entry) (:table entry)))));; Already fully populated:else entry))@@ -669,6 +673,16 @@;;; ------------------------------------------------- Misc -------------------------------------------------+(defn upsert-target-table!+ "Upsert a provisional table entry for a transform's target, creating it if it doesn't exist.+ Returns the table ID.++ Thin wrapper around [[metabase.warehouse-schema.models.table/upsert-transform-target-table!]] —+ exists because the `models` module cannot depend on `warehouse-schema` directly, but can+ depend on `transforms-base` (which is allowed to use `warehouse-schema`)."+ [db-id schema table-name]+ (table/upsert-transform-target-table! db-id schema table-name))+(defn is-temp-transform-table?"Return true when `table` matches the transform temporary table naming pattern."[table]diff --git a/src/metabase/transforms_inspector/context.clj b/src/metabase/transforms_inspector/context.cljindex dd1d9f7e7c0..ccdf214563f 100644--- a/src/metabase/transforms_inspector/context.clj+++ b/src/metabase/transforms_inspector/context.clj@@ -77,10 +77,16 @@;;; -------------------------------------------------- Target Table --------------------------------------------------(defn- get-target-table- "Get the target table for a transform. Returns nil if the target doesn't exist."+ "Get the target table for a transform. Returns nil if the target doesn't exist or hasn't been materialized yet.+ Provisional tables (inactive, never deactivated) created by the after-insert hook are excluded."[{:keys [target] :as transform}]- (let [db-id (transforms-base.i/target-db-id transform)]- (transforms-base.u/target-table db-id target)))+ (let [db-id (transforms-base.i/target-db-id transform)+ table (transforms-base.u/target-table db-id target)]+ ;; A provisional table for a transform that was never run has active=false and deactivated_at=nil+ ;; Having a table with active=false but a non-nil deactivated_at is a strong indication it ran before.+ ;; That said even an active table is not proof that is has run, as targets may have been swiveled around.+ (when (or (:active table) (:deactivated_at table))+ table)));;; -------------------------------------------------- Field Metadata --------------------------------------------------diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.cljindex fe22600b712..327d0f128a7 100644--- a/src/metabase/warehouse_schema/models/table.clj+++ b/src/metabase/warehouse_schema/models/table.clj@@ -14,6 +14,7 @@[metabase.util :as u][metabase.util.log :as log][metabase.util.malli :as mu]+ [metabase.workspaces.core :as workspaces][methodical.core :as methodical][toucan2.core :as t2]))@@ -126,7 +127,10 @@(partial mi/assert-optional-enum data-layers)(some-fn legacy-data-layer->current identity)):field_order mi/transform-keyword- :data_source (mi/transform-validator mi/transform-keyword (partial mi/assert-optional-enum data-sources))+ :data_source (mi/transform-validator-with-fixes+ mi/transform-keyword+ (partial mi/assert-optional-enum data-sources)+ (some-fn keyword identity));; Warning: by using a transform to handle unexpected enum values, serialization becomes lossy:data_authority transform-data-authority})@@ -399,6 +403,78 @@[_table][:schema :name (serdes/hydrated-hash :db :db_id)])+;;; ----------------------------------------- Transform Target Tables -----------------------------------------------++(defn upsert-transform-target-table!+ "Ensure a metabase_table row exists for the given (db_id, schema, name) triple.+ If the table doesn't exist, creates it as a transform target (inactive, not yet physically materialized).+ If it already exists (active or inactive), returns its id without modification.+ Returns the table id.++ The `transform_target` column is **conservative**: it is set eagerly to `true` when a transform is+ configured to target this table (synchronous, never stale), and cleared asynchronously when no+ transforms reference it anymore. This means the column may have false positives (a table marked as+ a target when no transform actually targets it) but **never false negatives** (a targeted table+ will always have `transform_target = true`)."+ [db-id schema table-name]+ (app-db/update-or-insert!+ :model/Table+ {:db_id db-id :schema schema :name table-name}+ (fn [existing]+ (when-not existing+ {:display_name (humanization/name->human-readable-name table-name)+ :active false+ :transform_target true+ :data_source :metabase-transform+ :data_authority :computed+ :initial_sync_status "complete"}))))++(defn- remove-referenced-candidates+ "Short-circuiting reduce: removes matched triples from triple->table."+ [triple->table target-triples]+ (reduce (fn [triple->table triple]+ (if (empty? triple->table)+ (reduced triple->table)+ (dissoc triple->table triple)))+ triple->table+ target-triples))++(defn gc-transform-target-tables!+ "Deletes provisional table rows (created by [[upsert-transform-target-table!]]) that are no longer+ referenced by any Transform or WorkspaceTransform. Safe because these rows were never active,+ so they have no child records.++ Note: this only handles *inactive* provisional tables. Active tables that were previously+ targeted by a transform retain `transform_target = true` even after the transform is deleted.+ A separate process to reset `transform_target` on active, unreferenced tables is not yet+ implemented.++ Future optimizations:+ - Add target_table_id FKs to avoid scanning transform rows.+ - Pre-filter via workspace_input/workspace_output table references."+ []+ (let [candidates (t2/select [:model/Table :id :db_id :schema :name]+ :active false :transform_target true :deactivated_at nil+ :data_source :metabase-transform)]+ (when (seq candidates)+ (let [db-ids (into #{} (map :db_id) candidates)+ triple->table (into {} (map (juxt (juxt :db_id :schema :name) identity)) candidates)+ remaining (remove-referenced-candidates+ triple->table+ (eduction+ (map (fn [{:keys [target target_db_id]}]+ (-> target+ (update :database #(or % target_db_id))+ workspaces/target->triple)))+ (t2/reducible-select [:model/Transform :target :target_db_id] :target_db_id [:in db-ids])))+ remaining (when (seq remaining)+ (remove-referenced-candidates+ remaining+ (workspaces/reducible-target-triples db-ids)))]+ (when-let [dead-ids (seq (mapv :id (vals remaining)))]+ (log/infof "Deleting %d orphaned transform target table(s)" (count dead-ids))+ (t2/delete! :model/Table :id [:in dead-ids]))))))+;;; ------------------------------------------------ Field ordering -------------------------------------------------(def field-order-rule@@ -598,7 +674,7 @@:points_of_interest :caveats :show_in_getting_started :field_order :initial_sync_status :is_upload:database_require_filter :is_defective_duplicate :unique_table_helper :is_writable :data_authority:data_source :owner_email :owner_user_id :is_published]- :skip [:estimated_row_count :view_count]+ :skip [:estimated_row_count :view_count :transform_target]:transform {:created_at (serdes/date):archived_at (serdes/date):deactivated_at (serdes/date)diff --git a/src/metabase/warehouse_schema_rest/api/table.clj b/src/metabase/warehouse_schema_rest/api/table.cljindex a933ec12ca0..68e5943e26e 100644--- a/src/metabase/warehouse_schema_rest/api/table.clj+++ b/src/metabase/warehouse_schema_rest/api/table.clj@@ -73,7 +73,7 @@- `can-write=true` - filter to only tables the user can edit metadata for"[_{:keys [term visibility-type data-layer data-source owner-user-id owner-email orphan-only unused-only- can-query can-write]}+ can-query can-write include-transform-targets]}:- [:map[:term {:optional true} :string][:visibility-type {:optional true} :string]@@ -84,7 +84,8 @@[:orphan-only {:optional true} [:maybe ms/BooleanValue]][:unused-only {:optional true} [:maybe ms/BooleanValue]][:can-query {:optional true} [:maybe ms/BooleanValue]]- [:can-write {:optional true} [:maybe ms/BooleanValue]]]]+ [:can-write {:optional true} [:maybe ms/BooleanValue]]+ [:include-transform-targets {:optional true} [:maybe ms/BooleanValue]]]](let [like (fn [field pattern](case (app-db/db-type)(:h2 :postgres) [:ilike field pattern]@@ -95,7 +96,9 @@(str/replace "%" "\\%")(str/replace "*" "%")(cond-> (not (str/ends-with? term "%")) (str "%")))- where (cond-> [:and [:= :active true]]+ where (cond-> [:and (if include-transform-targets+ [:or [:= :active true] [:= :transform_target true]]+ [:= :active true])](not (str/blank? term)) (conj [:or(like :name pattern)(like :display_name pattern)diff --git a/src/metabase/workspaces/core.clj b/src/metabase/workspaces/core.cljnew file mode 100644index 00000000000..572e66e39fd--- /dev/null+++ b/src/metabase/workspaces/core.clj@@ -0,0 +1,35 @@+(ns metabase.workspaces.core+ "Workspace utilities that operate on the raw database tables (which exist in all editions).+ These must run regardless of whether the workspaces feature flag is enabled, because workspace+ data can exist from a previous period when it was enabled, and we need to maintain invariants+ on that data so it isn't corrupt when the feature is re-enabled."+ (:require+ [metabase.util.json :as json]+ [toucan2.core :as t2]))++(defn target->triple+ "Extracts `[db_id schema name]` from a target map."+ [{:keys [database schema name]}]+ [database schema name])++(defn- parse-target+ "Parses target JSON — raw query bypasses Toucan2 model transforms."+ [target]+ (if (string? target)+ (try (json/decode+kw target)+ (catch Exception _ nil))+ target))++(defn reducible-target-triples+ "Reducible of `[db_id schema name]` triples for workspace transforms targeting any of the given `db-ids`.+ Used by [[metabase.warehouse-schema.models.table/gc-transform-target-tables!]]."+ [db-ids]+ (eduction+ (map (fn [{:keys [target database_id]}]+ (-> (parse-target target)+ (update :database #(or % database_id))+ target->triple)))+ (t2/reducible-query {:select [[:wt.target :target], [:w.database_id :database_id]]+ :from [[:workspace_transform :wt]]+ :join [[:workspace :w] [:= :wt.workspace_id :w.id]]+ :where [:in :w.database_id db-ids]})))diff --git a/test/metabase/app_db/custom_migrations_test.clj b/test/metabase/app_db/custom_migrations_test.cljindex 3240c6277c7..5ce7e015dfd 100644--- a/test/metabase/app_db/custom_migrations_test.clj+++ b/test/metabase/app_db/custom_migrations_test.clj@@ -3015,3 +3015,124 @@(is (map? st))(is (= 42 (get st "orders")))(is (= 99 (get st "products")))))))))++(deftest backfill-transform-target-tables-test+ (testing "v60.2026-03-07T00:00:04 : backfill transform target tables and workspace FK columns"+ (impl/test-migrations ["v60.2026-03-07T00:00:04"] [migrate!]+ (let [user-id (:id (new-instance-with-default :core_user))+ db-id (:id (new-instance-with-default :metabase_database))+ ws-id (:id (t2/insert-returning-instance!+ :workspace {:name "test-ws"+ :creator_id user-id+ :created_at :%now+ :updated_at :%now}))+ ref-id (str (random-uuid))+ source (json/encode {:type "query" :query {:database db-id}})+ target (json/encode {:type "table" :schema "public" :name "orders"})+ ;; -- 1. Transform with a target that has no existing metabase_table → should create provisional row --+ tx-id (t2/insert-returning-pk!+ :transform {:name "create-output"+ :source source+ :target (json/encode {:type "table" :schema "public" :name "new_target_table"})+ :source_type "mbql"+ :source_database_id db-id+ :target_db_id db-id+ :created_at :%now+ :updated_at :%now})+ ;; -- An existing metabase_table that inputs and outputs should resolve to --+ table-id (:id (t2/insert-returning-instance!+ :metabase_table {:db_id db-id+ :name "orders"+ :schema "public"+ :active true+ :created_at :%now+ :updated_at :%now}))+ ;; -- workspace_transform needed for workspace_output FK --+ _ (t2/insert! :workspace_transform+ {:ref_id ref-id+ :workspace_id ws-id+ :name "ws-tx"+ :source source+ :target target+ :created_at :%now+ :updated_at :%now})+ ;; -- 2-3. workspace_output with null table FKs --+ wo-id (:id (t2/insert-returning-instance!+ :workspace_output {:workspace_id ws-id+ :ref_id ref-id+ :db_id db-id+ :global_schema "public"+ :global_table "orders"+ :isolated_schema "public"+ :isolated_table "orders"+ :created_at :%now+ :updated_at :%now}))+ ;; -- 4-5. workspace_output_external with null table FKs --+ woe-id (:id (t2/insert-returning-instance!+ :workspace_output_external {:workspace_id ws-id+ :transform_id tx-id+ :db_id db-id+ :global_schema "public"+ :global_table "orders"+ :isolated_schema "public"+ :isolated_table "orders"+ :created_at :%now+ :updated_at :%now}))+ ;; -- 6. workspace_input with null table_id --+ wi-id (:id (t2/insert-returning-instance!+ :workspace_input {:workspace_id ws-id+ :db_id db-id+ :schema "public"+ :table "orders"+ :created_at :%now+ :updated_at :%now}))+ ;; -- 7. workspace_input_external with null table_id --+ wie-id (:id (t2/insert-returning-instance!+ :workspace_input_external {:workspace_id ws-id+ :db_id db-id+ :schema "public"+ :table "orders"+ :created_at :%now+ :updated_at :%now}))+ ;; -- workspace_input_external with no matching table — should stay nil --+ wie-no-match-id (:id (t2/insert-returning-instance!+ :workspace_input_external {:workspace_id ws-id+ :db_id db-id+ :schema "public"+ :table "no_such_table"+ :created_at :%now+ :updated_at :%now}))]+ (testing "Before backfill, all FKs are nil"+ (is (nil? (t2/select-one-fn :global_table_id :workspace_output :id wo-id)))+ (is (nil? (t2/select-one-fn :isolated_table_id :workspace_output :id wo-id)))+ (is (nil? (t2/select-one-fn :global_table_id :workspace_output_external :id woe-id)))+ (is (nil? (t2/select-one-fn :isolated_table_id :workspace_output_external :id woe-id)))+ (is (nil? (t2/select-one-fn :table_id :workspace_input :id wi-id)))+ (is (nil? (t2/select-one-fn :table_id :workspace_input_external :id wie-id)))+ (is (nil? (t2/select-one-fn :table_id :workspace_input_external :id wie-no-match-id))))+ (migrate!)+ (testing "Provisional metabase_table created for transform target"+ (let [provisional (first (t2/query {:select [:active :transform_target :data_source :data_authority :display_name]+ :from [:metabase_table]+ :where [:and+ [:= :db_id db-id]+ [:= :name "new_target_table"]+ [:= :schema "public"]]}))]+ (is (some? provisional))+ (is (false? (:active provisional)))+ (is (true? (:transform_target provisional)))+ (is (= "metabase-transform" (:data_source provisional)))+ (is (= "computed" (:data_authority provisional)))+ (is (= "New Target Table" (:display_name provisional)))))+ (testing "workspace_output FKs backfilled"+ (is (= table-id (t2/select-one-fn :global_table_id :workspace_output :id wo-id)))+ (is (= table-id (t2/select-one-fn :isolated_table_id :workspace_output :id wo-id))))+ (testing "workspace_output_external FKs backfilled"+ (is (= table-id (t2/select-one-fn :global_table_id :workspace_output_external :id woe-id)))+ (is (= table-id (t2/select-one-fn :isolated_table_id :workspace_output_external :id woe-id))))+ (testing "workspace_input table_id backfilled"+ (is (= table-id (t2/select-one-fn :table_id :workspace_input :id wi-id))))+ (testing "workspace_input_external table_id backfilled"+ (is (= table-id (t2/select-one-fn :table_id :workspace_input_external :id wie-id))))+ (testing "Non-matching input stays nil"+ (is (nil? (t2/select-one-fn :table_id :workspace_input_external :id wie-no-match-id))))))))diff --git a/test/metabase/sync/sync_metadata/tables_test.clj b/test/metabase/sync/sync_metadata/tables_test.cljindex 4482bcc8c92..faef2d2bd28 100644--- a/test/metabase/sync/sync_metadata/tables_test.clj+++ b/test/metabase/sync/sync_metadata/tables_test.clj@@ -213,6 +213,31 @@(is (= "active_table" (:name active-table)))(is (true? (:active active-table))))))))+(deftest archive-tables-skips-transform-targets-test+ (testing "Tables with transform_target=true should never be archived, even if old enough"+ (mt/with-temp [:model/Database db {}+ :model/Table provisional {:name "transform_output"+ :db_id (u/the-id db)+ :active false+ :transform_target true+ :deactivated_at (t/minus (t/offset-date-time) (t/days 30))}+ :model/Table normal {:name "old_table"+ :db_id (u/the-id db)+ :active false+ :transform_target false+ :deactivated_at (t/minus (t/offset-date-time) (t/days 30))}]+ (#'sync-tables/archive-tables! db)++ (testing "Transform target table is not archived or renamed"+ (let [table (t2/select-one :model/Table (:id provisional))]+ (is (nil? (:archived_at table)))+ (is (= "transform_output" (:name table)))))++ (testing "Normal table is archived as usual"+ (let [table (t2/select-one :model/Table (:id normal))]+ (is (some? (:archived_at table)))+ (is (str/starts-with? (:name table) "old_table__mbarchiv__")))))))+(deftest archive-tables-already-archived-test(testing "Already archived tables should not be processed again"(mt/with-temp [:model/Database db {}diff --git a/test/metabase/sync/util_test.clj b/test/metabase/sync/util_test.cljindex 6e262d0affc..4aaea4abec6 100644--- a/test/metabase/sync/util_test.clj+++ b/test/metabase/sync/util_test.clj@@ -300,7 +300,7 @@(testing "If `initial-sync-status` on a table is `incomplete`, it is marked as `complete` after the sync-fks stephas finished"- (let [table-id (t2/select-one-fn :id :model/Table :db_id (mt/id))+ (let [table-id (t2/select-one-fn :id :model/Table :db_id (mt/id) :active true)_ (t2/update! :model/Table table-id {:initial_sync_status "incomplete"})_table (t2/select-one :model/Table :id table-id)](sync/sync-database! (mt/db))@@ -309,7 +309,7 @@(testing "Database and table syncs are marked as complete even if the initial scan is :schema only"(let [_ (t2/update! :model/Database (mt/id) {:initial_sync_status "incomplete"})db (t2/select-one :model/Database :id (mt/id))- table-id (t2/select-one-fn :id :model/Table :db_id (mt/id))+ table-id (t2/select-one-fn :id :model/Table :db_id (mt/id) :active true)_ (t2/update! :model/Table table-id {:initial_sync_status "incomplete"})_table (t2/select-one :model/Table :id table-id)](sync/sync-database! db {:scan :schema})@@ -338,7 +338,7 @@;; incomplete, but then marked as complete after the sync is finished.(mt/dataset test-data(testing "If `initial-sync-status` on a DB is already `complete`"- (let [[active-table inactive-table] (t2/select :model/Table :db_id (mt/id))+ (let [[active-table inactive-table] (t2/select :model/Table :db_id (mt/id) :active true)get-active-table #(t2/select-one :model/Table :id (:id active-table))get-inactive-table #(t2/select-one :model/Table :id (:id inactive-table))](t2/update! :model/Table (:id active-table) {:initial_sync_status "complete" :active true})diff --git a/test/metabase/transforms/test_util.clj b/test/metabase/transforms/test_util.cljindex 07402356171..1597310ffd5 100644--- a/test/metabase/transforms/test_util.clj+++ b/test/metabase/transforms/test_util.clj@@ -21,6 +21,13 @@(let [{:keys [db_id schema]} (t2/select-one [:model/Table :db_id :schema] :id table-id)]{:alias alias :table_id table-id :database_id db_id :schema schema}))+(defn default-source-table-entry+ "Build a source-table-entry for the first active table in the current test database."+ ([]+ (default-source-table-entry "test"))+ ([alias]+ (source-table-entry alias (t2/select-one-pk :model/Table :db_id (mt/id) :active true))))+(defn drop-target!"Drop transform target `target` and clean up its metadata.`target` can be a string or a map. If `target` is a string, type :table is assumed.diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.cljindex 13fbf0029e1..ab8554da214 100644--- a/test/metabase/transforms/util_test.clj+++ b/test/metabase/transforms/util_test.clj@@ -210,10 +210,10 @@result (transforms-base.u/normalize-source-tables source-tables)](is (= 999 (:table_id (first result))))))- (testing "leaves table_id nil for non-existent table ref"+ (testing "creates transform target table for non-existent table ref"(let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "nonexistent"}]result (transforms-base.u/normalize-source-tables source-tables)]- (is (nil? (:table_id (first result))))))+ (is (int? (:table_id (first result))))))(testing "handles entries needing different kinds of enrichment"(let [source-tables [{:alias "t1" :table_id (:id t1)}@@ -355,22 +355,24 @@(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)(mt/with-premium-features #{:transforms-basic}(let [target {:type "table" :schema nil :name "test_output_table"}]+ ;; The Transform after-insert hook creates a provisional table for the target,+ ;; so we don't need to create one explicitly.(mt/with-temp [:model/Transform {transform-id :id :as transform}{:target target:source {:type "query"- :query (lib/query (mt/metadata-provider) (mt/mbql-query venues))}}- :model/Table {table-id :id} {:db_id (mt/id) :name "test_output_table" :schema nil}]- ;; Mock execute-base! to return success without actually running a query,- ;; run-cancelable-transform! to bypass schema creation / cancellation infra,- ;; and sync/indexes to skip driver calls. complete-execution! runs normally- ;; so it sets transform_id on the target table.- (with-redefs [transforms-base.i/execute-base! (constantly {:status :succeeded})- transforms-base.u/sync-target! (constantly nil)- transforms.u/run-cancelable-transform! (fn [_run-id _transform _driver _details run-fn & _opts]- (run-fn (a/promise-chan) nil))]- (transforms.execute/execute! transform {:run-method :manual})- (is (= transform-id- (t2/select-one-fn :transform_id :model/Table :id table-id))))))))))+ :query (lib/query (mt/metadata-provider) (mt/mbql-query venues))}}]+ (let [table-id (t2/select-one-fn :id :model/Table :db_id (mt/id) :name "test_output_table" :schema nil)]+ ;; Mock execute-base! to return success without actually running a query,+ ;; run-cancelable-transform! to bypass schema creation / cancellation infra,+ ;; and sync/indexes to skip driver calls. complete-execution! runs normally+ ;; so it sets transform_id on the target table.+ (with-redefs [transforms-base.i/execute-base! (constantly {:status :succeeded})+ transforms-base.u/sync-target! (constantly nil)+ transforms.u/run-cancelable-transform! (fn [_run-id _transform _driver _details run-fn & _opts]+ (run-fn (a/promise-chan) nil))]+ (transforms.execute/execute! transform {:run-method :manual})+ (is (= transform-id+ (t2/select-one-fn :transform_id :model/Table :id table-id)))))))))))(deftest transform-hydration-test(testing "hydrating :transform on a table"diff --git a/test/metabase/transforms_rest/api/transform_test.clj b/test/metabase/transforms_rest/api/transform_test.cljindex 63c6f839090..4ce6977ff7b 100644--- a/test/metabase/transforms_rest/api/transform_test.clj+++ b/test/metabase/transforms_rest/api/transform_test.clj@@ -1178,7 +1178,7 @@:query {:database (mt/id):type "native":native {:query "SELECT 1"}}}- :target {:type "table" :name "test_table"}})+ :target {:type "table" :name (mt/random-name)}})(mt/user-http-request :rasta :put 403 (str "transform/" (:id transform)){:name "Updated"})(mt/user-http-request :rasta :delete 403 (str "transform/" (:id transform))))@@ -1320,7 +1320,7 @@:type "native":native {:query "SELECT 1"}}}:target {:type "table"- :name "test_table"}}+ :name (mt/random-name)}}:model/TransformTag tag1 {:name "update-tag-1"}:model/TransformTag tag2 {:name "update-tag-2"}:model/TransformTag tag3 {:name "update-tag-3"}]diff --git a/test/metabase/xrays/automagic_dashboards/core_test.clj b/test/metabase/xrays/automagic_dashboards/core_test.cljindex 5b0a77ad3a4..e12f8cc297b 100644--- a/test/metabase/xrays/automagic_dashboards/core_test.clj+++ b/test/metabase/xrays/automagic_dashboards/core_test.clj@@ -217,7 +217,7 @@(mt/with-test-user :rasta(automagic-dashboards.test/with-rollback-only-transaction(doseq [[table cardinality] (map vector- (t2/select :model/Table :db_id (mt/id) {:order-by [[:name :asc]]})+ (t2/select :model/Table :db_id (mt/id) :active true {:order-by [[:name :asc]]})[2 8 11 11 15 17 5 7])](test-automagic-analysis table cardinality)))))diff --git a/test/metabase/xrays/transforms/core_test.clj b/test/metabase/xrays/transforms/core_test.cljindex 67964cbae5c..1a6d1ac0acd 100644--- a/test/metabase/xrays/transforms/core_test.clj+++ b/test/metabase/xrays/transforms/core_test.clj@@ -65,7 +65,7 @@(deftest ^:parallel tableset-test(testing "Can we get a tableset for a given schema?"- (is (= (t2/select-pks-set :model/Table :db_id (mt/id))+ (is (= (t2/select-pks-set :model/Table :db_id (mt/id) :schema "PUBLIC")(set (map u/the-id (#'tf/tableset (mt/id) "PUBLIC")))))))(deftest ^:parallel find-tables-with-domain-entity-testdiff --git a/test_resources/serialization_baseline/databases/My Database/schemas/public/tables/target_table/target_table.yaml b/test_resources/serialization_baseline/databases/My Database/schemas/public/tables/target_table/target_table.yamlnew file mode 100644index 00000000000..0765397063b--- /dev/null+++ b/test_resources/serialization_baseline/databases/My Database/schemas/public/tables/target_table/target_table.yaml@@ -0,0 +1,34 @@+name: target_table+display_name: Target Table+description: null+created_at: '2026-03-09T17:33:23.746321Z'+db_id: My Database+schema: public+entity_type: null+active: false+is_upload: false+field_order: database+visibility_type: null+show_in_getting_started: false+initial_sync_status: complete+points_of_interest: null+caveats: null+serdes/meta:+- id: My Database+ model: Database+- id: public+ model: Schema+- id: target_table+ model: Table+archived_at: null+collection_id: null+data_authority: computed+data_layer: internal+data_source: metabase-transform+database_require_filter: null+deactivated_at: null+is_published: false+is_writable: null+owner_email: null+owner_user_id: null+transform_id: null