Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Create table entries for transform targets on save (#70552)

Chris Truter7ecba8f6d49b
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index df5c6b0fea3..6126b70269d 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1676,7 +1676,8 @@
lib-be
query-processor
sync
- util}
+ util
+ warehouse-schema}
:friends #{transforms
enterprise/transforms-python
enterprise/workspaces}}
@@ -1847,7 +1848,8 @@
remote-sync
search
util
- warehouses}}
+ warehouses
+ workspaces}}
warehouse-schema-rest
{:team "UX West"
@@ -1921,6 +1923,11 @@
warehouses
enterprise/advanced-permissions}}
+ workspaces
+ {:team "Metabot"
+ :api #{metabase.workspaces.core}
+ :uses #{util}}
+
xrays
{:team "Querying"
:api #{metabase.xrays.api
@@ -2642,6 +2649,7 @@
transforms
transforms-base
util
+ warehouse-schema
enterprise/transforms-python}}
enterprise/writable-connection
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj b/enterprise/backend/src/metabase_enterprise/workspaces/api.clj
index 53b3fc1007c..89f98883867 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/api.clj
@@ -49,7 +49,8 @@
[:database {:optional true} ::ws.t/appdb-id]
[:type [:enum "table"]]
[:schema {:optional true} [:maybe [:string {:min 1}]]]
- [:name [:string {:min 1}]]])
+ [:name [:string {:min 1}]]
+ [:table_id {:optional true} [:maybe ::ws.t/appdb-id]]])
(def ^:private InputTable
[:map
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/api/common.clj b/enterprise/backend/src/metabase_enterprise/workspaces/api/common.clj
index ed029872d4c..a7ade2153c4 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/api/common.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/api/common.clj
@@ -26,6 +26,7 @@
[metabase.util :as u]
[metabase.util.i18n :refer [deferred-tru]]
[metabase.util.malli.registry :as mr]
+ [metabase.warehouse-schema.models.table :as table]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -79,7 +80,7 @@
;;; ------------------------------------------------- Utilities ------------------------------------------------------
-(defn flag-enabled?
+(defn- flag-enabled?
"Coerce a flag parameter (true, false, or 1) to boolean."
[v]
(boolean (#{1 true} v)))
@@ -230,6 +231,10 @@
;; Check for internal target conflict AFTER adding database to target
_ (api/check-400 (not (internal-target-conflict? ws-id (:target body)))
(deferred-tru "Another transform in this workspace already targets that table"))
+ ;; Eagerly create global target table row so it's immediately visible
+ {:keys [database schema name]} (:target body)
+ _ (when (and database name)
+ (table/upsert-transform-target-table! database schema name))
transform (ws.common/add-to-changeset! api/*current-user-id* workspace :transform global-id body
:ref-id ref-id)]
(-> (select-malli-keys WorkspaceTransform workspace-transform-alias transform)
@@ -406,14 +411,18 @@
;; Build a map of [d s t] => id for every table that has been synced since the output row was written.
fallback-map (merge
(ws.impl/table-ids-fallbacks :global_schema :global_table :global_table_id all-outputs)
- (ws.impl/table-ids-fallbacks :isolated_schema :isolated_table :isolated_table_id all-outputs))]
+ (ws.impl/table-ids-fallbacks :isolated_schema :isolated_table :isolated_table_id all-outputs))
+ table-ids (into (into #{} (keep :isolated_table_id) all-outputs) (vals fallback-map))
+ active-table-ids (t2/select-fn-set :id [:model/Table :id] :id [:in table-ids] :active true)]
{:inputs (sort-by (juxt :db_id :schema :table) inputs)
:outputs (sort-by
(juxt :db_id (comp (juxt :schema :table) :global))
(concat
;; Workspace transform outputs
(for [{:keys [ref_id db_id global_schema global_table global_table_id
- isolated_schema isolated_table isolated_table_id]} outputs]
+ isolated_schema isolated_table isolated_table_id]} outputs
+ :let [isolated_table_id (or isolated_table_id
+ (get fallback-map [db_id isolated_schema isolated_table]))]]
{:db_id db_id
:global {:transform_id nil
:schema global_schema
@@ -422,10 +431,14 @@
:isolated {:transform_id ref_id
:schema isolated_schema
:table isolated_table
- :table_id (or isolated_table_id (get fallback-map [db_id isolated_schema isolated_table]))}})
+ :table_id isolated_table_id
+ :active (contains? active-table-ids isolated_table_id)}})
+
;; External transform outputs
(for [{:keys [transform_id db_id global_schema global_table global_table_id
- isolated_schema isolated_table isolated_table_id]} external-outputs]
+ isolated_schema isolated_table isolated_table_id]} external-outputs
+ :let [isolated_table_id (or isolated_table_id
+ (get fallback-map [db_id isolated_schema isolated_table]))]]
{:db_id db_id
:global {:transform_id transform_id
:schema global_schema
@@ -434,7 +447,8 @@
:isolated {:transform_id transform_id
:schema isolated_schema
:table isolated_table
- :table_id (or isolated_table_id (get fallback-map [db_id isolated_schema isolated_table]))}})))}))
+ :table_id isolated_table_id
+ :active (contains? active-table-ids isolated_table_id)}})))}))
(defn get-workspace-log
"Handler body for GET /:ws-id/log."
@@ -593,6 +607,7 @@
(let [workspace (t2/select-one :model/Workspace :id ws-id)
_ (api/check-404 workspace)
_ (api/check-400 (not= :archived (:base_status workspace)) "Cannot execute archived workspace")
+ _ (check-transforms-enabled! (:database_id workspace))
graph (ws.impl/get-or-calculate-graph! workspace)]
(ws.impl/execute-workspace! workspace graph {:stale-only? (flag-enabled? stale_only)})))
@@ -673,7 +688,11 @@
;; If target is changing, check for conflicts with other transforms (excluding this one)
(when (:target body)
(api/check-400 (not (internal-target-conflict? ws-id (:target merged-body) tx-id))
- (deferred-tru "Another transform in this workspace already targets that table")))
+ (deferred-tru "Another transform in this workspace already targets that table"))
+ ;; Eagerly create global target table row so it's immediately visible
+ (let [{:keys [database schema name]} (:target merged-body)]
+ (when (and database name)
+ (table/upsert-transform-target-table! database schema name))))
(t2/update! :model/WorkspaceTransform {:workspace_id ws-id :ref_id tx-id} merged-body)
;; If source or target changed, increment versions for re-analysis
(when source-or-target-changed?
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj b/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj
index 5f4f6a732dc..68fc1720471 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj
@@ -55,9 +55,6 @@
[metabase.app-db.core :as app-db]
[metabase.driver :as driver]
[metabase.driver.sql :as driver.sql]
- ;; TODO (Chris 2025-12-17) -- I solemnly declare that we will clean up this coupling nightmare for table normalization
- ^{:clj-kondo/ignore [:metabase/modules]}
- [metabase.driver.sql.normalize :as sql.normalize]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.util :as u]
@@ -65,6 +62,7 @@
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.util.performance :as perf]
+ [metabase.warehouse-schema.models.table :as table]
[toucan2.core :as t2]))
;;; ---------------------------------------- Schemas ----------------------------------------
@@ -226,21 +224,12 @@
Stores both global (original) and isolated (workspace-specific) table identifiers.
With epochal versioning, we always insert new rows - cleanup of old versions happens separately.
Silently ignores constraint violations from concurrent inserts."
- [workspace-id ref-id isolated-schema {:keys [db_id schema table]} normalize-sql transform-version]
+ [workspace-id ref-id isolated-schema {:keys [db_id schema table]} transform-version]
+ ;; 5 queries here: 2 × upsert-transform-target-table! (2 queries each) + 1 insert.
+ ;; Worst case 9 on concurrent upsert conflicts.
(let [isolated-table (ws.u/isolated-table-name schema table)
- qry-table-id (fn [s t]
- (or (t2/select-one-fn :id [:model/Table :id]
- :db_id db_id
- :schema (normalize-sql s)
- :name (normalize-sql t))
- ;; Turns out transforms don't normalize the metadata they create
- ;; TODO (Chris 2026-01-26) -- This is getting really tangled and expensive, revisit
- (t2/select-one-fn :id [:model/Table :id]
- :db_id db_id
- :schema s
- :name t)))
- global-table-id (qry-table-id schema table)
- isolated-table-id (qry-table-id isolated-schema isolated-table)]
+ global-table-id (table/upsert-transform-target-table! db_id schema table)
+ isolated-table-id (table/upsert-transform-target-table! db_id isolated-schema isolated-table)]
(ws.u/ignore-constraint-violation
(t2/insert! :model/WorkspaceOutput
{:workspace_id workspace-id
@@ -264,19 +253,22 @@
"Ensure a workspace_input row exists for the given table coordinate.
Returns the workspace_input id (existing or newly created)."
[workspace-id {:keys [db_id schema table table_id]}]
- (app-db/update-or-insert!
- :model/WorkspaceInput
- {:workspace_id workspace-id
- :db_id db_id
- :schema schema
- :table table}
- (fn [existing]
- (cond-> {:workspace_id workspace-id
- :db_id db_id
- :schema schema
- :table table
- :table_id (:table_id existing)}
- table_id (assoc :table_id table_id)))))
+ (let [table_id (or table_id
+ (t2/select-one-fn :id [:model/Table :id]
+ :db_id db_id :schema schema :name table))]
+ (app-db/update-or-insert!
+ :model/WorkspaceInput
+ {:workspace_id workspace-id
+ :db_id db_id
+ :schema schema
+ :table table}
+ (fn [existing]
+ (cond-> {:workspace_id workspace-id
+ :db_id db_id
+ :schema schema
+ :table table
+ :table_id (:table_id existing)}
+ table_id (assoc :table_id table_id))))))
(defn- insert-workspace-inputs!
"Insert a single workspace_input record per table, with a join entry per transform that uses that table."
@@ -330,11 +322,10 @@
:id [:in {:select [:database_id]
:from [:workspace]
:where [:= :id workspace-id]}])
- normalize (partial sql.normalize/normalize-name driver)
default-schema (driver.sql/default-schema driver)
;; Normalize external inputs so schemas are consistent
normalized-inputs (map (partial normalize-input-schema default-schema) inputs)]
;; Insert inputs first, then output - output row acts as "commit marker" for version check
(insert-workspace-inputs! workspace-id ref-id normalized-inputs transform-version)
- (insert-workspace-output! workspace-id ref-id isolated-schema output normalize transform-version)
+ (insert-workspace-output! workspace-id ref-id isolated-schema output transform-version)
nil))))
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/impl.clj b/enterprise/backend/src/metabase_enterprise/workspaces/impl.clj
index c377b958874..2606b8d5e98 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/impl.clj
@@ -12,6 +12,7 @@
[metabase.driver.sql :as driver.sql]
[metabase.util :as u]
[metabase.util.log :as log]
+ [metabase.warehouse-schema.models.table :as table]
[toucan2.core :as t2]))
(defn- query-ungranted-external-inputs
@@ -566,16 +567,17 @@
(when-not (t2/exists? :model/WorkspaceOutputExternal :workspace_id workspace-id :graph_version [:>= graph-version])
(let [external-tx-ids (extract-external-transform-ids entities)]
(when (seq external-tx-ids)
+ ;; There are 4N + 3 queries for N transforms, we may want to optimize...
+ ;; 1 exists
+ ;; + 1 select
+ ;; + N × 2 upsert-transform-target-table! (2 queries each)
+ ;; + 1 batch insert
+ ;; Worst case 8N + 3 on concurrent upsert conflicts.
(let [transforms (t2/select [:model/Transform :id :target] :id [:in external-tx-ids])
rows (for [{tx-id :id, {:keys [database schema name]} :target} transforms]
(let [isolated-table (ws.u/isolated-table-name schema name)
- ;; TODO (Chris 2026-01-26) -- 2N + 1 is really not great here...
- global-table-id (t2/select-one-fn :id [:model/Table :id]
- :db_id database :schema schema :name name)
- isolated-table-id (t2/select-one-fn :id [:model/Table :id]
- :db_id database
- :schema isolated-schema
- :name isolated-table)]
+ global-table-id (table/upsert-transform-target-table! database schema name)
+ isolated-table-id (table/upsert-transform-target-table! database isolated-schema isolated-table)]
{:workspace_id workspace-id
:transform_id tx-id
:graph_version graph-version
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/validation.clj b/enterprise/backend/src/metabase_enterprise/workspaces/validation.clj
index 2d27343fd84..8dfae6e59eb 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/validation.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/validation.clj
@@ -77,7 +77,8 @@
(defn- workspace-outputs
"Get all workspace outputs with their global and isolated table info.
- Includes both workspace transform outputs and enclosed external transform outputs."
+ Includes both workspace transform outputs and enclosed external transform outputs.
+ Enriches each output with :isolated_active from the referenced metabase_table."
[workspace-id]
;; TODO (Chris 2026-02-02) -- rather fetch all this as an input to graph analysis, and save it in [[graph]].
(let [ws-outputs (t2/select [:model/WorkspaceOutput
@@ -92,8 +93,15 @@
:id :transform_id :db_id
:global_schema :global_table :global_table_id
:isolated_schema :isolated_table :isolated_table_id]
- :workspace_id workspace-id))]
- (concat ws-outputs ext-outputs)))
+ :workspace_id workspace-id))
+ all-outputs (concat ws-outputs ext-outputs)
+ ;; We could avoid this extra query by using a JOIN, but we'd need to define a derived model for this to preserve
+ ;; the model transforms (e.g. char(36) trimming on ref_id), and that feels excessive for a single query.
+ active-ids (t2/select-fn-set :id [:model/Table :id]
+ :id [:in (keep :isolated_table_id all-outputs)]
+ :active true)]
+ (for [output all-outputs]
+ (assoc output :isolated_active (contains? active-ids (:isolated_table_id output))))))
(defn- ids-for-type [entities desired-node-type]
(for [{:keys [node-type id]} entities, :when (= node-type desired-node-type)] id))
@@ -192,7 +200,7 @@
Returns a sequence of problem maps."
[output checked-out-ids enclosed-ids internal-dependents-map]
(let [{:keys [ref_id db_id global_schema global_table global_table_id
- isolated_table_id external?]} output
+ isolated_table_id isolated_active external?]} output
;; For external outputs, ref_id is actually the transform_id (an int)
transform-type (if external? :external-transform :workspace-transform)
external-transforms (external-transform-dependents global_table_id checked-out-ids enclosed-ids)
@@ -202,8 +210,8 @@
table-coord {:db_id db_id :schema global_schema :table global_table}]
(cond
- ;; Case 1: Isolated table doesn't exist yet
- (nil? isolated_table_id)
+ ;; Case 1: Isolated table is not active (no row, hasn't been run, or was deactivated)
+ (not isolated_active)
(cond
;; External transforms depend on this output that doesn't exist
has-external-dependents?
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
index 7ea1fa7c411..30d44d7d5d0 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
@@ -1341,21 +1341,25 @@
:source {:type :query
:query (lib/query mp products)}
:target {:schema "PUBLIC"
- :name "referenced_transform_table"}}
- :model/Table _ {:name "referenced_transform_table"
- :db_id (mt/id)
- :schema "PUBLIC"}]
+ :name "referenced_transform_table"}}]
+ ;; Simulate the referenced transform having been run: activate its provisional target table
+ ;; so the table→transform dep is visible in the dependency graph (which filters active=true).
+ (t2/update! :model/Table {:db_id (mt/id) :schema "PUBLIC" :name "referenced_transform_table"} {:active true})
(events/publish-event! :event/transform-run-complete
{:object {:db-id (mt/id)
:output-schema "PUBLIC"
:output-table "referenced_transform_table"
:transform-id referenced-transform-id}})
(while (#'dependencies.backfill/backfill-dependencies!))
- (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=transform&query=unreftest")]
- (is (=? {:data [{:id unreffed-transform-id
- :type "transform"
- :data {:name "Unreferenced Transform - unreftest"}}]}
- response))))))))
+ (try
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=transform&query=unreftest")]
+ (is (=? {:data [{:id unreffed-transform-id
+ :type "transform"
+ :data {:name "Unreferenced Transform - unreftest"}}]}
+ response)))
+ (finally
+ ;; Clean up provisional table rows created by define-after-insert
+ (t2/delete! :model/Table :db_id (mt/id) :name [:in ["referenced_transform_table" "unreferenced_transform_table"]]))))))))
(deftest ^:sequential unreferenced-snippets-test
(testing "GET /api/ee/dependencies/unreferenced - only unreferenced snippets are returned"
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
index d47ccb1f3a7..897d692b86b 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
@@ -346,21 +346,22 @@
(run-with-dependencies-setup!
(fn [_]
(let [target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]
- (mt/with-temp [:model/Transform {transform-id :id} {:target target}
- :model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}]
- (events/publish-event! :event/transform-run-complete
- {:object {:db-id (mt/id)
- :output-schema "Other"
- :output-table :test_table
- :transform-id transform-id}
- :user-id api/*current-user-id*})
- (is (= models.dependency/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))
- (is (=? [{:from_entity_type :table
- :from_entity_id table-id,
- :to_entity_type :transform,
- :to_entity_id transform-id}]
- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))
+ (mt/with-temp [:model/Transform {transform-id :id} {:target target}]
+ ;; Transform after-insert creates the target table row via upsert-transform-target-table!
+ (let [table-id (t2/select-one-pk :model/Table :db_id (mt/id) :schema "Other" :name "test_table")]
+ (events/publish-event! :event/transform-run-complete
+ {:object {:db-id (mt/id)
+ :output-schema "Other"
+ :output-table :test_table
+ :transform-id transform-id}
+ :user-id api/*current-user-id*})
+ (is (= models.dependency/current-dependency-analysis-version
+ (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))
+ (is (=? [{:from_entity_type :table
+ :from_entity_id table-id,
+ :to_entity_type :transform,
+ :to_entity_id transform-id}]
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))
(deftest ^:sequential python-transform-update-handles-downstream-dependencies-test
(testing "python transform update events handles downstream dependencies"
@@ -378,35 +379,36 @@
target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]
(mt/with-temp [:model/Transform {transform-id :id :as transform} {:target target
:source source}
- :model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}
+ ;; test_table is created by the Transform after-insert via upsert-transform-target-table!
:model/Table {} {:schema "Other", :db_id (mt/id), :name "test_table2"}]
- (testing "initial run"
- (events/publish-event! :event/transform-run-complete
- {:object {:db-id (mt/id)
- :output-schema "Other"
- :output-table :test_table
- :transform-id transform-id}
- :user-id api/*current-user-id*})
- (is (=? [{:from_entity_type :table
- :from_entity_id table-id,
- :to_entity_type :transform,
- :to_entity_id transform-id}]
- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
- (testing "keeping target"
- (events/publish-event! :event/update-transform
- {:object transform
- :user-id api/*current-user-id*})
- (is (=? [{:from_entity_type :table
- :from_entity_id table-id,
- :to_entity_type :transform,
- :to_entity_id transform-id}]
- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
- (testing "changing target update"
- (events/publish-event! :event/update-transform
- {:object (assoc-in transform [:target :name] "test_table2")
- :user-id api/*current-user-id*})
- (is (empty?
- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))
+ (let [table-id (t2/select-one-pk :model/Table :db_id (mt/id) :schema "Other" :name "test_table")]
+ (testing "initial run"
+ (events/publish-event! :event/transform-run-complete
+ {:object {:db-id (mt/id)
+ :output-schema "Other"
+ :output-table :test_table
+ :transform-id transform-id}
+ :user-id api/*current-user-id*})
+ (is (=? [{:from_entity_type :table
+ :from_entity_id table-id,
+ :to_entity_type :transform,
+ :to_entity_id transform-id}]
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
+ (testing "keeping target"
+ (events/publish-event! :event/update-transform
+ {:object transform
+ :user-id api/*current-user-id*})
+ (is (=? [{:from_entity_type :table
+ :from_entity_id table-id,
+ :to_entity_type :transform,
+ :to_entity_id transform-id}]
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
+ (testing "changing target update"
+ (events/publish-event! :event/update-transform
+ {:object (assoc-in transform [:target :name] "test_table2")
+ :user-id api/*current-user-id*})
+ (is (empty?
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))))
(deftest ^:sequential query-transform-update-handles-downstream-dependencies-test
(testing "query transform update events handles downstream dependencies"
@@ -416,35 +418,36 @@
:type :query}
target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]
(mt/with-temp [:model/Transform {transform-id :id :as transform} {:target target :source source}
- :model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}
+ ;; test_table is created by the Transform after-insert via upsert-transform-target-table!
:model/Table {} {:schema "Other", :db_id (mt/id), :name "test_table2"}]
- (testing "initial run"
- (events/publish-event! :event/transform-run-complete
- {:object {:db-id (mt/id)
- :output-schema "Other"
- :output-table :test_table
- :transform-id transform-id}
- :user-id api/*current-user-id*})
- (is (=? [{:from_entity_type :table
- :from_entity_id table-id,
- :to_entity_type :transform,
- :to_entity_id transform-id}]
- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
- (testing "keeping target"
- (events/publish-event! :event/update-transform
- {:object transform
- :user-id api/*current-user-id*})
- (is (=? [{:from_entity_type :table
- :from_entity_id table-id,
- :to_entity_type :transform,
- :to_entity_id transform-id}]
- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
- (testing "changing target update"
- (events/publish-event! :event/update-transform
- {:object (assoc-in transform [:target :name] "test_table2")
- :user-id api/*current-user-id*})
- (is (empty?
- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))
+ (let [table-id (t2/select-one-pk :model/Table :db_id (mt/id) :schema "Other" :name "test_table")]
+ (testing "initial run"
+ (events/publish-event! :event/transform-run-complete
+ {:object {:db-id (mt/id)
+ :output-schema "Other"
+ :output-table :test_table
+ :transform-id transform-id}
+ :user-id api/*current-user-id*})
+ (is (=? [{:from_entity_type :table
+ :from_entity_id table-id,
+ :to_entity_type :transform,
+ :to_entity_id transform-id}]
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
+ (testing "keeping target"
+ (events/publish-event! :event/update-transform
+ {:object transform
+ :user-id api/*current-user-id*})
+ (is (=? [{:from_entity_type :table
+ :from_entity_id table-id,
+ :to_entity_type :transform,
+ :to_entity_id transform-id}]
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
+ (testing "changing target update"
+ (events/publish-event! :event/update-transform
+ {:object (assoc-in transform [:target :name] "test_table2")
+ :user-id api/*current-user-id*})
+ (is (empty?
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))))
(deftest transform-dependency-calculation-error-handling-test
(testing "When transform dependency calculation throws an error, it should be logged and the version should still be updated"
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj
index 86d3c6fd975..50f46f7afbb 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj
@@ -59,7 +59,7 @@
:source {:type "query"
:query (mt/mbql-query nil
{:source-table (str "card__" card-id)})}
- :target {:type "table" :name "test_table"}
+ :target {:type "table" :name (mt/random-name)}
:dependency_analysis_version 0}]
(is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
(is (t2/exists? :model/Transform :id transform-id :dependency_analysis_version 0))
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/create_autogenerated_dashboard_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/create_autogenerated_dashboard_test.clj
index 88db5c9599f..c8c13d42245 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/create_autogenerated_dashboard_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/create_autogenerated_dashboard_test.clj
@@ -32,14 +32,14 @@
(testing "returns final-response? flag"
;; We need to test the structure without actually hitting the database
;; Using mt/with-temp for table creation
- (mt/with-temp [:model/Table table {:name "test_table" :db_id (mt/id)}]
+ (mt/with-temp [:model/Table table {:name (mt/random-name) :db_id (mt/id)}]
(let [result (autogen-dashboard/create-autogenerated-dashboard
{:source {:table_id (:id table)}
:memory-atom (atom {:state {}})})]
(is (true? (:final-response? result))))))
(testing "returns redirect reaction"
- (mt/with-temp [:model/Table table {:name "test_table" :db_id (mt/id)}]
+ (mt/with-temp [:model/Table table {:name (mt/random-name) :db_id (mt/id)}]
(let [result (autogen-dashboard/create-autogenerated-dashboard
{:source {:table_id (:id table)}
:memory-atom (atom {:state {}})})]
@@ -48,7 +48,7 @@
(is (str/starts-with? (:url (first (:reactions result))) "/auto/dashboard/")))))
(testing "returns user-friendly message"
- (mt/with-temp [:model/Table table {:name "test_table" :db_id (mt/id)}]
+ (mt/with-temp [:model/Table table {:name (mt/random-name) :db_id (mt/id)}]
(let [result (autogen-dashboard/create-autogenerated-dashboard
{:source {:table_id (:id table)}
:memory-atom (atom {:state {}})})]
@@ -56,7 +56,7 @@
(is (str/includes? (get-in result [:structured-output :message]) "dashboard")))))
(testing "returns path in structured output"
- (mt/with-temp [:model/Table table {:name "test_table" :db_id (mt/id)}]
+ (mt/with-temp [:model/Table table {:name (mt/random-name) :db_id (mt/id)}]
(let [result (autogen-dashboard/create-autogenerated-dashboard
{:source {:table_id (:id table)}
:memory-atom (atom {:state {}})})]
@@ -66,7 +66,7 @@
(deftest path-generation-test
(testing "table source generates correct path"
- (mt/with-temp [:model/Table table {:name "test_table" :db_id (mt/id)}]
+ (mt/with-temp [:model/Table table {:name (mt/random-name) :db_id (mt/id)}]
(let [result (autogen-dashboard/create-autogenerated-dashboard
{:source {:table_id (:id table)}
:memory-atom (atom {:state {}})})]
diff --git a/enterprise/backend/test/metabase_enterprise/permission_debug/impl_test.clj b/enterprise/backend/test/metabase_enterprise/permission_debug/impl_test.clj
index 9fc89743636..dc137d77170 100644
--- a/enterprise/backend/test/metabase_enterprise/permission_debug/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/permission_debug/impl_test.clj
@@ -5,7 +5,8 @@
[metabase.permissions.core :as perms]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
- [metabase.util.i18n :refer [tru]]))
+ [metabase.util.i18n :refer [tru]]
+ [metabase.warehouse-schema.models.table :as table]))
(use-fixtures :once (fixtures/initialize :db))
@@ -208,6 +209,8 @@
:dataset_query (mt/native-query {:query "SELECT * FROM checkins WHERE date > '2014-01-01'"})}]
(let [regular-user-id (mt/user->id :rasta)]
(perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :blocked)
+ ;; GC leaked provisional table rows so they don't appear in blocked-tables
+ (table/gc-transform-target-tables!)
(let [result (permission-debug.impl/debug-permissions
{:user-id regular-user-id
:model-id (str (:id card))
diff --git a/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj b/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj
index 602ab7d6c76..29cca6615ca 100644
--- a/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj
@@ -199,7 +199,7 @@
:source {:type "query"
:query (mt/native-query {:query "SELECT 1"})}
:target {:type "table"
- :name "test_table"}}
+ :name (mt/random-name)}}
:model/RecentViews _ (recent-view c1 now)]
(appdb.scoring-test/with-index-contents
[{:model "card" :id c1 :name "test card recent"}
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj
index 8b96157d4a6..599216046ef 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj
@@ -98,8 +98,7 @@
" print(\"out2\")"
" print(\"err2\", file=sys.stderr)"
" return pd.DataFrame({'x': [42, 43]})"]
- table-id (t2/select-one-pk :model/Table :db_id (mt/id))
- body {:source_tables [(transforms.tu/source-table-entry "test" table-id)]
+ body {:source_tables [(transforms.tu/default-source-table-entry)]
:code (str/join "\n" program)}
{:keys [error logs output]} (mt/user-http-request :crowberto :post 200 "ee/transforms-python/test-run" body)]
(is (nil? error))
@@ -111,7 +110,7 @@
user :crowberto
features #{:transforms :transforms-python}}}]
(let [source-tables (or source-tables
- [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id) :active true))])
+ [(transforms.tu/default-source-table-entry)])
body (merge {:source_tables source-tables, :code (str/join "\n" program)} extra-opts)]
(mt/with-premium-features features
(mt/user-http-request-full-response user :post "ee/transforms-python/test-run" body))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
index dd63ecdce62..6f095a12bda 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
@@ -5,9 +5,10 @@
[metabase.transforms-base.interface :as transforms-base.i]
[metabase.transforms-base.ordering :as ordering]
[metabase.transforms.test-util :as transforms.tu]
+ [metabase.warehouse-schema.models.table :as ws.table]
[toucan2.core :as t2]))
-(defn- make-transform [query & [name schema]]
+(defn- sql-tx [query & [name schema]]
(let [name (or name (mt/random-name))
schema (or schema (transforms.tu/default-schema-or-public))]
{:source {:type :query
@@ -17,7 +18,7 @@
:name name
:type :table}}))
-(defn- make-python-transform [source-tables & [name schema target-db]]
+(defn- py-tx [source-tables & [name schema target-db]]
(let [name (or name (mt/random-name))
schema (or schema (transforms.tu/default-schema-or-public))
target-db (or target-db (mt/id))]
@@ -37,8 +38,8 @@
(deftest python-transform-basic-dependencies-test
(testing "Python transforms with source-tables dependencies are extracted correctly"
- (mt/with-temp [:model/Transform {t1 :id} (make-python-transform [(transforms.tu/source-table-entry "orders" (mt/id :orders))
- (transforms.tu/source-table-entry "products" (mt/id :products))])]
+ (mt/with-temp [:model/Transform {t1 :id} (py-tx [(transforms.tu/source-table-entry "orders" (mt/id :orders))
+ (transforms.tu/source-table-entry "products" (mt/id :products))])]
(is (= #{{:table (mt/id :orders)}
{:table (mt/id :products)}}
(transform-deps-for-db (t2/select-one :model/Transform :id t1)))))))
@@ -47,12 +48,11 @@
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
(testing "Python transforms are ordered correctly based on source-tables dependencies"
(let [schema (transforms.tu/default-schema-or-public)]
- (mt/with-temp [:model/Table {table1 :id} {:schema schema
- :name "output_1"}
- :model/Field _ {:table_id table1
- :name "foo"}
- :model/Transform {t1 :id} (make-python-transform [(transforms.tu/source-table-entry "orders" (mt/id :orders))] "output_1")
- :model/Transform {t2 :id} (make-python-transform [(transforms.tu/source-table-entry "output_1" table1)] "output_2")]
+ (ws.table/gc-transform-target-tables!)
+ (mt/with-temp [:model/Table {table1 :id} {:schema schema :name "output_1"}
+ :model/Field _ {:table_id table1 :name "foo"}
+ :model/Transform {t1 :id} (py-tx [(transforms.tu/source-table-entry "orders" (mt/id :orders))] "output_1")
+ :model/Transform {t2 :id} (py-tx [(transforms.tu/source-table-entry "output_1" table1)] "output_2")]
(is (= {t1 #{}
t2 #{t1}}
(ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2]])))))))))
@@ -61,18 +61,15 @@
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python)
(testing "Python transforms with multiple table dependencies"
(let [schema (transforms.tu/default-schema-or-public)]
- (mt/with-temp [:model/Table {table1 :id} {:schema schema
- :name "output_1"}
- :model/Field _ {:table_id table1
- :name "foo"}
- :model/Table {table2 :id} {:schema schema
- :name "output_2"}
- :model/Field _ {:table_id table2
- :name "bar"}
- :model/Transform {t1 :id} (make-python-transform [(transforms.tu/source-table-entry "orders" (mt/id :orders))] "output_1")
- :model/Transform {t2 :id} (make-python-transform [(transforms.tu/source-table-entry "products" (mt/id :products))] "output_2")
- :model/Transform {t3 :id} (make-python-transform [(transforms.tu/source-table-entry "output_1" table1)
- (transforms.tu/source-table-entry "output_2" table2)] "final_output")]
+ (ws.table/gc-transform-target-tables!)
+ (mt/with-temp [:model/Table {table1 :id} {:schema schema :name "output_1"}
+ :model/Field _ {:table_id table1 :name "foo"}
+ :model/Table {table2 :id} {:schema schema :name "output_2"}
+ :model/Field _ {:table_id table2 :name "bar"}
+ :model/Transform {t1 :id} (py-tx [(transforms.tu/source-table-entry "orders" (mt/id :orders))] "output_1")
+ :model/Transform {t2 :id} (py-tx [(transforms.tu/source-table-entry "products" (mt/id :products))] "output_2")
+ :model/Transform {t3 :id} (py-tx [(transforms.tu/source-table-entry "output_1" table1)
+ (transforms.tu/source-table-entry "output_2" table2)] "final_output")]
(is (= {t1 #{}
t2 #{}
t3 #{t1 t2}}
@@ -82,28 +79,22 @@
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/python :transforms/table)
(testing "Python and query transforms are ordered together correctly"
(let [schema (transforms.tu/default-schema-or-public)]
- (mt/with-temp [:model/Table {table1 :id} {:schema schema
- :name "sql_output"}
- :model/Field _ {:table_id table1
- :name "foo"}
- :model/Table {table2 :id} {:schema schema
- :name "python_output"}
- :model/Field _ {:table_id table2
- :name "bar"}
+ (ws.table/gc-transform-target-tables!)
+ (mt/with-temp [:model/Table {table1 :id} {:schema schema :name "sql_output"}
+ :model/Field _ {:table_id table1 :name "foo"}
+ :model/Table {table2 :id} {:schema schema :name "python_output"}
+ :model/Field _ {:table_id table2 :name "bar"}
;; SQL transform that depends on orders table
- :model/Transform {t1 :id} (make-transform
- {:database (mt/id),
- :type "query",
- :query {:source-table (mt/id :orders)}}
- "sql_output")
+ :model/Transform {t1 :id} (sql-tx {:database (mt/id)
+ :type "query"
+ :query {:source-table (mt/id :orders)}}
+ "sql_output")
;; Python transform that depends on the SQL transform's output
- :model/Transform {t2 :id} (make-python-transform [(transforms.tu/source-table-entry "sql_output" table1)] "python_output")
+ :model/Transform {t2 :id} (py-tx [(transforms.tu/source-table-entry "sql_output" table1)] "python_output")
;; Another SQL transform that depends on the Python transform's output
- :model/Transform {t3 :id} (make-transform
- {:database (mt/id)
- :type "query"
- :query {:source-table table2}}
- "final_output")]
+ :model/Transform {t3 :id} (sql-tx {:database (mt/id)
+ :type "query"
+ :query {:source-table table2}} "final_output")]
(is (= {t1 #{}
t2 #{t1}
t3 #{t2}}
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
index fa62b6f6c38..76496d6afa4 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
@@ -223,7 +223,7 @@
:source {:type "python"
:body (program->source program)
:source-database (mt/id)
- :source-tables [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id)))]}
+ :source-tables [(transforms.tu/default-source-table-entry)]}
:target (assoc target :database (mt/id))})))
(block-on-run [{:keys [expect-status]} target transform-id]
@@ -329,7 +329,7 @@
:model/Transform transform {:name "Python Transform Cross DB"
:source {:type "python"
:source-database (mt/id)
- :source-tables [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id)))]
+ :source-tables [(transforms.tu/default-source-table-entry)]
:body "def transform():\n pass"}
:target {:type "table"
:schema "PUBLIC"
@@ -526,7 +526,7 @@
(let [initial-transform {:name "Schema Change Integration Test"
:source {:type "python"
:source-database (mt/id)
- :source-tables [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id)))]
+ :source-tables [(transforms.tu/default-source-table-entry)]
:body (str "import pandas as pd\n"
"\n"
"def transform():\n"
@@ -545,7 +545,7 @@
(let [updated-transform (assoc initial-transform
:source {:type "python"
:source-database (mt/id)
- :source-tables [(transforms.tu/source-table-entry "test" (t2/select-one-pk :model/Table :db_id (mt/id)))]
+ :source-tables [(transforms.tu/default-source-table-entry)]
:body (str "import pandas as pd\n"
"\n"
"def transform():\n"
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj
index 2f54fbae89c..0e260d7f240 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj
@@ -834,7 +834,7 @@
:isolated {:transform_id ref-id
#_#_:schema (:schema workspace)
:table string?
- :table_id nil}}]}
+ :table_id int?}}]}
(mt/user-http-request :crowberto :get 200 (ws-url (:id workspace) "/table")))))
(testing "and after we run the transform, id for isolated table appears"
(is (=? {:status "succeeded"}
@@ -1804,8 +1804,8 @@
(testing "fallback populates global table_id from metabase_table"
(is (= (:id table)
(-> result :outputs first :global :table_id))))
- (testing "isolated table_id remains nil when table doesn't exist"
- (is (nil? (-> result :outputs first :isolated :table_id))))))))))
+ (testing "isolated table_id is populated with transform target table"
+ (is (int? (-> result :outputs first :isolated :table_id))))))))))
(deftest ^:parallel tables-endpoint-fallback-isolated-table-id-test
(testing "GET /api/ee/workspace/:id/table uses fallback for null isolated_table_id"
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj
index 7a1f59a73f3..8ac5eabea6f 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj
@@ -57,7 +57,7 @@
(is (=? [{:workspace_id (:id workspace)
:global_table output-table
:global_schema nil
- :global_table_id nil
+ :global_table_id pos-int?
:isolated_schema string?
:isolated_table string?
:isolated_table_id number?}]
diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/pages/WorkspacePage/DataTab/DataTabSidebar.tsx b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/pages/WorkspacePage/DataTab/DataTabSidebar.tsx
index 4af6cd210cd..674f1148017 100644
--- a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/pages/WorkspacePage/DataTab/DataTabSidebar.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/pages/WorkspacePage/DataTab/DataTabSidebar.tsx
@@ -150,6 +150,7 @@ export const DataTabSidebar = ({
hasChanges={hasChanges}
transform={workspaceTransform}
tableId={tableId ?? undefined}
+ tableActive={table.isolated.active ?? false}
isSelected={tableId === selectedTableId}
isRunning={
workspaceTransform
diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/pages/WorkspacePage/DataTab/TableListItem.tsx b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/pages/WorkspacePage/DataTab/TableListItem.tsx
index d1a59b041e6..284778e20f3 100644
--- a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/pages/WorkspacePage/DataTab/TableListItem.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/pages/WorkspacePage/DataTab/TableListItem.tsx
@@ -27,6 +27,7 @@ type TableListItemProps = {
hasChanges?: boolean;
transform?: WorkspaceTransformListItem;
tableId?: TableId;
+ tableActive?: boolean;
isSelected?: boolean;
isRunning?: boolean;
readOnly?: boolean;
@@ -43,6 +44,7 @@ export const TableListItem = ({
hasChanges = false,
transform,
tableId,
+ tableActive = false,
isSelected = false,
isRunning = false,
readOnly = false,
@@ -59,29 +61,30 @@ export const TableListItem = ({
}
};
+ const tableExists = !!tableId && (type !== "output" || tableActive);
+
const handleTableClick = () => {
if (isRunning) {
return;
}
- if (tableId && onTableClick) {
+ if (tableExists && tableId && onTableClick) {
onTableClick({ tableId, name, schema, transformId: transform?.ref_id });
} else if (type === "output" && transform && onRunTransform && !readOnly) {
onRunTransform(transform);
}
};
- const hasResults = type === "output" && !!tableId;
const canRunTransform =
- type === "output" && !hasResults && transform && onRunTransform;
- const isClickable = (tableId && onTableClick) || canRunTransform;
- const displayTooltip = type !== "input" && !hasResults && !isRunning;
+ type === "output" && !tableExists && transform && onRunTransform;
+ const isClickable = (tableExists && onTableClick) || canRunTransform;
+ const displayTooltip = type !== "input" && !tableExists && !isRunning;
return (
<Box
className={cx(S.root, {
[S.selected]: isSelected,
[S.clickable]: isClickable,
- [S.noResults]: type === "output" && !tableId,
+ [S.noResults]: type === "output" && !tableExists,
})}
aria-label={displayName}
onClick={isClickable ? handleTableClick : undefined}
diff --git a/frontend/src/metabase-types/api/workspace.ts b/frontend/src/metabase-types/api/workspace.ts
index 504a2acdfde..ac206b5d2a2 100644
--- a/frontend/src/metabase-types/api/workspace.ts
+++ b/frontend/src/metabase-types/api/workspace.ts
@@ -250,6 +250,7 @@ export type WorkspaceOutputTableEntry = {
schema: string;
table: string;
table_id: number | null;
+ active?: boolean;
};
export type WorkspaceOutputTable = {
diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.clj
index 9c22f5cd006..e16e9848a84 100644
--- a/mage/src/mage/modules.clj
+++ b/mage/src/mage/modules.clj
@@ -137,7 +137,8 @@
users
util
version
- view-log})
+ view-log
+ warehouse-schema})
(defn- affected-modules
"Set of modules that are direct or indirect dependents of `modules`, and thus are affected by changes to them.
diff --git a/resources/migrations/060/20260307_table_transform_target.yaml b/resources/migrations/060/20260307_table_transform_target.yaml
new file mode 100644
index 00000000000..f47fae91ff1
--- /dev/null
+++ b/resources/migrations/060/20260307_table_transform_target.yaml
@@ -0,0 +1,35 @@
+## Add transform_target column to metabase_table and backfill existing transforms
+
+databaseChangeLog:
+ - logicalFilePath: migrations/060_update_migrations.yaml
+ - changeSet:
+ id: v60.2026-03-07T00:00:03
+ author: christruter
+ comment: Add transform_target column to metabase_table
+ changes:
+ - addColumn:
+ tableName: metabase_table
+ columns:
+ - column:
+ name: transform_target
+ type: ${boolean.type}
+ defaultValueBoolean: false
+ constraints:
+ nullable: false
+ remarks: >-
+ Whether any transform is configured to target this table. Conservative: eagerly set to true
+ when a transform targets this table (never stale in this direction), cleared asynchronously
+ when no transforms reference it anymore (may have false positives, never false negatives).
+ rollback:
+ - dropColumn:
+ tableName: metabase_table
+ columnName: transform_target
+
+ - changeSet:
+ id: v60.2026-03-07T00:00:04
+ author: christruter
+ comment: Backfill transform target tables for existing transforms and workspace outputs
+ changes:
+ - customChange:
+ class: metabase.app_db.custom_migrations.BackfillTransformTargetTables
+ rollback:
diff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.clj
index 7eedb67395e..7987c43ba24 100644
--- a/src/metabase/app_db/custom_migrations.clj
+++ b/src/metabase/app_db/custom_migrations.clj
@@ -2051,3 +2051,85 @@
(t2/query {:update :transform
:set {:source (json-in (dissoc parsed :source-incremental-strategy))}
:where [:= :id id]}))))))
+
+(define-migration BackfillTransformTargetTables
+ ;; For each transform that has a target (database, schema, name), ensure a metabase_table row exists.
+ ;; If the table already exists (active or inactive), do nothing. Otherwise, insert a transform target row.
+ ;;
+ ;; Similarly, backfill workspace_output rows missing global_table_id or isolated_table_id.
+ (let [;; Reproduces humanization/name->human-readable-name :simple (can't call library code in migrations)
+ acronyms #{"id" "url" "ip" "uid" "uuid" "guid"}
+ cap-word (fn [word]
+ (let [lower (lower-case-en word)]
+ (if (contains? acronyms lower)
+ (upper-case-en word)
+ (if (= word (upper-case-en word))
+ (str/capitalize word)
+ (str (str/capitalize (subs word 0 1)) (subs word 1))))))
+ humanize (fn [s]
+ (when (seq s)
+ (let [result (str/join " " (for [part (str/split s #"[-_\s]+")
+ :when (not (str/blank? part))]
+ (cap-word part)))]
+ (if (str/blank? result) s result))))
+ find-table-id (fn [db-id schema table-name]
+ (:id (first (t2/query {:select [:id]
+ :from [:metabase_table]
+ :where [:and
+ [:= :db_id db-id]
+ (if (some? schema)
+ [:= [:lower :schema] (lower-case-en schema)]
+ [:is :schema nil])
+ [:= [:lower :name] (lower-case-en table-name)]]
+ :limit 1}))))
+ upsert-table! (fn [db-id schema table-name]
+ (or (find-table-id db-id schema table-name)
+ (try
+ (t2/query {:insert-into :metabase_table
+ :values [{:db_id db-id
+ :schema schema
+ :name table-name
+ :display_name (humanize table-name)
+ :active false
+ :transform_target true
+ :data_source "metabase-transform"
+ :data_authority "computed"
+ :initial_sync_status "complete"
+ :created_at :%now
+ :updated_at :%now}]})
+ (find-table-id db-id schema table-name)
+ (catch Exception _
+ (find-table-id db-id schema table-name)))))]
+ ;; 1. Backfill for transforms: create provisional tables for targets that don't exist
+ (doseq [{:keys [target target_db_id]} (t2/query {:select [:target :target_db_id]
+ :from [:transform]
+ :where [:not= :target_db_id nil]})]
+ (let [target-map (json-out target false)]
+ (when-let [table-name (get target-map "name")]
+ (let [schema (get target-map "schema")
+ db-id target_db_id]
+ (upsert-table! db-id schema table-name)))))
+ ;; 2-5. Backfill workspace_output and workspace_output_external rows missing table FKs
+ (let [backfill-table-fk! (fn [from-table schema-col table-col fk-col]
+ (doseq [{:keys [id db_id] :as row}
+ (t2/query {:select [:id :db_id schema-col table-col]
+ :from [from-table]
+ :where [:= fk-col nil]})]
+ (when-let [table-id (upsert-table! db_id (get row schema-col) (get row table-col))]
+ (t2/query {:update from-table
+ :set {fk-col table-id}
+ :where [:= :id id]}))))]
+ (backfill-table-fk! :workspace_output :global_schema :global_table :global_table_id)
+ (backfill-table-fk! :workspace_output :isolated_schema :isolated_table :isolated_table_id)
+ (backfill-table-fk! :workspace_output_external :global_schema :global_table :global_table_id)
+ (backfill-table-fk! :workspace_output_external :isolated_schema :isolated_table :isolated_table_id))
+ ;; 6-7. Backfill workspace_input and workspace_input_external rows missing table_id
+ (doseq [from-table [:workspace_input :workspace_input_external]
+ :let [rows (t2/query {:select [:id :db_id :schema :table]
+ :from [from-table]
+ :where [:= :table_id nil]})]
+ {:keys [id db_id schema table]} rows]
+ (when-let [table-id (find-table-id db_id schema table)]
+ (t2/query {:update from-table
+ :set {:table_id table-id}
+ :where [:= :id id]})))))
diff --git a/src/metabase/app_db/query.clj b/src/metabase/app_db/query.clj
index 93bcde0d750..51303e02bd4 100644
--- a/src/metabase/app_db/query.clj
+++ b/src/metabase/app_db/query.clj
@@ -187,7 +187,9 @@
(merge instance select-map))]
(with-conflict-retry
(or (apply t2/select-one model select-kvs)
- (t2/insert-returning-instance! model (insert-fn))))))
+ ;; Wrap INSERT in a savepoint — see comment in [[update-or-insert!]].
+ (t2/with-transaction [_conn]
+ (t2/insert-returning-instance! model (insert-fn)))))))
(defn update-or-insert!
"Update a database record, if it exists, otherwise create it.
@@ -200,6 +202,9 @@
Note that the generated entity must be consistent with `select-map`, if it disagrees on any keys then an exception
will be thrown. It is OK for the entity to omit fields from `select-map`, they will implicitly be added on.
+ When called with an existing record, `update-fn` may return nil to signal that no update is needed. In this case
+ the existing row is left untouched and its primary key is returned. This is useful for \"get-or-create\" patterns.
+
This is more general than using `UPSERT`, `MERGE` or `INSERT .. ON CONFLICT`, and it also allows one to avoid
calculating initial values that may be expensive, or require side effects.
@@ -217,16 +222,25 @@
pk-key (keyword (first pks))
update-fn (fn [existing]
(let [updated (update-fn existing)]
- ;; the inserted / updated values must be consistent with the select query
- (assert (not (u/conflicting-keys? select-map updated))
- "This should not be used to change any of the identifying values")
- ;; For convenience, we allow the update-fn to omit fields in the search-map
- (merge updated select-map)))]
+ ;; When called with an existing record, returning nil signals "no update needed".
+ (when (or updated (not existing))
+ ;; the inserted / updated values must be consistent with the select query
+ (assert (not (u/conflicting-keys? select-map updated))
+ "This should not be used to change any of the identifying values")
+ ;; For convenience, we allow the update-fn to omit fields in the search-map
+ (merge updated select-map))))]
(with-conflict-retry
(if-let [existing (apply t2/select-one model select-kvs)]
- (let [pk (pk-key existing)
- updated (update-fn existing)]
- (t2/update! model pk updated)
- ;; the private key may have been changed by the update, and this is OK.
- (pk-key updated pk))
- (t2/insert-returning-pk! model (update-fn nil))))))
+ (let [pk (pk-key existing)]
+ (if-let [updated (update-fn existing)]
+ (do (t2/update! model pk updated)
+ ;; the primary key may have been changed by the update, and this is OK.
+ (pk-key updated pk))
+ ;; update-fn returned nil — no update needed, return existing pk.
+ pk))
+ ;; Wrap INSERT in a savepoint so that a constraint violation inside an existing
+ ;; PostgreSQL transaction doesn't abort the outer transaction. The savepoint is
+ ;; rolled back on failure, keeping the outer transaction usable for the retry
+ ;; in [[with-conflict-retry]].
+ (t2/with-transaction [_conn]
+ (t2/insert-returning-pk! model (update-fn nil)))))))
diff --git a/src/metabase/models/transforms/transform.clj b/src/metabase/models/transforms/transform.clj
index 06c50b9a61b..704294dd47e 100644
--- a/src/metabase/models/transforms/transform.clj
+++ b/src/metabase/models/transforms/transform.clj
@@ -108,11 +108,16 @@
;; In practice, our serialized representation should not contain any database ids, and this should
;; not be required.
;; TODO (Chris 2026-02-02) -- Update tests so this workaround is unnecessary.
- valid-db-id? (and target-db-id (t2/exists? :model/Database :id target-db-id))]
+ valid-db-id? (and target-db-id (t2/exists? :model/Database :id target-db-id))
+ target-name (get-in transform [:target :name])
+ table-id (when (and valid-db-id? target-name)
+ (transforms-base.u/upsert-target-table!
+ target-db-id (get-in transform [:target :schema]) target-name))]
(when-not valid-db-id?
(log/warnf "Invalid target database id (%s) ignored for new transform (%s)" target-db-id (:name transform)))
(-> transform
(assoc-in [:target :database] target-db-id)
+ (cond-> table-id (assoc-in [:target :table_id] table-id))
(assoc
:source_type (transforms-base.u/transform-source-type source)
:target_db_id (when valid-db-id? target-db-id)
@@ -123,15 +128,27 @@
(when-let [new-collection (:collection_id (t2/changes transform))]
(collection/check-collection-namespace :model/Transform new-collection)
(collection/check-allowed-content :model/Transform new-collection))
- (let [changes (t2/changes transform)]
+ ;; The target db is recomputed when source changes because for MBQL transforms,
+ ;; the source query's :database is the source of truth for the target database.
+ (let [target-changed? (or (:source (t2/changes transform)) (:target (t2/changes transform)))
+ target-db-id (when target-changed?
+ ;; No database existence check added here, unlike for insert.
+ ;; Just allow updates for an invalid target to fail.
+ (transforms-base.i/target-db-id transform))
+ target-name (get-in transform [:target :name])
+ table-id (when (and target-db-id target-name)
+ (transforms-base.u/upsert-target-table!
+ target-db-id (get-in transform [:target :schema]) target-name))]
(cond-> transform
source
(assoc :source_type (transforms-base.u/transform-source-type source)
:source_database_id (or source_database_id (transforms-base.i/source-db-id transform)))
- (or (:source changes) (:target changes))
- ;; No database existence check added here, unlike for insert. Just allow updates for an invalid target to fail.
- (assoc :target_db_id (transforms-base.i/target-db-id transform))
+ target-changed?
+ (assoc :target_db_id target-db-id)
+
+ table-id
+ (assoc-in [:target :table_id] table-id)
;; Reset checkpoint when the incremental filter field changes
(let [old-field-id (get-in (t2/original transform) [:source :source-incremental-strategy :checkpoint-filter-field-id])
@@ -228,6 +245,9 @@
transform)
(t2/define-before-delete :model/Transform [transform]
+ ;; TODO (Chris 2026-03-19) -- Once we have indexed FKs pointing from both global and workspace transforms
+ ;; to their target table, we should consider synchronously deleting orphaned provisional table rows
+ ;; (tables that have never been physically materialized). See gc-transform-target-tables! for the batch equivalent.
(events/publish-event! :event/delete-transform {:id (:id transform)})
(search.core/delete! :model/Transform [(str (:id transform))])
transform)
@@ -345,7 +365,8 @@
(transforms-base.u/source-tables-map->vec st)
st)))
(m/update-existing :query serdes/import-mbql)))}
- :target {:export serdes/export-mbql :import serdes/import-mbql}
+ :target {:export #(serdes/export-mbql (dissoc % :table_id))
+ :import serdes/import-mbql}
:tags (serdes/nested :model/TransformTransformTag :transform_id opts)}})
(defmethod serdes/dependencies "Transform"
diff --git a/src/metabase/sync/sync_metadata/tables.clj b/src/metabase/sync/sync_metadata/tables.clj
index bedb8c65dfd..7749993cd43 100644
--- a/src/metabase/sync/sync_metadata/tables.clj
+++ b/src/metabase/sync/sync_metadata/tables.clj
@@ -307,7 +307,8 @@
(defn- archive-tables!
"Mark tables that have been deactivated for longer than the configured threshold as archived
- and suffixes their names."
+ and suffixes their names. Skips tables with `transform_target = true` (provisional transform
+ output entries) since transforms still reference them by their original name."
[database]
(let [;; we use UTC offset time for suffix, may not match db time but
;; it doesn't matter much, the source of time truth is `archived_at`,
@@ -320,6 +321,7 @@
:db_id (u/the-id database)
:active false
:archived_at nil
+ :transform_target false
:deactivated_at [:< threshold-expr])
archived (atom 0)]
(doseq [table tables-to-archive
diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.clj
index 56a3da85774..333dab141f3 100644
--- a/src/metabase/transforms_base/util.clj
+++ b/src/metabase/transforms_base/util.clj
@@ -28,6 +28,7 @@
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
+ [metabase.warehouse-schema.models.table :as table]
[toucan2.core :as t2])
(:import
(java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
@@ -552,9 +553,12 @@
(and (:table_id entry) (not (:table entry)))
(merge (int-id->metadata (:table_id entry)) entry)
- ;; Has table metadata but no table_id — look it up
+ ;; Has table metadata but no table_id — look it up, upsert transform target if not found
(missing-table-id? entry)
- (assoc entry :table_id (ref-lookup (source-table-ref->key entry)))
+ (assoc entry :table_id (or (ref-lookup (source-table-ref->key entry))
+ (when (and (:database_id entry) (:table entry))
+ (table/upsert-transform-target-table!
+ (:database_id entry) (:schema entry) (:table entry)))))
;; Already fully populated
:else entry))
@@ -669,6 +673,16 @@
;;; ------------------------------------------------- Misc -------------------------------------------------
+(defn upsert-target-table!
+ "Upsert a provisional table entry for a transform's target, creating it if it doesn't exist.
+ Returns the table ID.
+
+ Thin wrapper around [[metabase.warehouse-schema.models.table/upsert-transform-target-table!]] —
+ exists because the `models` module cannot depend on `warehouse-schema` directly, but can
+ depend on `transforms-base` (which is allowed to use `warehouse-schema`)."
+ [db-id schema table-name]
+ (table/upsert-transform-target-table! db-id schema table-name))
+
(defn is-temp-transform-table?
"Return true when `table` matches the transform temporary table naming pattern."
[table]
diff --git a/src/metabase/transforms_inspector/context.clj b/src/metabase/transforms_inspector/context.clj
index dd1d9f7e7c0..ccdf214563f 100644
--- a/src/metabase/transforms_inspector/context.clj
+++ b/src/metabase/transforms_inspector/context.clj
@@ -77,10 +77,16 @@
;;; -------------------------------------------------- Target Table --------------------------------------------------
(defn- get-target-table
- "Get the target table for a transform. Returns nil if the target doesn't exist."
+ "Get the target table for a transform. Returns nil if the target doesn't exist or hasn't been materialized yet.
+ Provisional tables (inactive, never deactivated) created by the after-insert hook are excluded."
[{:keys [target] :as transform}]
- (let [db-id (transforms-base.i/target-db-id transform)]
- (transforms-base.u/target-table db-id target)))
+ (let [db-id (transforms-base.i/target-db-id transform)
+ table (transforms-base.u/target-table db-id target)]
+ ;; A provisional table for a transform that was never run has active=false and deactivated_at=nil
+ ;; Having a table with active=false but a non-nil deactivated_at is a strong indication it ran before.
+ ;; That said even an active table is not proof that is has run, as targets may have been swiveled around.
+ (when (or (:active table) (:deactivated_at table))
+ table)))
;;; -------------------------------------------------- Field Metadata --------------------------------------------------
diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.clj
index fe22600b712..327d0f128a7 100644
--- a/src/metabase/warehouse_schema/models/table.clj
+++ b/src/metabase/warehouse_schema/models/table.clj
@@ -14,6 +14,7 @@
[metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
+ [metabase.workspaces.core :as workspaces]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -126,7 +127,10 @@
(partial mi/assert-optional-enum data-layers)
(some-fn legacy-data-layer->current identity))
:field_order mi/transform-keyword
- :data_source (mi/transform-validator mi/transform-keyword (partial mi/assert-optional-enum data-sources))
+ :data_source (mi/transform-validator-with-fixes
+ mi/transform-keyword
+ (partial mi/assert-optional-enum data-sources)
+ (some-fn keyword identity))
;; Warning: by using a transform to handle unexpected enum values, serialization becomes lossy
:data_authority transform-data-authority})
@@ -399,6 +403,78 @@
[_table]
[:schema :name (serdes/hydrated-hash :db :db_id)])
+;;; ----------------------------------------- Transform Target Tables -----------------------------------------------
+
+(defn upsert-transform-target-table!
+ "Ensure a metabase_table row exists for the given (db_id, schema, name) triple.
+ If the table doesn't exist, creates it as a transform target (inactive, not yet physically materialized).
+ If it already exists (active or inactive), returns its id without modification.
+ Returns the table id.
+
+ The `transform_target` column is **conservative**: it is set eagerly to `true` when a transform is
+ configured to target this table (synchronous, never stale), and cleared asynchronously when no
+ transforms reference it anymore. This means the column may have false positives (a table marked as
+ a target when no transform actually targets it) but **never false negatives** (a targeted table
+ will always have `transform_target = true`)."
+ [db-id schema table-name]
+ (app-db/update-or-insert!
+ :model/Table
+ {:db_id db-id :schema schema :name table-name}
+ (fn [existing]
+ (when-not existing
+ {:display_name (humanization/name->human-readable-name table-name)
+ :active false
+ :transform_target true
+ :data_source :metabase-transform
+ :data_authority :computed
+ :initial_sync_status "complete"}))))
+
+(defn- remove-referenced-candidates
+ "Short-circuiting reduce: removes matched triples from triple->table."
+ [triple->table target-triples]
+ (reduce (fn [triple->table triple]
+ (if (empty? triple->table)
+ (reduced triple->table)
+ (dissoc triple->table triple)))
+ triple->table
+ target-triples))
+
+(defn gc-transform-target-tables!
+ "Deletes provisional table rows (created by [[upsert-transform-target-table!]]) that are no longer
+ referenced by any Transform or WorkspaceTransform. Safe because these rows were never active,
+ so they have no child records.
+
+ Note: this only handles *inactive* provisional tables. Active tables that were previously
+ targeted by a transform retain `transform_target = true` even after the transform is deleted.
+ A separate process to reset `transform_target` on active, unreferenced tables is not yet
+ implemented.
+
+ Future optimizations:
+ - Add target_table_id FKs to avoid scanning transform rows.
+ - Pre-filter via workspace_input/workspace_output table references."
+ []
+ (let [candidates (t2/select [:model/Table :id :db_id :schema :name]
+ :active false :transform_target true :deactivated_at nil
+ :data_source :metabase-transform)]
+ (when (seq candidates)
+ (let [db-ids (into #{} (map :db_id) candidates)
+ triple->table (into {} (map (juxt (juxt :db_id :schema :name) identity)) candidates)
+ remaining (remove-referenced-candidates
+ triple->table
+ (eduction
+ (map (fn [{:keys [target target_db_id]}]
+ (-> target
+ (update :database #(or % target_db_id))
+ workspaces/target->triple)))
+ (t2/reducible-select [:model/Transform :target :target_db_id] :target_db_id [:in db-ids])))
+ remaining (when (seq remaining)
+ (remove-referenced-candidates
+ remaining
+ (workspaces/reducible-target-triples db-ids)))]
+ (when-let [dead-ids (seq (mapv :id (vals remaining)))]
+ (log/infof "Deleting %d orphaned transform target table(s)" (count dead-ids))
+ (t2/delete! :model/Table :id [:in dead-ids]))))))
+
;;; ------------------------------------------------ Field ordering -------------------------------------------------
(def field-order-rule
@@ -598,7 +674,7 @@
:points_of_interest :caveats :show_in_getting_started :field_order :initial_sync_status :is_upload
:database_require_filter :is_defective_duplicate :unique_table_helper :is_writable :data_authority
:data_source :owner_email :owner_user_id :is_published]
- :skip [:estimated_row_count :view_count]
+ :skip [:estimated_row_count :view_count :transform_target]
:transform {:created_at (serdes/date)
:archived_at (serdes/date)
:deactivated_at (serdes/date)
diff --git a/src/metabase/warehouse_schema_rest/api/table.clj b/src/metabase/warehouse_schema_rest/api/table.clj
index a933ec12ca0..68e5943e26e 100644
--- a/src/metabase/warehouse_schema_rest/api/table.clj
+++ b/src/metabase/warehouse_schema_rest/api/table.clj
@@ -73,7 +73,7 @@
- `can-write=true` - filter to only tables the user can edit metadata for"
[_
{:keys [term visibility-type data-layer data-source owner-user-id owner-email orphan-only unused-only
- can-query can-write]}
+ can-query can-write include-transform-targets]}
:- [:map
[:term {:optional true} :string]
[:visibility-type {:optional true} :string]
@@ -84,7 +84,8 @@
[:orphan-only {:optional true} [:maybe ms/BooleanValue]]
[:unused-only {:optional true} [:maybe ms/BooleanValue]]
[:can-query {:optional true} [:maybe ms/BooleanValue]]
- [:can-write {:optional true} [:maybe ms/BooleanValue]]]]
+ [:can-write {:optional true} [:maybe ms/BooleanValue]]
+ [:include-transform-targets {:optional true} [:maybe ms/BooleanValue]]]]
(let [like (fn [field pattern]
(case (app-db/db-type)
(:h2 :postgres) [:ilike field pattern]
@@ -95,7 +96,9 @@
(str/replace "%" "\\%")
(str/replace "*" "%")
(cond-> (not (str/ends-with? term "%")) (str "%")))
- where (cond-> [:and [:= :active true]]
+ where (cond-> [:and (if include-transform-targets
+ [:or [:= :active true] [:= :transform_target true]]
+ [:= :active true])]
(not (str/blank? term)) (conj [:or
(like :name pattern)
(like :display_name pattern)
diff --git a/src/metabase/workspaces/core.clj b/src/metabase/workspaces/core.clj
new file mode 100644
index 00000000000..572e66e39fd
--- /dev/null
+++ b/src/metabase/workspaces/core.clj
@@ -0,0 +1,35 @@
+(ns metabase.workspaces.core
+ "Workspace utilities that operate on the raw database tables (which exist in all editions).
+ These must run regardless of whether the workspaces feature flag is enabled, because workspace
+ data can exist from a previous period when it was enabled, and we need to maintain invariants
+ on that data so it isn't corrupt when the feature is re-enabled."
+ (:require
+ [metabase.util.json :as json]
+ [toucan2.core :as t2]))
+
+(defn target->triple
+ "Extracts `[db_id schema name]` from a target map."
+ [{:keys [database schema name]}]
+ [database schema name])
+
+(defn- parse-target
+ "Parses target JSON — raw query bypasses Toucan2 model transforms."
+ [target]
+ (if (string? target)
+ (try (json/decode+kw target)
+ (catch Exception _ nil))
+ target))
+
+(defn reducible-target-triples
+ "Reducible of `[db_id schema name]` triples for workspace transforms targeting any of the given `db-ids`.
+ Used by [[metabase.warehouse-schema.models.table/gc-transform-target-tables!]]."
+ [db-ids]
+ (eduction
+ (map (fn [{:keys [target database_id]}]
+ (-> (parse-target target)
+ (update :database #(or % database_id))
+ target->triple)))
+ (t2/reducible-query {:select [[:wt.target :target], [:w.database_id :database_id]]
+ :from [[:workspace_transform :wt]]
+ :join [[:workspace :w] [:= :wt.workspace_id :w.id]]
+ :where [:in :w.database_id db-ids]})))
diff --git a/test/metabase/app_db/custom_migrations_test.clj b/test/metabase/app_db/custom_migrations_test.clj
index 3240c6277c7..5ce7e015dfd 100644
--- a/test/metabase/app_db/custom_migrations_test.clj
+++ b/test/metabase/app_db/custom_migrations_test.clj
@@ -3015,3 +3015,124 @@
(is (map? st))
(is (= 42 (get st "orders")))
(is (= 99 (get st "products")))))))))
+
+(deftest backfill-transform-target-tables-test
+ (testing "v60.2026-03-07T00:00:04 : backfill transform target tables and workspace FK columns"
+ (impl/test-migrations ["v60.2026-03-07T00:00:04"] [migrate!]
+ (let [user-id (:id (new-instance-with-default :core_user))
+ db-id (:id (new-instance-with-default :metabase_database))
+ ws-id (:id (t2/insert-returning-instance!
+ :workspace {:name "test-ws"
+ :creator_id user-id
+ :created_at :%now
+ :updated_at :%now}))
+ ref-id (str (random-uuid))
+ source (json/encode {:type "query" :query {:database db-id}})
+ target (json/encode {:type "table" :schema "public" :name "orders"})
+ ;; -- 1. Transform with a target that has no existing metabase_table → should create provisional row --
+ tx-id (t2/insert-returning-pk!
+ :transform {:name "create-output"
+ :source source
+ :target (json/encode {:type "table" :schema "public" :name "new_target_table"})
+ :source_type "mbql"
+ :source_database_id db-id
+ :target_db_id db-id
+ :created_at :%now
+ :updated_at :%now})
+ ;; -- An existing metabase_table that inputs and outputs should resolve to --
+ table-id (:id (t2/insert-returning-instance!
+ :metabase_table {:db_id db-id
+ :name "orders"
+ :schema "public"
+ :active true
+ :created_at :%now
+ :updated_at :%now}))
+ ;; -- workspace_transform needed for workspace_output FK --
+ _ (t2/insert! :workspace_transform
+ {:ref_id ref-id
+ :workspace_id ws-id
+ :name "ws-tx"
+ :source source
+ :target target
+ :created_at :%now
+ :updated_at :%now})
+ ;; -- 2-3. workspace_output with null table FKs --
+ wo-id (:id (t2/insert-returning-instance!
+ :workspace_output {:workspace_id ws-id
+ :ref_id ref-id
+ :db_id db-id
+ :global_schema "public"
+ :global_table "orders"
+ :isolated_schema "public"
+ :isolated_table "orders"
+ :created_at :%now
+ :updated_at :%now}))
+ ;; -- 4-5. workspace_output_external with null table FKs --
+ woe-id (:id (t2/insert-returning-instance!
+ :workspace_output_external {:workspace_id ws-id
+ :transform_id tx-id
+ :db_id db-id
+ :global_schema "public"
+ :global_table "orders"
+ :isolated_schema "public"
+ :isolated_table "orders"
+ :created_at :%now
+ :updated_at :%now}))
+ ;; -- 6. workspace_input with null table_id --
+ wi-id (:id (t2/insert-returning-instance!
+ :workspace_input {:workspace_id ws-id
+ :db_id db-id
+ :schema "public"
+ :table "orders"
+ :created_at :%now
+ :updated_at :%now}))
+ ;; -- 7. workspace_input_external with null table_id --
+ wie-id (:id (t2/insert-returning-instance!
+ :workspace_input_external {:workspace_id ws-id
+ :db_id db-id
+ :schema "public"
+ :table "orders"
+ :created_at :%now
+ :updated_at :%now}))
+ ;; -- workspace_input_external with no matching table — should stay nil --
+ wie-no-match-id (:id (t2/insert-returning-instance!
+ :workspace_input_external {:workspace_id ws-id
+ :db_id db-id
+ :schema "public"
+ :table "no_such_table"
+ :created_at :%now
+ :updated_at :%now}))]
+ (testing "Before backfill, all FKs are nil"
+ (is (nil? (t2/select-one-fn :global_table_id :workspace_output :id wo-id)))
+ (is (nil? (t2/select-one-fn :isolated_table_id :workspace_output :id wo-id)))
+ (is (nil? (t2/select-one-fn :global_table_id :workspace_output_external :id woe-id)))
+ (is (nil? (t2/select-one-fn :isolated_table_id :workspace_output_external :id woe-id)))
+ (is (nil? (t2/select-one-fn :table_id :workspace_input :id wi-id)))
+ (is (nil? (t2/select-one-fn :table_id :workspace_input_external :id wie-id)))
+ (is (nil? (t2/select-one-fn :table_id :workspace_input_external :id wie-no-match-id))))
+ (migrate!)
+ (testing "Provisional metabase_table created for transform target"
+ (let [provisional (first (t2/query {:select [:active :transform_target :data_source :data_authority :display_name]
+ :from [:metabase_table]
+ :where [:and
+ [:= :db_id db-id]
+ [:= :name "new_target_table"]
+ [:= :schema "public"]]}))]
+ (is (some? provisional))
+ (is (false? (:active provisional)))
+ (is (true? (:transform_target provisional)))
+ (is (= "metabase-transform" (:data_source provisional)))
+ (is (= "computed" (:data_authority provisional)))
+ (is (= "New Target Table" (:display_name provisional)))))
+ (testing "workspace_output FKs backfilled"
+ (is (= table-id (t2/select-one-fn :global_table_id :workspace_output :id wo-id)))
+ (is (= table-id (t2/select-one-fn :isolated_table_id :workspace_output :id wo-id))))
+ (testing "workspace_output_external FKs backfilled"
+ (is (= table-id (t2/select-one-fn :global_table_id :workspace_output_external :id woe-id)))
+ (is (= table-id (t2/select-one-fn :isolated_table_id :workspace_output_external :id woe-id))))
+ (testing "workspace_input table_id backfilled"
+ (is (= table-id (t2/select-one-fn :table_id :workspace_input :id wi-id))))
+ (testing "workspace_input_external table_id backfilled"
+ (is (= table-id (t2/select-one-fn :table_id :workspace_input_external :id wie-id))))
+ (testing "Non-matching input stays nil"
+ (is (nil? (t2/select-one-fn :table_id :workspace_input_external :id wie-no-match-id))))))))
diff --git a/test/metabase/sync/sync_metadata/tables_test.clj b/test/metabase/sync/sync_metadata/tables_test.clj
index 4482bcc8c92..faef2d2bd28 100644
--- a/test/metabase/sync/sync_metadata/tables_test.clj
+++ b/test/metabase/sync/sync_metadata/tables_test.clj
@@ -213,6 +213,31 @@
(is (= "active_table" (:name active-table)))
(is (true? (:active active-table))))))))
+(deftest archive-tables-skips-transform-targets-test
+ (testing "Tables with transform_target=true should never be archived, even if old enough"
+ (mt/with-temp [:model/Database db {}
+ :model/Table provisional {:name "transform_output"
+ :db_id (u/the-id db)
+ :active false
+ :transform_target true
+ :deactivated_at (t/minus (t/offset-date-time) (t/days 30))}
+ :model/Table normal {:name "old_table"
+ :db_id (u/the-id db)
+ :active false
+ :transform_target false
+ :deactivated_at (t/minus (t/offset-date-time) (t/days 30))}]
+ (#'sync-tables/archive-tables! db)
+
+ (testing "Transform target table is not archived or renamed"
+ (let [table (t2/select-one :model/Table (:id provisional))]
+ (is (nil? (:archived_at table)))
+ (is (= "transform_output" (:name table)))))
+
+ (testing "Normal table is archived as usual"
+ (let [table (t2/select-one :model/Table (:id normal))]
+ (is (some? (:archived_at table)))
+ (is (str/starts-with? (:name table) "old_table__mbarchiv__")))))))
+
(deftest archive-tables-already-archived-test
(testing "Already archived tables should not be processed again"
(mt/with-temp [:model/Database db {}
diff --git a/test/metabase/sync/util_test.clj b/test/metabase/sync/util_test.clj
index 6e262d0affc..4aaea4abec6 100644
--- a/test/metabase/sync/util_test.clj
+++ b/test/metabase/sync/util_test.clj
@@ -300,7 +300,7 @@
(testing "If `initial-sync-status` on a table is `incomplete`, it is marked as `complete` after the sync-fks step
has finished"
- (let [table-id (t2/select-one-fn :id :model/Table :db_id (mt/id))
+ (let [table-id (t2/select-one-fn :id :model/Table :db_id (mt/id) :active true)
_ (t2/update! :model/Table table-id {:initial_sync_status "incomplete"})
_table (t2/select-one :model/Table :id table-id)]
(sync/sync-database! (mt/db))
@@ -309,7 +309,7 @@
(testing "Database and table syncs are marked as complete even if the initial scan is :schema only"
(let [_ (t2/update! :model/Database (mt/id) {:initial_sync_status "incomplete"})
db (t2/select-one :model/Database :id (mt/id))
- table-id (t2/select-one-fn :id :model/Table :db_id (mt/id))
+ table-id (t2/select-one-fn :id :model/Table :db_id (mt/id) :active true)
_ (t2/update! :model/Table table-id {:initial_sync_status "incomplete"})
_table (t2/select-one :model/Table :id table-id)]
(sync/sync-database! db {:scan :schema})
@@ -338,7 +338,7 @@
;; incomplete, but then marked as complete after the sync is finished.
(mt/dataset test-data
(testing "If `initial-sync-status` on a DB is already `complete`"
- (let [[active-table inactive-table] (t2/select :model/Table :db_id (mt/id))
+ (let [[active-table inactive-table] (t2/select :model/Table :db_id (mt/id) :active true)
get-active-table #(t2/select-one :model/Table :id (:id active-table))
get-inactive-table #(t2/select-one :model/Table :id (:id inactive-table))]
(t2/update! :model/Table (:id active-table) {:initial_sync_status "complete" :active true})
diff --git a/test/metabase/transforms/test_util.clj b/test/metabase/transforms/test_util.clj
index 07402356171..1597310ffd5 100644
--- a/test/metabase/transforms/test_util.clj
+++ b/test/metabase/transforms/test_util.clj
@@ -21,6 +21,13 @@
(let [{:keys [db_id schema]} (t2/select-one [:model/Table :db_id :schema] :id table-id)]
{:alias alias :table_id table-id :database_id db_id :schema schema}))
+(defn default-source-table-entry
+ "Build a source-table-entry for the first active table in the current test database."
+ ([]
+ (default-source-table-entry "test"))
+ ([alias]
+ (source-table-entry alias (t2/select-one-pk :model/Table :db_id (mt/id) :active true))))
+
(defn drop-target!
"Drop transform target `target` and clean up its metadata.
`target` can be a string or a map. If `target` is a string, type :table is assumed.
diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.clj
index 13fbf0029e1..ab8554da214 100644
--- a/test/metabase/transforms/util_test.clj
+++ b/test/metabase/transforms/util_test.clj
@@ -210,10 +210,10 @@
result (transforms-base.u/normalize-source-tables source-tables)]
(is (= 999 (:table_id (first result))))))
- (testing "leaves table_id nil for non-existent table ref"
+ (testing "creates transform target table for non-existent table ref"
(let [source-tables [{:alias "t" :database_id (:id db) :schema nil :table "nonexistent"}]
result (transforms-base.u/normalize-source-tables source-tables)]
- (is (nil? (:table_id (first result))))))
+ (is (int? (:table_id (first result))))))
(testing "handles entries needing different kinds of enrichment"
(let [source-tables [{:alias "t1" :table_id (:id t1)}
@@ -355,22 +355,24 @@
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:transforms-basic}
(let [target {:type "table" :schema nil :name "test_output_table"}]
+ ;; The Transform after-insert hook creates a provisional table for the target,
+ ;; so we don't need to create one explicitly.
(mt/with-temp [:model/Transform {transform-id :id :as transform}
{:target target
:source {:type "query"
- :query (lib/query (mt/metadata-provider) (mt/mbql-query venues))}}
- :model/Table {table-id :id} {:db_id (mt/id) :name "test_output_table" :schema nil}]
- ;; Mock execute-base! to return success without actually running a query,
- ;; run-cancelable-transform! to bypass schema creation / cancellation infra,
- ;; and sync/indexes to skip driver calls. complete-execution! runs normally
- ;; so it sets transform_id on the target table.
- (with-redefs [transforms-base.i/execute-base! (constantly {:status :succeeded})
- transforms-base.u/sync-target! (constantly nil)
- transforms.u/run-cancelable-transform! (fn [_run-id _transform _driver _details run-fn & _opts]
- (run-fn (a/promise-chan) nil))]
- (transforms.execute/execute! transform {:run-method :manual})
- (is (= transform-id
- (t2/select-one-fn :transform_id :model/Table :id table-id))))))))))
+ :query (lib/query (mt/metadata-provider) (mt/mbql-query venues))}}]
+ (let [table-id (t2/select-one-fn :id :model/Table :db_id (mt/id) :name "test_output_table" :schema nil)]
+ ;; Mock execute-base! to return success without actually running a query,
+ ;; run-cancelable-transform! to bypass schema creation / cancellation infra,
+ ;; and sync/indexes to skip driver calls. complete-execution! runs normally
+ ;; so it sets transform_id on the target table.
+ (with-redefs [transforms-base.i/execute-base! (constantly {:status :succeeded})
+ transforms-base.u/sync-target! (constantly nil)
+ transforms.u/run-cancelable-transform! (fn [_run-id _transform _driver _details run-fn & _opts]
+ (run-fn (a/promise-chan) nil))]
+ (transforms.execute/execute! transform {:run-method :manual})
+ (is (= transform-id
+ (t2/select-one-fn :transform_id :model/Table :id table-id)))))))))))
(deftest transform-hydration-test
(testing "hydrating :transform on a table"
diff --git a/test/metabase/transforms_rest/api/transform_test.clj b/test/metabase/transforms_rest/api/transform_test.clj
index 63c6f839090..4ce6977ff7b 100644
--- a/test/metabase/transforms_rest/api/transform_test.clj
+++ b/test/metabase/transforms_rest/api/transform_test.clj
@@ -1178,7 +1178,7 @@
:query {:database (mt/id)
:type "native"
:native {:query "SELECT 1"}}}
- :target {:type "table" :name "test_table"}})
+ :target {:type "table" :name (mt/random-name)}})
(mt/user-http-request :rasta :put 403 (str "transform/" (:id transform))
{:name "Updated"})
(mt/user-http-request :rasta :delete 403 (str "transform/" (:id transform))))
@@ -1320,7 +1320,7 @@
:type "native"
:native {:query "SELECT 1"}}}
:target {:type "table"
- :name "test_table"}}
+ :name (mt/random-name)}}
:model/TransformTag tag1 {:name "update-tag-1"}
:model/TransformTag tag2 {:name "update-tag-2"}
:model/TransformTag tag3 {:name "update-tag-3"}]
diff --git a/test/metabase/xrays/automagic_dashboards/core_test.clj b/test/metabase/xrays/automagic_dashboards/core_test.clj
index 5b0a77ad3a4..e12f8cc297b 100644
--- a/test/metabase/xrays/automagic_dashboards/core_test.clj
+++ b/test/metabase/xrays/automagic_dashboards/core_test.clj
@@ -217,7 +217,7 @@
(mt/with-test-user :rasta
(automagic-dashboards.test/with-rollback-only-transaction
(doseq [[table cardinality] (map vector
- (t2/select :model/Table :db_id (mt/id) {:order-by [[:name :asc]]})
+ (t2/select :model/Table :db_id (mt/id) :active true {:order-by [[:name :asc]]})
[2 8 11 11 15 17 5 7])]
(test-automagic-analysis table cardinality)))))
diff --git a/test/metabase/xrays/transforms/core_test.clj b/test/metabase/xrays/transforms/core_test.clj
index 67964cbae5c..1a6d1ac0acd 100644
--- a/test/metabase/xrays/transforms/core_test.clj
+++ b/test/metabase/xrays/transforms/core_test.clj
@@ -65,7 +65,7 @@
(deftest ^:parallel tableset-test
(testing "Can we get a tableset for a given schema?"
- (is (= (t2/select-pks-set :model/Table :db_id (mt/id))
+ (is (= (t2/select-pks-set :model/Table :db_id (mt/id) :schema "PUBLIC")
(set (map u/the-id (#'tf/tableset (mt/id) "PUBLIC")))))))
(deftest ^:parallel find-tables-with-domain-entity-test
diff --git a/test_resources/serialization_baseline/databases/My Database/schemas/public/tables/target_table/target_table.yaml b/test_resources/serialization_baseline/databases/My Database/schemas/public/tables/target_table/target_table.yaml
new file mode 100644
index 00000000000..0765397063b
--- /dev/null
+++ b/test_resources/serialization_baseline/databases/My Database/schemas/public/tables/target_table/target_table.yaml
@@ -0,0 +1,34 @@
+name: target_table
+display_name: Target Table
+description: null
+created_at: '2026-03-09T17:33:23.746321Z'
+db_id: My Database
+schema: public
+entity_type: null
+active: false
+is_upload: false
+field_order: database
+visibility_type: null
+show_in_getting_started: false
+initial_sync_status: complete
+points_of_interest: null
+caveats: null
+serdes/meta:
+- id: My Database
+ model: Database
+- id: public
+ model: Schema
+- id: target_table
+ model: Table
+archived_at: null
+collection_id: null
+data_authority: computed
+data_layer: internal
+data_source: metabase-transform
+database_require_filter: null
+deactivated_at: null
+is_published: false
+is_writable: null
+owner_email: null
+owner_user_id: null
+transform_id: null