Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add FKs from transforms back to tables (#70617)

Chris Truterc940bb3935fc
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 2ee556c945d..5afb83cddf9 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1336,7 +1336,8 @@
settings
transforms
transforms-base
- util}
+ util
+ warehouse-schema}
:friends #{transforms}
:model-exports #{:model/Transform
:model/TransformJob
@@ -2451,8 +2452,7 @@
remote-sync
search
util
- warehouses
- workspaces}
+ warehouses}
:model-exports #{:model/Dimension
:model/Field
:model/FieldUserSettings
@@ -2544,7 +2544,6 @@
workspaces
{:team "Metabot"
- :api #{metabase.workspaces.core}
:uses #{util}}
xrays
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
index 30d44d7d5d0..f466f68ce61 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
@@ -1358,7 +1358,7 @@
:data {:name "Unreferenced Transform - unreftest"}}]}
response)))
(finally
- ;; Clean up provisional table rows created by define-after-insert
+ ;; Clean up provisional table rows created by define-before-insert
(t2/delete! :model/Table :db_id (mt/id) :name [:in ["referenced_transform_table" "unreferenced_transform_table"]]))))))))
(deftest ^:sequential unreferenced-snippets-test
diff --git a/resources/migrations/060/20260310_table_transform_target_fk.yaml b/resources/migrations/060/20260310_table_transform_target_fk.yaml
new file mode 100644
index 00000000000..816bb3620ac
--- /dev/null
+++ b/resources/migrations/060/20260310_table_transform_target_fk.yaml
@@ -0,0 +1,235 @@
+## Add target_table_id FK to transform and table FKs on workspace input/output columns
+
+databaseChangeLog:
+ - logicalFilePath: migrations/060_update_migrations.yaml
+ - changeSet:
+ id: v60.2026-03-10T00:00:00
+ author: christruter
+ comment: Add target_table_id column to transform table
+ changes:
+ - addColumn:
+ tableName: transform
+ columns:
+ - column:
+ name: target_table_id
+ type: int
+ remarks: FK to metabase_table for the transform's target table
+ constraints:
+ nullable: true
+ rollback:
+ - dropColumn:
+ tableName: transform
+ columnName: target_table_id
+
+ - changeSet:
+ id: v60.2026-03-10T00:00:10
+ author: christruter
+ comment: Add index on transform.target_table_id
+ changes:
+ - createIndex:
+ tableName: transform
+ indexName: idx_transform_target_table_id
+ columns:
+ - column:
+ name: target_table_id
+
+ - changeSet:
+ id: v60.2026-03-10T00:00:20
+ author: christruter
+ comment: Add FK from transform.target_table_id to metabase_table with SET NULL
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - foreignKeyConstraintExists:
+ foreignKeyName: fk_transform_target_table_id
+ changes:
+ - addForeignKeyConstraint:
+ baseTableName: transform
+ baseColumnNames: target_table_id
+ constraintName: fk_transform_target_table_id
+ referencedTableName: metabase_table
+ referencedColumnNames: id
+ onDelete: SET NULL
+ rollback:
+ - dropForeignKeyConstraint:
+ baseTableName: transform
+ constraintName: fk_transform_target_table_id
+
+ - changeSet:
+ id: v60.2026-03-10T00:00:30
+ author: christruter
+ comment: Backfill transform.target_table_id from target JSON column
+ changes:
+ - customChange:
+ class: metabase.app_db.custom_migrations.BackfillTransformTargetTableId
+ rollback:
+
+ - changeSet:
+ id: v60.2026-03-10T00:00:40
+ author: christruter
+ comment: Add index on workspace_output.global_table_id
+ changes:
+ - createIndex:
+ tableName: workspace_output
+ indexName: idx_workspace_output_global_table_id
+ columns:
+ - column:
+ name: global_table_id
+
+ - changeSet:
+ id: v60.2026-03-10T00:00:50
+ author: christruter
+ comment: Add FK from workspace_output.global_table_id to metabase_table with SET NULL
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - foreignKeyConstraintExists:
+ foreignKeyName: fk_workspace_output_global_table_id
+ changes:
+ - addForeignKeyConstraint:
+ baseTableName: workspace_output
+ baseColumnNames: global_table_id
+ constraintName: fk_workspace_output_global_table_id
+ referencedTableName: metabase_table
+ referencedColumnNames: id
+ onDelete: SET NULL
+ rollback:
+ - dropForeignKeyConstraint:
+ baseTableName: workspace_output
+ constraintName: fk_workspace_output_global_table_id
+
+ - changeSet:
+ id: v60.2026-03-10T00:01:00
+ author: christruter
+ comment: Add index on workspace_output.isolated_table_id
+ changes:
+ - createIndex:
+ tableName: workspace_output
+ indexName: idx_workspace_output_isolated_table_id
+ columns:
+ - column:
+ name: isolated_table_id
+
+ - changeSet:
+ id: v60.2026-03-10T00:01:10
+ author: christruter
+ comment: Add FK from workspace_output.isolated_table_id to metabase_table with SET NULL
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - foreignKeyConstraintExists:
+ foreignKeyName: fk_workspace_output_isolated_table_id
+ changes:
+ - addForeignKeyConstraint:
+ baseTableName: workspace_output
+ baseColumnNames: isolated_table_id
+ constraintName: fk_workspace_output_isolated_table_id
+ referencedTableName: metabase_table
+ referencedColumnNames: id
+ onDelete: SET NULL
+ rollback:
+ - dropForeignKeyConstraint:
+ baseTableName: workspace_output
+ constraintName: fk_workspace_output_isolated_table_id
+
+ - changeSet:
+ id: v60.2026-03-10T00:01:20
+ author: christruter
+ comment: Add index on workspace_input_external.table_id
+ changes:
+ - createIndex:
+ tableName: workspace_input_external
+ indexName: idx_workspace_input_external_table_id
+ columns:
+ - column:
+ name: table_id
+
+ - changeSet:
+ id: v60.2026-03-10T00:01:30
+ author: christruter
+ comment: Add FK from workspace_input_external.table_id to metabase_table with SET NULL
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - foreignKeyConstraintExists:
+ foreignKeyName: fk_workspace_input_external_table_id
+ changes:
+ - addForeignKeyConstraint:
+ baseTableName: workspace_input_external
+ baseColumnNames: table_id
+ constraintName: fk_workspace_input_external_table_id
+ referencedTableName: metabase_table
+ referencedColumnNames: id
+ onDelete: SET NULL
+ rollback:
+ - dropForeignKeyConstraint:
+ baseTableName: workspace_input_external
+ constraintName: fk_workspace_input_external_table_id
+
+ - changeSet:
+ id: v60.2026-03-10T00:01:40
+ author: christruter
+ comment: Add index on workspace_output_external.global_table_id
+ changes:
+ - createIndex:
+ tableName: workspace_output_external
+ indexName: idx_workspace_output_external_global_table_id
+ columns:
+ - column:
+ name: global_table_id
+
+ - changeSet:
+ id: v60.2026-03-10T00:01:50
+ author: christruter
+ comment: Add FK from workspace_output_external.global_table_id to metabase_table with SET NULL
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - foreignKeyConstraintExists:
+ foreignKeyName: fk_workspace_output_external_global_table_id
+ changes:
+ - addForeignKeyConstraint:
+ baseTableName: workspace_output_external
+ baseColumnNames: global_table_id
+ constraintName: fk_workspace_output_external_global_table_id
+ referencedTableName: metabase_table
+ referencedColumnNames: id
+ onDelete: SET NULL
+ rollback:
+ - dropForeignKeyConstraint:
+ baseTableName: workspace_output_external
+ constraintName: fk_workspace_output_external_global_table_id
+
+ - changeSet:
+ id: v60.2026-03-10T00:02:00
+ author: christruter
+ comment: Add index on workspace_output_external.isolated_table_id
+ changes:
+ - createIndex:
+ tableName: workspace_output_external
+ indexName: idx_workspace_output_external_isolated_table_id
+ columns:
+ - column:
+ name: isolated_table_id
+
+ - changeSet:
+ id: v60.2026-03-10T00:02:10
+ author: christruter
+ comment: Add FK from workspace_output_external.isolated_table_id to metabase_table with SET NULL
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - foreignKeyConstraintExists:
+ foreignKeyName: fk_workspace_output_external_isolated_table_id
+ changes:
+ - addForeignKeyConstraint:
+ baseTableName: workspace_output_external
+ baseColumnNames: isolated_table_id
+ constraintName: fk_workspace_output_external_isolated_table_id
+ referencedTableName: metabase_table
+ referencedColumnNames: id
+ onDelete: SET NULL
+ rollback:
+ - dropForeignKeyConstraint:
+ baseTableName: workspace_output_external
+ constraintName: fk_workspace_output_external_isolated_table_id
diff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.clj
index 1f3a2620086..6ec500bcea8 100644
--- a/src/metabase/app_db/custom_migrations.clj
+++ b/src/metabase/app_db/custom_migrations.clj
@@ -2187,3 +2187,44 @@
(t2/query {:update from-table
:set {:table_id table-id}
:where [:= :id id]})))))
+
+(define-migration BackfillTransformTargetTableId
+ ;; For each transform with a non-null target_db_id, extract the table_id from the target JSON column.
+ ;; If table_id is present in the JSON, use it directly.
+ ;; If table_id is missing but name is present, look up the table by (db_id, schema, name).
+ ;; Uses case-insensitive matching with exact-case preference for both schema and name.
+ (let [find-table-id (fn [db-id schema table-name]
+ (:id (first (t2/query {:select [:id]
+ :from [:metabase_table]
+ :where [:and
+ [:= :db_id db-id]
+ (if (some? schema)
+ [:= [:lower :schema] [:lower schema]]
+ [:is :schema nil])
+ [:= [:lower :name] [:lower table-name]]]
+ :order-by [[[:case
+ [:and
+ [:= :name table-name]
+ (if (some? schema)
+ [:= :schema schema]
+ [:is :schema nil])]
+ 0
+ :else 1]
+ :asc]]
+ :limit 1}))))]
+ (doseq [{:keys [id target target_db_id]} (t2/query {:select [:id :target :target_db_id]
+ :from [:transform]
+ :where [:and
+ [:not= :target_db_id nil]
+ [:= :target_table_id nil]]})]
+ (let [target-map (json-out target false)
+ table-id (or (get target-map "table_id")
+ (when-let [table-name (get target-map "name")]
+ (find-table-id target_db_id (get target-map "schema") table-name)))]
+ (when table-id
+ (t2/query {:update :transform
+ :set {:target_table_id table-id}
+ :where [:= :id id]})))))
+ ;; Invalidate workspace graph caches so they recalculate with target_table_id
+ (t2/query {:update :workspace
+ :set {:graph_version [:+ :graph_version 1]}}))
diff --git a/src/metabase/models/transforms/transform.clj b/src/metabase/models/transforms/transform.clj
index 4cc36f24454..a13b2fed7c5 100644
--- a/src/metabase/models/transforms/transform.clj
+++ b/src/metabase/models/transforms/transform.clj
@@ -20,6 +20,7 @@
[metabase.transforms.util :as transforms.u]
[metabase.util :as u]
[metabase.util.log :as log]
+ [metabase.warehouse-schema.models.table :as ws.table]
[methodical.core :as methodical]
[toucan2.core :as t2]
[toucan2.instance :as t2.instance]))
@@ -117,7 +118,8 @@
(log/warnf "Invalid target database id (%s) ignored for new transform (%s)" target-db-id (:name transform)))
(-> transform
(assoc-in [:target :database] target-db-id)
- (cond-> table-id (assoc-in [:target :table_id] table-id))
+ (cond-> table-id (-> (assoc-in [:target :table_id] table-id)
+ (assoc :target_table_id table-id)))
(assoc
:source_type (transforms-base.u/transform-source-type source)
:target_db_id (when valid-db-id? target-db-id)
@@ -148,7 +150,8 @@
(assoc :target_db_id target-db-id)
table-id
- (assoc-in [:target :table_id] table-id)
+ (-> (assoc-in [:target :table_id] table-id)
+ (assoc :target_table_id table-id))
;; Reset checkpoint when the incremental filter field changes
(let [old-field-id (get-in (t2/original transform) [:source :source-incremental-strategy :checkpoint-filter-field-id])
@@ -255,9 +258,7 @@
transform)
(t2/define-before-delete :model/Transform [transform]
- ;; TODO (Chris 2026-03-19) -- Once we have indexed FKs pointing from both global and workspace transforms
- ;; to their target table, we should consider synchronously deleting orphaned provisional table rows
- ;; (tables that have never been physically materialized). See gc-transform-target-tables! for the batch equivalent.
+ (ws.table/delete-orphaned-provisional-table! (:target_table_id transform) (:id transform))
(events/publish-event! :event/delete-transform {:id (:id transform)})
(search.core/delete! :model/Transform [(str (:id transform))])
transform)
@@ -331,27 +332,13 @@
:table-with-db-and-fields
"Fetch tables with their fields. The tables show up under the `:table` property."
[transforms]
- (let [table-key-fn (fn [{:keys [target] :as transform}]
- [(transforms-base.i/target-db-id transform) (:schema target) (:name target)])
- table-keys (into #{} (map table-key-fn) transforms)
- table-keys-with-schema (filter second table-keys)
- table-keys-without-schema (keep (fn [[db-id schema table-name]]
- (when-not schema
- [db-id table-name]))
- table-keys)
- tables (when (or (seq table-keys-with-schema) (seq table-keys-without-schema))
- (-> (t2/select :model/Table
- {:where [:or
- (when (seq table-keys-with-schema)
- [:in [:composite :db_id :schema :name] table-keys-with-schema])
- (when (seq table-keys-without-schema)
- [:and
- [:= :schema nil]
- [:in [:composite :db_id :name] table-keys-without-schema]])]})
- (t2/hydrate :db :fields)))
- table-keys->table (m/index-by (juxt :db_id :schema :name) tables)]
+ (let [table-ids (into #{} (keep :target_table_id) transforms)
+ id->table (when (seq table-ids)
+ (m/index-by :id (-> (t2/select :model/Table :id [:in table-ids])
+ (t2/hydrate :db :fields))))]
(for [transform transforms]
- (assoc transform :table (get table-keys->table (table-key-fn transform))))))
+ (assoc transform :table
+ (get id->table (:target_table_id transform))))))
(defmethod serdes/hash-fields :model/Transform
[_transform]
@@ -360,7 +347,7 @@
(defmethod serdes/make-spec "Transform"
[_model-name opts]
{:copy [:name :description :entity_id :owner_email]
- :skip [:dependency_analysis_version :source_type :target_db_id :last_checkpoint_value]
+ :skip [:dependency_analysis_version :source_type :target_db_id :target_table_id :last_checkpoint_value]
:transform {:created_at (serdes/date)
:creator_id (serdes/fk :model/User)
:owner_user_id (serdes/fk :model/User)
diff --git a/src/metabase/transforms_base/ordering.clj b/src/metabase/transforms_base/ordering.clj
index 64b6998b614..efa1836664b 100644
--- a/src/metabase/transforms_base/ordering.clj
+++ b/src/metabase/transforms_base/ordering.clj
@@ -7,11 +7,9 @@
[clojure.string :as str]
[flatland.ordered.set :refer [ordered-set]]
[metabase.driver :as driver]
- [metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.schema.id :as lib.schema.id]
- [metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.query-processor.preprocess :as qp.preprocess]
[metabase.transforms-base.interface :as transforms-base.i]
[metabase.transforms-base.util :as transforms-base.u]
@@ -70,35 +68,38 @@
transforms))
(mu/defn- output-table-map
- [mp :- ::lib.schema.metadata/metadata-provider transforms]
- (let [table-map (into {}
- (map (fn [{:keys [schema name id]}]
- [[schema name] id]))
- (lib.metadata/tables mp))]
- (into {}
- (keep (fn [transform]
- (when-let [output-table (table-map [(get-in transform [:target :schema])
- (get-in transform [:target :name])])]
- [output-table (:id transform)])))
- transforms)))
-
-(defn- target-ref-map
- "Build a map from [database_id schema table_name] -> transform_id for all transforms."
[transforms]
(into {}
- (map (fn [{:keys [id target]}]
- [[(:database target) (:schema target) (:name target)] id]))
+ (keep (fn [{:keys [target_table_id id]}]
+ (when target_table_id
+ [target_table_id id])))
transforms))
+(defn- target-ref-map
+ "Build maps for resolving dependencies to transform ids.
+ Returns {:by-triple {[database_id schema table_name] -> transform_id}
+ :by-table-id {table_id -> transform_id}}"
+ [transforms]
+ {:by-triple (into {}
+ (map (fn [{:keys [id target]}]
+ [[(:database target) (:schema target) (:name target)] id]))
+ transforms)
+ :by-table-id (into {}
+ (keep (fn [{:keys [id target_table_id]}]
+ (when target_table_id
+ [target_table_id id])))
+ transforms)})
+
(defn- resolve-dependency
"Resolve a single dependency to a transform id, or nil if not resolvable.
Used to map table/transform/table-ref dependencies to actual transform ids."
- [{:keys [table transform table-ref]} output-tables transform-ids target-refs]
- (or (output-tables table)
- (transform-ids transform)
+ [{dep-table :table dep-transform :transform :keys [table-ref]} output-tables transform-ids target-refs]
+ (or (get output-tables dep-table)
+ (get (:by-table-id target-refs) dep-table)
+ (get transform-ids dep-transform)
(when table-ref
(let [{:keys [database_id schema table]} table-ref]
- (target-refs [database_id schema table])))))
+ (get (:by-triple target-refs) [database_id schema table])))))
(defn transform-ordering
"Computes an 'ordering' of a given list of transforms.
@@ -117,12 +118,11 @@
target-refs (target-ref-map transforms)
{:keys [output-tables
dependencies]} (->> transforms-by-db
- (map (mu/fn [[db-id db-transforms] :- [:tuple
- [:maybe ::lib.schema.id/database]
- [:maybe [:sequential :any]]]]
- (let [mp (lib-be/application-database-metadata-provider db-id)]
- {:output-tables (output-table-map mp db-transforms)
- :dependencies (dependency-map db-transforms)})))
+ (map (mu/fn [[_db-id db-transforms] :- [:tuple
+ [:maybe ::lib.schema.id/database]
+ [:maybe [:sequential :any]]]]
+ {:output-tables (output-table-map db-transforms)
+ :dependencies (dependency-map db-transforms)}))
(apply merge-with merge))]
;; Transforms without a target database are invalid and shouldn't form part of the dependency graph.
;; Give them empty dependency sets so they don't interfere with ordering.
@@ -177,9 +177,8 @@
(map (juxt :id identity))
transforms)
db-id (get-in to-check [:source :query :database])
- mp (lib-be/application-database-metadata-provider db-id)
db-transforms (filter #(= (get-in % [:source :query :database]) db-id) transforms)
- output-tables (output-table-map mp db-transforms)
+ output-tables (output-table-map db-transforms)
transform-ids (into #{} (map :id) db-transforms)
target-refs (target-ref-map transforms)
node->children #(->> % transforms-by-id transforms-base.i/table-dependencies
diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.clj
index a128b8e5f67..b7f9402c373 100644
--- a/src/metabase/transforms_base/util.clj
+++ b/src/metabase/transforms_base/util.clj
@@ -473,13 +473,9 @@
:or {publish-events? true}} opts
db-id (transforms-base.i/target-db-id transform)
database (t2/select-one :model/Database db-id)]
- ;; Sync target table
- (sync-target! target database)
- ;; Mark the table as owned by this transform
- (when-let [table (t2/select-one :model/Table
- :db_id db-id
- :schema (:schema target)
- :name (:name target))]
+ ;; Sync target table, set target_table_id on transform, and mark table as owned by this transform
+ (when-let [table (sync-target! target database)]
+ (t2/update! :model/Transform (:id transform) {:target_table_id (:id table)})
(t2/update! :model/Table (:id table) {:transform_id (:id transform)}))
;; Publish event after sync so the table exists in AppDB.
(when publish-events?
diff --git a/src/metabase/transforms_rest/api/transform.clj b/src/metabase/transforms_rest/api/transform.clj
index 94d9dc30126..9931dac6558 100644
--- a/src/metabase/transforms_rest/api/transform.clj
+++ b/src/metabase/transforms_rest/api/transform.clj
@@ -91,6 +91,7 @@
[:last_run {:optional true} [:maybe TransformLastRunResponse]]
[:tag_ids {:optional true} [:sequential pos-int?]]
[:table {:optional true} [:maybe :map]]
+ [:target_table_id {:optional true} [:maybe pos-int?]]
[:owner_user_id {:optional true} [:maybe pos-int?]]
[:owner_email {:optional true} [:maybe :string]]
[:owner {:optional true} [:maybe OwnerResponse]]
diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.clj
index 327d0f128a7..a80255fa728 100644
--- a/src/metabase/warehouse_schema/models/table.clj
+++ b/src/metabase/warehouse_schema/models/table.clj
@@ -14,7 +14,6 @@
[metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
- [metabase.workspaces.core :as workspaces]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -429,19 +428,39 @@
:data_authority :computed
:initial_sync_status "complete"}))))
-(defn- remove-referenced-candidates
- "Short-circuiting reduce: removes matched triples from triple->table."
- [triple->table target-triples]
- (reduce (fn [triple->table triple]
- (if (empty? triple->table)
- (reduced triple->table)
- (dissoc triple->table triple)))
- triple->table
- target-triples))
+(defn delete-orphaned-provisional-table!
+ "If `table-id` points at an inactive provisional table that is not referenced by any other
+ transform or workspace row (excluding `exclude-transform-id`), delete it."
+ [table-id exclude-transform-id]
+ (when table-id
+ (when-let [table (t2/select-one :model/Table :id table-id
+ :active false :transform_target true :deactivated_at nil
+ :data_source :metabase-transform)]
+ (let [referenced?
+ (seq
+ (t2/query {:union
+ (cons
+ {:select [[[:inline 1] :ref]]
+ :from [:transform]
+ :where [:and
+ [:= :target_table_id (:id table)]
+ [:not= :id exclude-transform-id]]}
+ (for [[table-name column-name]
+ [["workspace_input" "table_id"]
+ ["workspace_output" "global_table_id"]
+ ["workspace_output" "isolated_table_id"]
+ ["workspace_output_external" "global_table_id"]
+ ["workspace_output_external" "isolated_table_id"]
+ ["workspace_input_external" "table_id"]]]
+ {:select [[[:inline 1] :ref]]
+ :from [(keyword table-name)]
+ :where [:= (keyword column-name) (:id table)]}))}))]
+ (when-not referenced?
+ (t2/delete! :model/Table :id (:id table)))))))
(defn gc-transform-target-tables!
"Deletes provisional table rows (created by [[upsert-transform-target-table!]]) that are no longer
- referenced by any Transform or WorkspaceTransform. Safe because these rows were never active,
+ referenced by any Transform or workspace table. Safe because these rows were never active,
so they have no child records.
Note: this only handles *inactive* provisional tables. Active tables that were previously
@@ -449,29 +468,31 @@
A separate process to reset `transform_target` on active, unreferenced tables is not yet
implemented.
- Future optimizations:
- - Add target_table_id FKs to avoid scanning transform rows.
- - Pre-filter via workspace_input/workspace_output table references."
+ Uses FK-based NOT IN queries rather than scanning JSON columns."
[]
- (let [candidates (t2/select [:model/Table :id :db_id :schema :name]
- :active false :transform_target true :deactivated_at nil
- :data_source :metabase-transform)]
- (when (seq candidates)
- (let [db-ids (into #{} (map :db_id) candidates)
- triple->table (into {} (map (juxt (juxt :db_id :schema :name) identity)) candidates)
- remaining (remove-referenced-candidates
- triple->table
- (eduction
- (map (fn [{:keys [target target_db_id]}]
- (-> target
- (update :database #(or % target_db_id))
- workspaces/target->triple)))
- (t2/reducible-select [:model/Transform :target :target_db_id] :target_db_id [:in db-ids])))
- remaining (when (seq remaining)
- (remove-referenced-candidates
- remaining
- (workspaces/reducible-target-triples db-ids)))]
- (when-let [dead-ids (seq (mapv :id (vals remaining)))]
+ (let [candidate-ids (t2/select-fn-set :id :model/Table
+ :active false :transform_target true :deactivated_at nil
+ :data_source :metabase-transform)]
+ (when (seq candidate-ids)
+ (let [referenced-ids
+ (into #{}
+ (map :id)
+ (t2/query {:union
+ (for [[table-name column-name]
+ [["transform" "target_table_id"]
+ ["workspace_input" "table_id"]
+ ["workspace_output" "global_table_id"]
+ ["workspace_output" "isolated_table_id"]
+ ["workspace_output_external" "global_table_id"]
+ ["workspace_output_external" "isolated_table_id"]
+ ["workspace_input_external" "table_id"]]]
+ {:select [[(keyword column-name) :id]]
+ :from [(keyword table-name)]
+ :where [:and
+ [:not= (keyword column-name) nil]
+ [:in (keyword column-name) candidate-ids]]})}))
+ dead-ids (into [] (remove referenced-ids) candidate-ids)]
+ (when (seq dead-ids)
(log/infof "Deleting %d orphaned transform target table(s)" (count dead-ids))
(t2/delete! :model/Table :id [:in dead-ids]))))))
diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.clj
index ab8554da214..bb34305a80b 100644
--- a/test/metabase/transforms/util_test.clj
+++ b/test/metabase/transforms/util_test.clj
@@ -364,12 +364,14 @@
(let [table-id (t2/select-one-fn :id :model/Table :db_id (mt/id) :name "test_output_table" :schema nil)]
;; Mock execute-base! to return success without actually running a query,
;; run-cancelable-transform! to bypass schema creation / cancellation infra,
- ;; and sync/indexes to skip driver calls. complete-execution! runs normally
- ;; so it sets transform_id on the target table.
- (with-redefs [transforms-base.i/execute-base! (constantly {:status :succeeded})
- transforms-base.u/sync-target! (constantly nil)
- transforms.u/run-cancelable-transform! (fn [_run-id _transform _driver _details run-fn & _opts]
- (run-fn (a/promise-chan) nil))]
+ ;; and sync-target! to skip driver calls but still return the provisional table
+ ;; so complete-execution! can set transform_id on it.
+ (mt/with-dynamic-fn-redefs
+ [transforms-base.i/execute-base! (constantly {:status :succeeded})
+ transforms-base.u/sync-target! (fn [_target _database]
+ (t2/select-one :model/Table table-id))
+ transforms.u/run-cancelable-transform! (fn [_run-id _transform _driver _details run-fn & _opts]
+ (run-fn (a/promise-chan) nil))]
(transforms.execute/execute! transform {:run-method :manual})
(is (= transform-id
(t2/select-one-fn :transform_id :model/Table :id table-id)))))))))))