Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add metabase_table.transform_id column (#69125)

Nathan Voxlandeeea8c5f3f53
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 8d7a7fd6c4e..ec491f5f30f 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1714,7 +1714,6 @@
api
app-db
audit-app
- config
driver
lib
lib-be
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
index aa184684f22..95d1f71841b 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
@@ -296,7 +296,7 @@
:dashboard [:id :name :description :created_at :creator_id :collection_id :view_count]
:document [:id :name :created_at :creator_id :collection_id :view_count]
:table [:id :name :description :display_name :db_id :schema
- :owner_user_id :owner_email]
+ :owner_user_id :owner_email :transform_id]
:transform [:id :name :description :creator_id
;; :source has to be selected otherwise the BE won't know what DB it belongs to
:source
diff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yaml
index f6dcda8faf0..48b704477f0 100644
--- a/resources/migrations/059_update_migrations.yaml
+++ b/resources/migrations/059_update_migrations.yaml
@@ -2933,6 +2933,44 @@ databaseChangeLog:
constraintName: fk_transform_target_db_id
onDelete: SET NULL
+ - changeSet:
+ id: v59.2026-02-04T10:45:00
+ author: nvoxland
+ comment: Add transform_id column to metabase_table
+ changes:
+ - addColumn:
+ tableName: metabase_table
+ columns:
+ - column:
+ name: transform_id
+ type: int
+ remarks: Foreign key to the transform that created this table
+
+ - changeSet:
+ id: v59.2026-02-04T10:45:01
+ author: nvoxland
+ comment: Add index for metabase_table.transform_id
+ changes:
+ - createIndex:
+ tableName: metabase_table
+ indexName: idx_metabase_table_transform_id
+ columns:
+ - column:
+ name: transform_id
+
+ - changeSet:
+ id: v59.2026-02-04T10:45:02
+ author: nvoxland
+ comment: Add foreign key on metabase_table(transform_id) to transform(id)
+ changes:
+ - addForeignKeyConstraint:
+ baseTableName: metabase_table
+ baseColumnNames: transform_id
+ referencedTableName: transform
+ referencedColumnNames: id
+ constraintName: fk_metabase_table_transform_id
+ onDelete: SET NULL
+
# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
########################################################################################################################
diff --git a/src/metabase/transforms/util.clj b/src/metabase/transforms/util.clj
index a91da0f7637..5afaad54ba6 100644
--- a/src/metabase/transforms/util.clj
+++ b/src/metabase/transforms/util.clj
@@ -716,7 +716,8 @@
[:db [:fn {:error/message "Must a t2 database object"} #(= (t2/model %) :model/Database)]]]]
(let [target (:target transform)]
(transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]
- (sync-target! target db)
+ (when-let [table (sync-target! target db)]
+ (t2/update! :model/Table (:id table) {:transform_id (:id transform)}))
;; This event must be published only after the sync is complete - the new table needs to be in AppDB.
(events/publish-event! :event/transform-run-complete
{:object {:db-id (:id db)
diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.clj
index ab11ebf6e7b..12654e98306 100644
--- a/src/metabase/warehouse_schema/models/table.clj
+++ b/src/metabase/warehouse_schema/models/table.clj
@@ -3,7 +3,6 @@
[metabase.api.common :as api]
[metabase.app-db.core :as app-db]
[metabase.audit-app.core :as audit]
- [metabase.config.core :as config]
[metabase.driver :as driver]
[metabase.models.humanization :as humanization]
[metabase.models.interface :as mi]
@@ -454,21 +453,19 @@
(methodical/defmethod t2/batched-hydrate [:model/Table :transform]
"Hydrate transforms that created the tables."
[_model k tables]
- (if config/ee-available?
- (mi/instances-with-hydrated-data
- tables k
- #(let [table-ids (map :id tables)
- table-id->transform-id (t2/select-fn->fn :from_entity_id :to_entity_id :model/Dependency
- :from_entity_type "table"
- :from_entity_id [:in table-ids]
- :to_entity_type "transform")
- transform-id->transform (when-let [transform-ids (seq (vals table-id->transform-id))]
- (t2/select-fn->fn :id identity :model/Transform :id [:in transform-ids]))]
- (update-vals table-id->transform-id transform-id->transform))
- :id
- {:default nil})
- ;; EE not available, so no transforms
- tables))
+ (mi/instances-with-hydrated-data
+ tables k
+ #(let [transform-ids (->> tables (keep :transform_id) distinct)
+ id->transform (when (seq transform-ids)
+ (t2/select-fn->fn :id identity :model/Transform
+ :id [:in transform-ids]))]
+ (into {}
+ (keep (fn [{:keys [id transform_id]}]
+ (when transform_id
+ [id (get id->transform transform_id)])))
+ tables))
+ :id
+ {:default nil}))
(methodical/defmethod t2/batched-hydrate [:model/Table :pk_field]
[_model k tables]
@@ -595,7 +592,8 @@
:deactivated_at (serdes/date)
:data_layer (serdes/optional-kw)
:db_id (serdes/fk :model/Database :name)
- :collection_id (serdes/fk :model/Collection)}})
+ :collection_id (serdes/fk :model/Collection)
+ :transform_id (serdes/fk :model/Transform)}})
(defmethod serdes/storage-path "Table" [table _ctx]
(concat (serdes/storage-path-prefixes (serdes/path table))
diff --git a/test/metabase/sync/sync_test.clj b/test/metabase/sync/sync_test.clj
index 33d450f66f4..035c8f9655c 100644
--- a/test/metabase/sync/sync_test.clj
+++ b/test/metabase/sync/sync_test.clj
@@ -126,7 +126,8 @@
:archived_at false
:deactivated_at false
:updated_at true
- :owner_user_id false}))
+ :owner_user_id false
+ :transform_id false}))
(defn- field-defaults []
(merge
diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.clj
index cac57184531..4ee72b294af 100644
--- a/test/metabase/transforms/util_test.clj
+++ b/test/metabase/transforms/util_test.clj
@@ -5,11 +5,13 @@
[metabase.api.common :as api]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
+ [metabase.events.core :as events]
[metabase.permissions.models.data-permissions :as data-perms]
[metabase.permissions.models.permissions-group :as perms-group]
[metabase.test :as mt]
[metabase.test.data.sql :as sql.tx]
- [metabase.transforms.util :as transforms.util]))
+ [metabase.transforms.util :as transforms.util]
+ [toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -316,3 +318,36 @@
(binding [api/*current-user-id* (:id user)]
(is (false? (transforms.util/source-tables-readable? transform))
"User who cannot read all source tables should have source_readable=false")))))))))))))
+
+(deftest handle-transform-complete-sets-transform-id-test
+ (testing "handle-transform-complete! sets transform_id on the target table"
+ (mt/with-premium-features #{:transforms}
+ (let [target {:type "table" :schema nil :name "test_output_table"}]
+ (mt/with-temp [:model/Database {db-id :id :as db} {:engine :h2}
+ :model/Transform {transform-id :id :as transform} {:target target}
+ :model/Table {table-id :id :as table} {:db_id db-id :name "test_output_table" :schema nil}
+ :model/TransformRun {run-id :id} {:transform_id transform-id
+ :status "running"
+ :run_method "manual"}]
+ (with-redefs [transforms.util/sync-target! (constantly table)
+ events/publish-event! (constantly nil)
+ transforms.util/execute-secondary-index-ddl-if-required! (constantly nil)]
+ (transforms.util/handle-transform-complete! :run-id run-id :transform transform :db db)
+ (is (= transform-id
+ (t2/select-one-fn :transform_id :model/Table :id table-id)))))))))
+
+(deftest transform-hydration-test
+ (testing "hydrating :transform on a table"
+ (mt/with-premium-features #{:transforms}
+ (let [target {:type "table" :schema nil :name "hydration_test_table"}]
+ (mt/with-temp [:model/Transform {transform-id :id} {:target target :name "Test Hydration Transform"}
+ :model/Table table {:transform_id transform-id}]
+ (let [hydrated (t2/hydrate table :transform)]
+ (is (some? (:transform hydrated)))
+ (is (= transform-id (-> hydrated :transform :id))))))))
+
+ (testing "hydrating :transform returns nil when transform_id is nil"
+ (mt/with-premium-features #{:transforms}
+ (mt/with-temp [:model/Table table {:transform_id nil}]
+ (let [hydrated (t2/hydrate table :transform)]
+ (is (nil? (:transform hydrated))))))))
diff --git a/test_resources/serialization_baseline/databases/My Database/tables/Schemaless Table/Schemaless Table.yaml b/test_resources/serialization_baseline/databases/My Database/tables/Schemaless Table/Schemaless Table.yaml
index fbb7740515e..a26b69db4f6 100644
--- a/test_resources/serialization_baseline/databases/My Database/tables/Schemaless Table/Schemaless Table.yaml
+++ b/test_resources/serialization_baseline/databases/My Database/tables/Schemaless Table/Schemaless Table.yaml
@@ -29,3 +29,4 @@ is_published: false
is_writable: null
owner_email: null
owner_user_id: null
+transform_id: null
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CATEGORIES/CATEGORIES.yaml b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CATEGORIES/CATEGORIES.yaml
index ac332054ec8..7c55d83630f 100644
--- a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CATEGORIES/CATEGORIES.yaml
+++ b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CATEGORIES/CATEGORIES.yaml
@@ -31,3 +31,4 @@ is_published: false
is_writable: null
owner_email: null
owner_user_id: null
+transform_id: null
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CHECKINS/CHECKINS.yaml b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CHECKINS/CHECKINS.yaml
index 8db321a7d6e..b087766f73e 100644
--- a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CHECKINS/CHECKINS.yaml
+++ b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CHECKINS/CHECKINS.yaml
@@ -31,3 +31,4 @@ is_published: false
is_writable: null
owner_email: null
owner_user_id: null
+transform_id: null
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/ORDERS.yaml b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/ORDERS.yaml
index da7cb68fd55..744909365fe 100644
--- a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/ORDERS.yaml
+++ b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/ORDERS.yaml
@@ -31,3 +31,4 @@ is_published: false
is_writable: null
owner_email: null
owner_user_id: null
+transform_id: null
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/PEOPLE.yaml b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/PEOPLE.yaml
index 9d757511302..67299df928c 100644
--- a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/PEOPLE.yaml
+++ b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/PEOPLE.yaml
@@ -31,3 +31,4 @@ is_published: false
is_writable: null
owner_email: null
owner_user_id: null
+transform_id: null
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/PRODUCTS.yaml b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/PRODUCTS.yaml
index f1a73db23f6..b5e7592a978 100644
--- a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/PRODUCTS.yaml
+++ b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/PRODUCTS.yaml
@@ -31,3 +31,4 @@ is_published: false
is_writable: null
owner_email: null
owner_user_id: null
+transform_id: null
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/REVIEWS.yaml b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/REVIEWS.yaml
index 06ce03a1c32..f6b018b7774 100644
--- a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/REVIEWS.yaml
+++ b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/REVIEWS.yaml
@@ -31,3 +31,4 @@ is_published: false
is_writable: null
owner_email: null
owner_user_id: null
+transform_id: null
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/USERS.yaml b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/USERS.yaml
index 85e1abbe330..b7be1900da5 100644
--- a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/USERS.yaml
+++ b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/USERS.yaml
@@ -31,3 +31,4 @@ is_published: false
is_writable: null
owner_email: null
owner_user_id: null
+transform_id: null
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/VENUES.yaml b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/VENUES.yaml
index 24f14b9552d..cfec668ab14 100644
--- a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/VENUES.yaml
+++ b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/VENUES.yaml
@@ -31,3 +31,4 @@ is_published: false
is_writable: null
owner_email: null
owner_user_id: null
+transform_id: null
diff --git a/test_resources/serialization_baseline/databases/test-data (h2)/tables/linked_table/linked_table.yaml b/test_resources/serialization_baseline/databases/test-data (h2)/tables/linked_table/linked_table.yaml
index 9933feb7aa2..0292d30d8a4 100644
--- a/test_resources/serialization_baseline/databases/test-data (h2)/tables/linked_table/linked_table.yaml
+++ b/test_resources/serialization_baseline/databases/test-data (h2)/tables/linked_table/linked_table.yaml
@@ -29,3 +29,4 @@ is_published: false
is_writable: null
owner_email: null
owner_user_id: null
+transform_id: null