The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add metabase_table.transform_id column (#69125)
Nathan Voxlandeeea8c5f3f53
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 8d7a7fd6c4e..ec491f5f30f 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1714,7 +1714,6 @@apiapp-dbaudit-app- configdriverliblib-bediff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.cljindex aa184684f22..95d1f71841b 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj@@ -296,7 +296,7 @@:dashboard [:id :name :description :created_at :creator_id :collection_id :view_count]:document [:id :name :created_at :creator_id :collection_id :view_count]:table [:id :name :description :display_name :db_id :schema- :owner_user_id :owner_email]+ :owner_user_id :owner_email :transform_id]:transform [:id :name :description :creator_id;; :source has to be selected otherwise the BE won't know what DB it belongs to:sourcediff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yamlindex f6dcda8faf0..48b704477f0 100644--- a/resources/migrations/059_update_migrations.yaml+++ b/resources/migrations/059_update_migrations.yaml@@ -2933,6 +2933,44 @@ databaseChangeLog:constraintName: fk_transform_target_db_idonDelete: SET NULL+ - changeSet:+ id: v59.2026-02-04T10:45:00+ author: nvoxland+ comment: Add transform_id column to metabase_table+ changes:+ - addColumn:+ tableName: metabase_table+ columns:+ - column:+ name: transform_id+ type: int+ remarks: Foreign key to the transform that created this table++ - changeSet:+ id: v59.2026-02-04T10:45:01+ author: nvoxland+ comment: Add index for metabase_table.transform_id+ changes:+ - createIndex:+ tableName: metabase_table+ indexName: idx_metabase_table_transform_id+ columns:+ - column:+ name: transform_id++ - changeSet:+ id: v59.2026-02-04T10:45:02+ author: nvoxland+ comment: Add foreign key on metabase_table(transform_id) to transform(id)+ changes:+ - addForeignKeyConstraint:+ baseTableName: metabase_table+ baseColumnNames: transform_id+ referencedTableName: transform+ referencedColumnNames: id+ constraintName: fk_metabase_table_transform_id+ onDelete: SET NULL+# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<########################################################################################################################diff --git a/src/metabase/transforms/util.clj b/src/metabase/transforms/util.cljindex a91da0f7637..5afaad54ba6 100644--- a/src/metabase/transforms/util.clj+++ b/src/metabase/transforms/util.clj@@ -716,7 +716,8 @@[:db [:fn {:error/message "Must a t2 database object"} #(= (t2/model %) :model/Database)]]]](let [target (:target transform)](transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]- (sync-target! target db)+ (when-let [table (sync-target! target db)]+ (t2/update! :model/Table (:id table) {:transform_id (:id transform)}));; This event must be published only after the sync is complete - the new table needs to be in AppDB.(events/publish-event! :event/transform-run-complete{:object {:db-id (:id db)diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.cljindex ab11ebf6e7b..12654e98306 100644--- a/src/metabase/warehouse_schema/models/table.clj+++ b/src/metabase/warehouse_schema/models/table.clj@@ -3,7 +3,6 @@[metabase.api.common :as api][metabase.app-db.core :as app-db][metabase.audit-app.core :as audit]- [metabase.config.core :as config][metabase.driver :as driver][metabase.models.humanization :as humanization][metabase.models.interface :as mi]@@ -454,21 +453,19 @@(methodical/defmethod t2/batched-hydrate [:model/Table :transform]"Hydrate transforms that created the tables."[_model k tables]- (if config/ee-available?- (mi/instances-with-hydrated-data- tables k- #(let [table-ids (map :id tables)- table-id->transform-id (t2/select-fn->fn :from_entity_id :to_entity_id :model/Dependency- :from_entity_type "table"- :from_entity_id [:in table-ids]- :to_entity_type "transform")- transform-id->transform (when-let [transform-ids (seq (vals table-id->transform-id))]- (t2/select-fn->fn :id identity :model/Transform :id [:in transform-ids]))]- (update-vals table-id->transform-id transform-id->transform))- :id- {:default nil})- ;; EE not available, so no transforms- tables))+ (mi/instances-with-hydrated-data+ tables k+ #(let [transform-ids (->> tables (keep :transform_id) distinct)+ id->transform (when (seq transform-ids)+ (t2/select-fn->fn :id identity :model/Transform+ :id [:in transform-ids]))]+ (into {}+ (keep (fn [{:keys [id transform_id]}]+ (when transform_id+ [id (get id->transform transform_id)])))+ tables))+ :id+ {:default nil}))(methodical/defmethod t2/batched-hydrate [:model/Table :pk_field][_model k tables]@@ -595,7 +592,8 @@:deactivated_at (serdes/date):data_layer (serdes/optional-kw):db_id (serdes/fk :model/Database :name)- :collection_id (serdes/fk :model/Collection)}})+ :collection_id (serdes/fk :model/Collection)+ :transform_id (serdes/fk :model/Transform)}})(defmethod serdes/storage-path "Table" [table _ctx](concat (serdes/storage-path-prefixes (serdes/path table))diff --git a/test/metabase/sync/sync_test.clj b/test/metabase/sync/sync_test.cljindex 33d450f66f4..035c8f9655c 100644--- a/test/metabase/sync/sync_test.clj+++ b/test/metabase/sync/sync_test.clj@@ -126,7 +126,8 @@:archived_at false:deactivated_at false:updated_at true- :owner_user_id false}))+ :owner_user_id false+ :transform_id false}))(defn- field-defaults [](mergediff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.cljindex cac57184531..4ee72b294af 100644--- a/test/metabase/transforms/util_test.clj+++ b/test/metabase/transforms/util_test.clj@@ -5,11 +5,13 @@[metabase.api.common :as api][metabase.driver :as driver][metabase.driver.util :as driver.u]+ [metabase.events.core :as events][metabase.permissions.models.data-permissions :as data-perms][metabase.permissions.models.permissions-group :as perms-group][metabase.test :as mt][metabase.test.data.sql :as sql.tx]- [metabase.transforms.util :as transforms.util]))+ [metabase.transforms.util :as transforms.util]+ [toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -316,3 +318,36 @@(binding [api/*current-user-id* (:id user)](is (false? (transforms.util/source-tables-readable? transform))"User who cannot read all source tables should have source_readable=false")))))))))))))++(deftest handle-transform-complete-sets-transform-id-test+ (testing "handle-transform-complete! sets transform_id on the target table"+ (mt/with-premium-features #{:transforms}+ (let [target {:type "table" :schema nil :name "test_output_table"}]+ (mt/with-temp [:model/Database {db-id :id :as db} {:engine :h2}+ :model/Transform {transform-id :id :as transform} {:target target}+ :model/Table {table-id :id :as table} {:db_id db-id :name "test_output_table" :schema nil}+ :model/TransformRun {run-id :id} {:transform_id transform-id+ :status "running"+ :run_method "manual"}]+ (with-redefs [transforms.util/sync-target! (constantly table)+ events/publish-event! (constantly nil)+ transforms.util/execute-secondary-index-ddl-if-required! (constantly nil)]+ (transforms.util/handle-transform-complete! :run-id run-id :transform transform :db db)+ (is (= transform-id+ (t2/select-one-fn :transform_id :model/Table :id table-id)))))))))++(deftest transform-hydration-test+ (testing "hydrating :transform on a table"+ (mt/with-premium-features #{:transforms}+ (let [target {:type "table" :schema nil :name "hydration_test_table"}]+ (mt/with-temp [:model/Transform {transform-id :id} {:target target :name "Test Hydration Transform"}+ :model/Table table {:transform_id transform-id}]+ (let [hydrated (t2/hydrate table :transform)]+ (is (some? (:transform hydrated)))+ (is (= transform-id (-> hydrated :transform :id))))))))++ (testing "hydrating :transform returns nil when transform_id is nil"+ (mt/with-premium-features #{:transforms}+ (mt/with-temp [:model/Table table {:transform_id nil}]+ (let [hydrated (t2/hydrate table :transform)]+ (is (nil? (:transform hydrated))))))))diff --git a/test_resources/serialization_baseline/databases/My Database/tables/Schemaless Table/Schemaless Table.yaml b/test_resources/serialization_baseline/databases/My Database/tables/Schemaless Table/Schemaless Table.yamlindex fbb7740515e..a26b69db4f6 100644--- a/test_resources/serialization_baseline/databases/My Database/tables/Schemaless Table/Schemaless Table.yaml+++ b/test_resources/serialization_baseline/databases/My Database/tables/Schemaless Table/Schemaless Table.yaml@@ -29,3 +29,4 @@ is_published: falseis_writable: nullowner_email: nullowner_user_id: null+transform_id: nulldiff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CATEGORIES/CATEGORIES.yaml b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CATEGORIES/CATEGORIES.yamlindex ac332054ec8..7c55d83630f 100644--- a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CATEGORIES/CATEGORIES.yaml+++ b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CATEGORIES/CATEGORIES.yaml@@ -31,3 +31,4 @@ is_published: falseis_writable: nullowner_email: nullowner_user_id: null+transform_id: nulldiff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CHECKINS/CHECKINS.yaml b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CHECKINS/CHECKINS.yamlindex 8db321a7d6e..b087766f73e 100644--- a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CHECKINS/CHECKINS.yaml+++ b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/CHECKINS/CHECKINS.yaml@@ -31,3 +31,4 @@ is_published: falseis_writable: nullowner_email: nullowner_user_id: null+transform_id: nulldiff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/ORDERS.yaml b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/ORDERS.yamlindex da7cb68fd55..744909365fe 100644--- a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/ORDERS.yaml+++ b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/ORDERS/ORDERS.yaml@@ -31,3 +31,4 @@ is_published: falseis_writable: nullowner_email: nullowner_user_id: null+transform_id: nulldiff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/PEOPLE.yaml b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/PEOPLE.yamlindex 9d757511302..67299df928c 100644--- a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/PEOPLE.yaml+++ b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PEOPLE/PEOPLE.yaml@@ -31,3 +31,4 @@ is_published: falseis_writable: nullowner_email: nullowner_user_id: null+transform_id: nulldiff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/PRODUCTS.yaml b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/PRODUCTS.yamlindex f1a73db23f6..b5e7592a978 100644--- a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/PRODUCTS.yaml+++ b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/PRODUCTS/PRODUCTS.yaml@@ -31,3 +31,4 @@ is_published: falseis_writable: nullowner_email: nullowner_user_id: null+transform_id: nulldiff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/REVIEWS.yaml b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/REVIEWS.yamlindex 06ce03a1c32..f6b018b7774 100644--- a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/REVIEWS.yaml+++ b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/REVIEWS/REVIEWS.yaml@@ -31,3 +31,4 @@ is_published: falseis_writable: nullowner_email: nullowner_user_id: null+transform_id: nulldiff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/USERS.yaml b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/USERS.yamlindex 85e1abbe330..b7be1900da5 100644--- a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/USERS.yaml+++ b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/USERS/USERS.yaml@@ -31,3 +31,4 @@ is_published: falseis_writable: nullowner_email: nullowner_user_id: null+transform_id: nulldiff --git a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/VENUES.yaml b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/VENUES.yamlindex 24f14b9552d..cfec668ab14 100644--- a/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/VENUES.yaml+++ b/test_resources/serialization_baseline/databases/test-data (h2)/schemas/PUBLIC/tables/VENUES/VENUES.yaml@@ -31,3 +31,4 @@ is_published: falseis_writable: nullowner_email: nullowner_user_id: null+transform_id: nulldiff --git a/test_resources/serialization_baseline/databases/test-data (h2)/tables/linked_table/linked_table.yaml b/test_resources/serialization_baseline/databases/test-data (h2)/tables/linked_table/linked_table.yamlindex 9933feb7aa2..0292d30d8a4 100644--- a/test_resources/serialization_baseline/databases/test-data (h2)/tables/linked_table/linked_table.yaml+++ b/test_resources/serialization_baseline/databases/test-data (h2)/tables/linked_table/linked_table.yaml@@ -29,3 +29,4 @@ is_published: falseis_writable: nullowner_email: nullowner_user_id: null+transform_id: null