Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Optimize transform target table backfill migrations (#71487)

Chris Truter7180c6ae4bc4
diff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.clj
index 6ec500bcea8..44fb49b2fdf 100644
--- a/src/metabase/app_db/custom_migrations.clj
+++ b/src/metabase/app_db/custom_migrations.clj
@@ -2106,11 +2106,32 @@
["copper" "bronze" "silver" "gold"]
[:case [:= :data_layer "hidden"] "copper" :else "bronze"]))))
+(defn- find-table-id
+ "Find a metabase_table by db-id, schema, and name using case-insensitive matching,
+ preferring an exact case match when one exists."
+ [db-id schema table-name]
+ (:id (first (t2/query {:select [:id]
+ :from [:metabase_table]
+ :where [:and
+ [:= :db_id db-id]
+ (if (some? schema)
+ [:= [:lower :schema] [:lower schema]]
+ [:is :schema nil])
+ [:= [:lower :name] [:lower table-name]]]
+ :order-by [[[:case
+ [:and
+ [:= :name table-name]
+ (if (some? schema)
+ [:= :schema schema]
+ [:is :schema nil])]
+ 0
+ :else 1]
+ :asc]]
+ :limit 1}))))
+
(define-migration BackfillTransformTargetTables
;; For each transform that has a target (database, schema, name), ensure a metabase_table row exists.
;; If the table already exists (active or inactive), do nothing. Otherwise, insert a transform target row.
- ;;
- ;; Similarly, backfill workspace_output rows missing global_table_id or isolated_table_id.
(let [;; Reproduces humanization/name->human-readable-name :simple (can't call library code in migrations)
acronyms #{"id" "url" "ip" "uid" "uuid" "guid"}
cap-word (fn [word]
@@ -2126,16 +2147,6 @@
:when (not (str/blank? part))]
(cap-word part)))]
(if (str/blank? result) s result))))
- find-table-id (fn [db-id schema table-name]
- (:id (first (t2/query {:select [:id]
- :from [:metabase_table]
- :where [:and
- [:= :db_id db-id]
- (if (some? schema)
- [:= [:lower :schema] (lower-case-en schema)]
- [:is :schema nil])
- [:= [:lower :name] (lower-case-en table-name)]]
- :limit 1}))))
upsert-table! (fn [db-id schema table-name]
(or (find-table-id db-id schema table-name)
(try
@@ -2154,77 +2165,44 @@
(find-table-id db-id schema table-name)
(catch Exception _
(find-table-id db-id schema table-name)))))]
- ;; 1. Backfill for transforms: create provisional tables for targets that don't exist
- (doseq [{:keys [target target_db_id]} (t2/query {:select [:target :target_db_id]
- :from [:transform]
- :where [:not= :target_db_id nil]})]
- (let [target-map (json-out target false)]
- (when-let [table-name (get target-map "name")]
- (let [schema (get target-map "schema")
- db-id target_db_id]
- (upsert-table! db-id schema table-name)))))
- ;; 2-5. Backfill workspace_output and workspace_output_external rows missing table FKs
- (let [backfill-table-fk! (fn [from-table schema-col table-col fk-col]
- (doseq [{:keys [id db_id] :as row}
- (t2/query {:select [:id :db_id schema-col table-col]
- :from [from-table]
- :where [:= fk-col nil]})]
- (when-let [table-id (upsert-table! db_id (get row schema-col) (get row table-col))]
- (t2/query {:update from-table
- :set {fk-col table-id}
- :where [:= :id id]}))))]
- (backfill-table-fk! :workspace_output :global_schema :global_table :global_table_id)
- (backfill-table-fk! :workspace_output :isolated_schema :isolated_table :isolated_table_id)
- (backfill-table-fk! :workspace_output_external :global_schema :global_table :global_table_id)
- (backfill-table-fk! :workspace_output_external :isolated_schema :isolated_table :isolated_table_id))
- ;; 6-7. Backfill workspace_input and workspace_input_external rows missing table_id
- (doseq [from-table [:workspace_input :workspace_input_external]
- :let [rows (t2/query {:select [:id :db_id :schema :table]
- :from [from-table]
- :where [:= :table_id nil]})]
- {:keys [id db_id schema table]} rows]
- (when-let [table-id (find-table-id db_id schema table)]
- (t2/query {:update from-table
- :set {:table_id table-id}
- :where [:= :id id]})))))
+ ;; Create provisional tables for transform targets that don't exist yet
+ (run! (fn [{:keys [target target_db_id]}]
+ (let [target-map (json-out target false)]
+ (when-let [table-name (get target-map "name")]
+ (let [schema (get target-map "schema")
+ db-id target_db_id]
+ (upsert-table! db-id schema table-name)))))
+ (t2/reducible-query {:select [:target :target_db_id]
+ :from [:transform]
+ :where [:not= :target_db_id nil]})))
+ ;; Invalidate workspace caches so the app re-analyzes and backfills workspace_ table FKs lazily
+ (t2/query {:update :workspace_transform
+ :set {:analysis_version [:+ :analysis_version 1]
+ :definition_changed true}})
+ (t2/query {:update :workspace
+ :set {:graph_version [:+ :graph_version 1]}}))
(define-migration BackfillTransformTargetTableId
;; For each transform with a non-null target_db_id, extract the table_id from the target JSON column.
;; If table_id is present in the JSON, use it directly.
;; If table_id is missing but name is present, look up the table by (db_id, schema, name).
- ;; Uses case-insensitive matching with exact-case preference for both schema and name.
- (let [find-table-id (fn [db-id schema table-name]
- (:id (first (t2/query {:select [:id]
- :from [:metabase_table]
- :where [:and
- [:= :db_id db-id]
- (if (some? schema)
- [:= [:lower :schema] [:lower schema]]
- [:is :schema nil])
- [:= [:lower :name] [:lower table-name]]]
- :order-by [[[:case
- [:and
- [:= :name table-name]
- (if (some? schema)
- [:= :schema schema]
- [:is :schema nil])]
- 0
- :else 1]
- :asc]]
- :limit 1}))))]
- (doseq [{:keys [id target target_db_id]} (t2/query {:select [:id :target :target_db_id]
- :from [:transform]
- :where [:and
- [:not= :target_db_id nil]
- [:= :target_table_id nil]]})]
- (let [target-map (json-out target false)
- table-id (or (get target-map "table_id")
- (when-let [table-name (get target-map "name")]
- (find-table-id target_db_id (get target-map "schema") table-name)))]
- (when table-id
- (t2/query {:update :transform
- :set {:target_table_id table-id}
- :where [:= :id id]})))))
- ;; Invalidate workspace graph caches so they recalculate with target_table_id
+ (run! (fn [{:keys [id target target_db_id]}]
+ (let [target-map (json-out target false)
+ table-id (or (get target-map "table_id")
+ (when-let [table-name (get target-map "name")]
+ (find-table-id target_db_id (get target-map "schema") table-name)))]
+ (when table-id
+ (t2/query {:update :transform
+ :set {:target_table_id table-id}
+ :where [:= :id id]}))))
+ (t2/reducible-query {:select [:id :target :target_db_id]
+ :from [:transform]
+ :where [:and
+ [:not= :target_db_id nil]
+ [:= :target_table_id nil]]}))
+ ;; Invalidate workspace caches so they recalculate with target_table_id
+ (t2/query {:update :workspace_transform
+ :set {:analysis_version [:+ :analysis_version 1]
+ :definition_changed true}})
(t2/query {:update :workspace
:set {:graph_version [:+ :graph_version 1]}}))
diff --git a/test/metabase/app_db/custom_migrations_test.clj b/test/metabase/app_db/custom_migrations_test.clj
index 0224f505ab7..049fc1527c8 100644
--- a/test/metabase/app_db/custom_migrations_test.clj
+++ b/test/metabase/app_db/custom_migrations_test.clj
@@ -3017,7 +3017,7 @@
(is (= 99 (get st "products")))))))))
(deftest backfill-transform-target-tables-test
- (testing "v60.2026-03-07T00:00:04 : backfill transform target tables and workspace FK columns"
+ (testing "v60.2026-03-07T00:00:04 : backfill transform target tables and invalidate workspace caches"
(impl/test-migrations ["v60.2026-03-07T00:00:04"] [migrate!]
(let [user-id (:id (new-instance-with-default :core_user))
db-id (:id (new-instance-with-default :metabase_database))
@@ -3026,11 +3026,9 @@
:creator_id user-id
:created_at :%now
:updated_at :%now}))
- ref-id (str (random-uuid))
source (json/encode {:type "query" :query {:database db-id}})
- target (json/encode {:type "table" :schema "public" :name "orders"})
- ;; -- 1. Transform with a target that has no existing metabase_table → should create provisional row --
- tx-id (t2/insert-returning-pk!
+ ;; -- Transform with a target that has no existing metabase_table → should create provisional row --
+ _ (t2/insert-returning-pk!
:transform {:name "create-output"
:source source
:target (json/encode {:type "table" :schema "public" :name "new_target_table"})
@@ -3039,77 +3037,18 @@
:target_db_id db-id
:created_at :%now
:updated_at :%now})
- ;; -- An existing metabase_table that inputs and outputs should resolve to --
- table-id (:id (t2/insert-returning-instance!
- :metabase_table {:db_id db-id
- :name "orders"
- :schema "public"
- :active true
- :created_at :%now
- :updated_at :%now}))
- ;; -- workspace_transform needed for workspace_output FK --
+ ;; -- workspace_transform to verify analysis_version bump --
_ (t2/insert! :workspace_transform
- {:ref_id ref-id
+ {:ref_id (str (random-uuid))
:workspace_id ws-id
:name "ws-tx"
:source source
- :target target
+ :target (json/encode {:type "table" :schema "public" :name "orders"})
:created_at :%now
- :updated_at :%now})
- ;; -- 2-3. workspace_output with null table FKs --
- wo-id (:id (t2/insert-returning-instance!
- :workspace_output {:workspace_id ws-id
- :ref_id ref-id
- :db_id db-id
- :global_schema "public"
- :global_table "orders"
- :isolated_schema "public"
- :isolated_table "orders"
- :created_at :%now
- :updated_at :%now}))
- ;; -- 4-5. workspace_output_external with null table FKs --
- woe-id (:id (t2/insert-returning-instance!
- :workspace_output_external {:workspace_id ws-id
- :transform_id tx-id
- :db_id db-id
- :global_schema "public"
- :global_table "orders"
- :isolated_schema "public"
- :isolated_table "orders"
- :created_at :%now
- :updated_at :%now}))
- ;; -- 6. workspace_input with null table_id --
- wi-id (:id (t2/insert-returning-instance!
- :workspace_input {:workspace_id ws-id
- :db_id db-id
- :schema "public"
- :table "orders"
- :created_at :%now
- :updated_at :%now}))
- ;; -- 7. workspace_input_external with null table_id --
- wie-id (:id (t2/insert-returning-instance!
- :workspace_input_external {:workspace_id ws-id
- :db_id db-id
- :schema "public"
- :table "orders"
- :created_at :%now
- :updated_at :%now}))
- ;; -- workspace_input_external with no matching table — should stay nil --
- wie-no-match-id (:id (t2/insert-returning-instance!
- :workspace_input_external {:workspace_id ws-id
- :db_id db-id
- :schema "public"
- :table "no_such_table"
- :created_at :%now
- :updated_at :%now}))]
- (testing "Before backfill, all FKs are nil"
- (is (nil? (t2/select-one-fn :global_table_id :workspace_output :id wo-id)))
- (is (nil? (t2/select-one-fn :isolated_table_id :workspace_output :id wo-id)))
- (is (nil? (t2/select-one-fn :global_table_id :workspace_output_external :id woe-id)))
- (is (nil? (t2/select-one-fn :isolated_table_id :workspace_output_external :id woe-id)))
- (is (nil? (t2/select-one-fn :table_id :workspace_input :id wi-id)))
- (is (nil? (t2/select-one-fn :table_id :workspace_input_external :id wie-id)))
- (is (nil? (t2/select-one-fn :table_id :workspace_input_external :id wie-no-match-id))))
+ :updated_at :%now})]
+ (testing "Before migration"
+ (is (= 1 (:graph_version (t2/select-one :workspace :id ws-id))))
+ (is (= 1 (:analysis_version (first (t2/select :workspace_transform :workspace_id ws-id))))))
(migrate!)
(testing "Provisional metabase_table created for transform target"
(let [provisional (first (t2/query {:select [:active :transform_target :data_source :data_authority :display_name]
@@ -3124,15 +3063,7 @@
(is (= "metabase-transform" (:data_source provisional)))
(is (= "computed" (:data_authority provisional)))
(is (= "New Target Table" (:display_name provisional)))))
- (testing "workspace_output FKs backfilled"
- (is (= table-id (t2/select-one-fn :global_table_id :workspace_output :id wo-id)))
- (is (= table-id (t2/select-one-fn :isolated_table_id :workspace_output :id wo-id))))
- (testing "workspace_output_external FKs backfilled"
- (is (= table-id (t2/select-one-fn :global_table_id :workspace_output_external :id woe-id)))
- (is (= table-id (t2/select-one-fn :isolated_table_id :workspace_output_external :id woe-id))))
- (testing "workspace_input table_id backfilled"
- (is (= table-id (t2/select-one-fn :table_id :workspace_input :id wi-id))))
- (testing "workspace_input_external table_id backfilled"
- (is (= table-id (t2/select-one-fn :table_id :workspace_input_external :id wie-id))))
- (testing "Non-matching input stays nil"
- (is (nil? (t2/select-one-fn :table_id :workspace_input_external :id wie-no-match-id))))))))
+ (testing "Workspace caches invalidated"
+ ;; Both BackfillTransformTargetTables and BackfillTransformTargetTableId bump these
+ (is (= 3 (:graph_version (t2/select-one :workspace :id ws-id))))
+ (is (= 3 (:analysis_version (first (t2/select :workspace_transform :workspace_id ws-id))))))))))