The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Optimize transform target table backfill migrations (#71487)
Chris Truter7180c6ae4bc4
diff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.cljindex 6ec500bcea8..44fb49b2fdf 100644--- a/src/metabase/app_db/custom_migrations.clj+++ b/src/metabase/app_db/custom_migrations.clj@@ -2106,11 +2106,32 @@["copper" "bronze" "silver" "gold"][:case [:= :data_layer "hidden"] "copper" :else "bronze"]))))+(defn- find-table-id+ "Find a metabase_table by db-id, schema, and name using case-insensitive matching,+ preferring an exact case match when one exists."+ [db-id schema table-name]+ (:id (first (t2/query {:select [:id]+ :from [:metabase_table]+ :where [:and+ [:= :db_id db-id]+ (if (some? schema)+ [:= [:lower :schema] [:lower schema]]+ [:is :schema nil])+ [:= [:lower :name] [:lower table-name]]]+ :order-by [[[:case+ [:and+ [:= :name table-name]+ (if (some? schema)+ [:= :schema schema]+ [:is :schema nil])]+ 0+ :else 1]+ :asc]]+ :limit 1}))))+(define-migration BackfillTransformTargetTables;; For each transform that has a target (database, schema, name), ensure a metabase_table row exists.;; If the table already exists (active or inactive), do nothing. Otherwise, insert a transform target row.- ;;- ;; Similarly, backfill workspace_output rows missing global_table_id or isolated_table_id.(let [;; Reproduces humanization/name->human-readable-name :simple (can't call library code in migrations)acronyms #{"id" "url" "ip" "uid" "uuid" "guid"}cap-word (fn [word]@@ -2126,16 +2147,6 @@:when (not (str/blank? part))](cap-word part)))](if (str/blank? result) s result))))- find-table-id (fn [db-id schema table-name]- (:id (first (t2/query {:select [:id]- :from [:metabase_table]- :where [:and- [:= :db_id db-id]- (if (some? schema)- [:= [:lower :schema] (lower-case-en schema)]- [:is :schema nil])- [:= [:lower :name] (lower-case-en table-name)]]- :limit 1}))))upsert-table! (fn [db-id schema table-name](or (find-table-id db-id schema table-name)(try@@ -2154,77 +2165,44 @@(find-table-id db-id schema table-name)(catch Exception _(find-table-id db-id schema table-name)))))]- ;; 1. Backfill for transforms: create provisional tables for targets that don't exist- (doseq [{:keys [target target_db_id]} (t2/query {:select [:target :target_db_id]- :from [:transform]- :where [:not= :target_db_id nil]})]- (let [target-map (json-out target false)]- (when-let [table-name (get target-map "name")]- (let [schema (get target-map "schema")- db-id target_db_id]- (upsert-table! db-id schema table-name)))))- ;; 2-5. Backfill workspace_output and workspace_output_external rows missing table FKs- (let [backfill-table-fk! (fn [from-table schema-col table-col fk-col]- (doseq [{:keys [id db_id] :as row}- (t2/query {:select [:id :db_id schema-col table-col]- :from [from-table]- :where [:= fk-col nil]})]- (when-let [table-id (upsert-table! db_id (get row schema-col) (get row table-col))]- (t2/query {:update from-table- :set {fk-col table-id}- :where [:= :id id]}))))]- (backfill-table-fk! :workspace_output :global_schema :global_table :global_table_id)- (backfill-table-fk! :workspace_output :isolated_schema :isolated_table :isolated_table_id)- (backfill-table-fk! :workspace_output_external :global_schema :global_table :global_table_id)- (backfill-table-fk! :workspace_output_external :isolated_schema :isolated_table :isolated_table_id))- ;; 6-7. Backfill workspace_input and workspace_input_external rows missing table_id- (doseq [from-table [:workspace_input :workspace_input_external]- :let [rows (t2/query {:select [:id :db_id :schema :table]- :from [from-table]- :where [:= :table_id nil]})]- {:keys [id db_id schema table]} rows]- (when-let [table-id (find-table-id db_id schema table)]- (t2/query {:update from-table- :set {:table_id table-id}- :where [:= :id id]})))))+ ;; Create provisional tables for transform targets that don't exist yet+ (run! (fn [{:keys [target target_db_id]}]+ (let [target-map (json-out target false)]+ (when-let [table-name (get target-map "name")]+ (let [schema (get target-map "schema")+ db-id target_db_id]+ (upsert-table! db-id schema table-name)))))+ (t2/reducible-query {:select [:target :target_db_id]+ :from [:transform]+ :where [:not= :target_db_id nil]})))+ ;; Invalidate workspace caches so the app re-analyzes and backfills workspace_ table FKs lazily+ (t2/query {:update :workspace_transform+ :set {:analysis_version [:+ :analysis_version 1]+ :definition_changed true}})+ (t2/query {:update :workspace+ :set {:graph_version [:+ :graph_version 1]}}))(define-migration BackfillTransformTargetTableId;; For each transform with a non-null target_db_id, extract the table_id from the target JSON column.;; If table_id is present in the JSON, use it directly.;; If table_id is missing but name is present, look up the table by (db_id, schema, name).- ;; Uses case-insensitive matching with exact-case preference for both schema and name.- (let [find-table-id (fn [db-id schema table-name]- (:id (first (t2/query {:select [:id]- :from [:metabase_table]- :where [:and- [:= :db_id db-id]- (if (some? schema)- [:= [:lower :schema] [:lower schema]]- [:is :schema nil])- [:= [:lower :name] [:lower table-name]]]- :order-by [[[:case- [:and- [:= :name table-name]- (if (some? schema)- [:= :schema schema]- [:is :schema nil])]- 0- :else 1]- :asc]]- :limit 1}))))]- (doseq [{:keys [id target target_db_id]} (t2/query {:select [:id :target :target_db_id]- :from [:transform]- :where [:and- [:not= :target_db_id nil]- [:= :target_table_id nil]]})]- (let [target-map (json-out target false)- table-id (or (get target-map "table_id")- (when-let [table-name (get target-map "name")]- (find-table-id target_db_id (get target-map "schema") table-name)))]- (when table-id- (t2/query {:update :transform- :set {:target_table_id table-id}- :where [:= :id id]})))))- ;; Invalidate workspace graph caches so they recalculate with target_table_id+ (run! (fn [{:keys [id target target_db_id]}]+ (let [target-map (json-out target false)+ table-id (or (get target-map "table_id")+ (when-let [table-name (get target-map "name")]+ (find-table-id target_db_id (get target-map "schema") table-name)))]+ (when table-id+ (t2/query {:update :transform+ :set {:target_table_id table-id}+ :where [:= :id id]}))))+ (t2/reducible-query {:select [:id :target :target_db_id]+ :from [:transform]+ :where [:and+ [:not= :target_db_id nil]+ [:= :target_table_id nil]]}))+ ;; Invalidate workspace caches so they recalculate with target_table_id+ (t2/query {:update :workspace_transform+ :set {:analysis_version [:+ :analysis_version 1]+ :definition_changed true}})(t2/query {:update :workspace:set {:graph_version [:+ :graph_version 1]}}))diff --git a/test/metabase/app_db/custom_migrations_test.clj b/test/metabase/app_db/custom_migrations_test.cljindex 0224f505ab7..049fc1527c8 100644--- a/test/metabase/app_db/custom_migrations_test.clj+++ b/test/metabase/app_db/custom_migrations_test.clj@@ -3017,7 +3017,7 @@(is (= 99 (get st "products")))))))))(deftest backfill-transform-target-tables-test- (testing "v60.2026-03-07T00:00:04 : backfill transform target tables and workspace FK columns"+ (testing "v60.2026-03-07T00:00:04 : backfill transform target tables and invalidate workspace caches"(impl/test-migrations ["v60.2026-03-07T00:00:04"] [migrate!](let [user-id (:id (new-instance-with-default :core_user))db-id (:id (new-instance-with-default :metabase_database))@@ -3026,11 +3026,9 @@:creator_id user-id:created_at :%now:updated_at :%now}))- ref-id (str (random-uuid))source (json/encode {:type "query" :query {:database db-id}})- target (json/encode {:type "table" :schema "public" :name "orders"})- ;; -- 1. Transform with a target that has no existing metabase_table → should create provisional row --- tx-id (t2/insert-returning-pk!+ ;; -- Transform with a target that has no existing metabase_table → should create provisional row --+ _ (t2/insert-returning-pk!:transform {:name "create-output":source source:target (json/encode {:type "table" :schema "public" :name "new_target_table"})@@ -3039,77 +3037,18 @@:target_db_id db-id:created_at :%now:updated_at :%now})- ;; -- An existing metabase_table that inputs and outputs should resolve to --- table-id (:id (t2/insert-returning-instance!- :metabase_table {:db_id db-id- :name "orders"- :schema "public"- :active true- :created_at :%now- :updated_at :%now}))- ;; -- workspace_transform needed for workspace_output FK --+ ;; -- workspace_transform to verify analysis_version bump --_ (t2/insert! :workspace_transform- {:ref_id ref-id+ {:ref_id (str (random-uuid)):workspace_id ws-id:name "ws-tx":source source- :target target+ :target (json/encode {:type "table" :schema "public" :name "orders"}):created_at :%now- :updated_at :%now})- ;; -- 2-3. workspace_output with null table FKs --- wo-id (:id (t2/insert-returning-instance!- :workspace_output {:workspace_id ws-id- :ref_id ref-id- :db_id db-id- :global_schema "public"- :global_table "orders"- :isolated_schema "public"- :isolated_table "orders"- :created_at :%now- :updated_at :%now}))- ;; -- 4-5. workspace_output_external with null table FKs --- woe-id (:id (t2/insert-returning-instance!- :workspace_output_external {:workspace_id ws-id- :transform_id tx-id- :db_id db-id- :global_schema "public"- :global_table "orders"- :isolated_schema "public"- :isolated_table "orders"- :created_at :%now- :updated_at :%now}))- ;; -- 6. workspace_input with null table_id --- wi-id (:id (t2/insert-returning-instance!- :workspace_input {:workspace_id ws-id- :db_id db-id- :schema "public"- :table "orders"- :created_at :%now- :updated_at :%now}))- ;; -- 7. workspace_input_external with null table_id --- wie-id (:id (t2/insert-returning-instance!- :workspace_input_external {:workspace_id ws-id- :db_id db-id- :schema "public"- :table "orders"- :created_at :%now- :updated_at :%now}))- ;; -- workspace_input_external with no matching table — should stay nil --- wie-no-match-id (:id (t2/insert-returning-instance!- :workspace_input_external {:workspace_id ws-id- :db_id db-id- :schema "public"- :table "no_such_table"- :created_at :%now- :updated_at :%now}))]- (testing "Before backfill, all FKs are nil"- (is (nil? (t2/select-one-fn :global_table_id :workspace_output :id wo-id)))- (is (nil? (t2/select-one-fn :isolated_table_id :workspace_output :id wo-id)))- (is (nil? (t2/select-one-fn :global_table_id :workspace_output_external :id woe-id)))- (is (nil? (t2/select-one-fn :isolated_table_id :workspace_output_external :id woe-id)))- (is (nil? (t2/select-one-fn :table_id :workspace_input :id wi-id)))- (is (nil? (t2/select-one-fn :table_id :workspace_input_external :id wie-id)))- (is (nil? (t2/select-one-fn :table_id :workspace_input_external :id wie-no-match-id))))+ :updated_at :%now})]+ (testing "Before migration"+ (is (= 1 (:graph_version (t2/select-one :workspace :id ws-id))))+ (is (= 1 (:analysis_version (first (t2/select :workspace_transform :workspace_id ws-id))))))(migrate!)(testing "Provisional metabase_table created for transform target"(let [provisional (first (t2/query {:select [:active :transform_target :data_source :data_authority :display_name]@@ -3124,15 +3063,7 @@(is (= "metabase-transform" (:data_source provisional)))(is (= "computed" (:data_authority provisional)))(is (= "New Target Table" (:display_name provisional)))))- (testing "workspace_output FKs backfilled"- (is (= table-id (t2/select-one-fn :global_table_id :workspace_output :id wo-id)))- (is (= table-id (t2/select-one-fn :isolated_table_id :workspace_output :id wo-id))))- (testing "workspace_output_external FKs backfilled"- (is (= table-id (t2/select-one-fn :global_table_id :workspace_output_external :id woe-id)))- (is (= table-id (t2/select-one-fn :isolated_table_id :workspace_output_external :id woe-id))))- (testing "workspace_input table_id backfilled"- (is (= table-id (t2/select-one-fn :table_id :workspace_input :id wi-id))))- (testing "workspace_input_external table_id backfilled"- (is (= table-id (t2/select-one-fn :table_id :workspace_input_external :id wie-id))))- (testing "Non-matching input stays nil"- (is (nil? (t2/select-one-fn :table_id :workspace_input_external :id wie-no-match-id))))))))+ (testing "Workspace caches invalidated"+ ;; Both BackfillTransformTargetTables and BackfillTransformTargetTableId bump these+ (is (= 3 (:graph_version (t2/select-one :workspace :id ws-id))))+ (is (= 3 (:analysis_version (first (t2/select :workspace_transform :workspace_id ws-id))))))))))