Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Batch metabase_table update migration (#70996)

Nicola Mometto4937134c3ef5
diff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yaml
index 526cb863c9f..867b8cc1729 100644
--- a/resources/migrations/059_update_migrations.yaml
+++ b/resources/migrations/059_update_migrations.yaml
@@ -1225,34 +1225,6 @@ databaseChangeLog:
tableName: python_library
constraintName: unique_python_library_entity_id
- - changeSet:
- id: v59.2026-01-27T13:00:00
- author: wotbrew
- comment: Drop medallion names for data_layer enum use (hidden, internal instead)
- preConditions:
- - onFail: MARK_RAN
- - not:
- changeSetExecuted:
- id: v58.2026-01-27T13:00:00
- author: wotbrew
- changeLogFile: migrations/058_update_migrations.yaml
- changes:
- - sql:
- sql: >-
- UPDATE metabase_table
- SET data_layer = CASE data_layer
- WHEN 'copper' THEN 'hidden'
- ELSE 'final'
- END
- rollback:
- - sql:
- sql: >-
- UPDATE metabase_table
- SET data_layer = CASE data_layer
- WHEN 'hidden' THEN 'copper'
- ELSE 'bronze'
- END
-
- changeSet:
id: v59.2026-01-28T10:00:00
author: metamben
@@ -3476,6 +3448,28 @@ databaseChangeLog:
constraints:
nullable: true
+ - changeSet:
+ id: v59.2026-03-17T13:00:00
+ author: bronsa
+ comment: Drop medallion names for data_layer enum use (hidden, internal instead)
+ preConditions:
+ - onFail: MARK_RAN
+ - and:
+ - not:
+ changeSetExecuted:
+ id: v58.2026-01-27T13:00:00
+ author: wotbrew
+ changeLogFile: migrations/058_update_migrations.yaml
+ - not:
+ changeSetExecuted:
+ id: v59.2026-01-27T13:00:00
+ author: wotbrew
+ changeLogFile: migrations/059_update_migrations.yaml
+ runInTransaction: false
+ changes:
+ - customChange:
+ class: "metabase.app_db.custom_migrations.DropMedallionNamesForDataLayer"
+
# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
########################################################################################################################
diff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.clj
index 7987c43ba24..1f3a2620086 100644
--- a/src/metabase/app_db/custom_migrations.clj
+++ b/src/metabase/app_db/custom_migrations.clj
@@ -2052,6 +2052,60 @@
:set {:source (json-in (dissoc parsed :source-incremental-strategy))}
:where [:= :id id]}))))))
+(defn- batched-data-layer-update!
+ "Update `metabase_table.data_layer` in 500K ID-range chunks to avoid timeout on large instances.
+ Computes min/max ID once and strides through the range. A final unbounded UPDATE catches any
+ rows inserted during the migration."
+ [from-values case-expr]
+ (let [batch-size 500000
+ {:keys [min-id max-id]} (t2/query-one
+ {:select [[[:min :id] :min-id]
+ [[:max :id] :max-id]]
+ :from [:metabase_table]})]
+ (when (and min-id max-id)
+ (loop [start (long min-id)]
+ (when (<= start (long max-id))
+ (t2.execute/query-one
+ {:update :metabase_table
+ :set {:data_layer case-expr}
+ :where [:and
+ [:>= :id start]
+ [:< :id (+ start batch-size)]
+ [:not-in :data_layer from-values]]})
+ (recur (+ start batch-size))))
+ ;; catch any rows inserted above the original max-id during the migration
+ (t2.execute/query-one
+ {:update :metabase_table
+ :set {:data_layer case-expr}
+ :where [:and
+ [:> :id max-id]
+ [:not-in :data_layer from-values]]}))))
+
+;; Intentionally not using define-reversible-migration here to avoid wrapping in a transaction.
+;; A single long transaction on millions of rows can hit connection/transaction timeouts.
+;; Each batch runs as its own implicit transaction instead.
+;; Partial completion is safe: :model/Table's transform-data-layer handles on-read conversion
+;; of any unconverted rows in both directions (58<->59).
+(defrecord DropMedallionNamesForDataLayer []
+ CustomTaskChange
+ (execute [_ _database]
+ (batched-data-layer-update!
+ ["hidden" "internal" "final"]
+ [:case [:= :data_layer "copper"] "hidden" :else "final"]))
+ (getConfirmationMessage [_]
+ "Custom migration: DropMedallionNamesForDataLayer")
+ (setUp [_])
+ (validate [_ _database]
+ (ValidationErrors.))
+ (setFileOpener [_ _resourceAccessor])
+
+ CustomTaskRollback
+ (rollback [_ _database]
+ (when (should-execute-change?)
+ (batched-data-layer-update!
+ ["copper" "bronze" "silver" "gold"]
+ [:case [:= :data_layer "hidden"] "copper" :else "bronze"]))))
+
(define-migration BackfillTransformTargetTables
;; For each transform that has a target (database, schema, name), ensure a metabase_table row exists.
;; If the table already exists (active or inactive), do nothing. Otherwise, insert a transform target row.