The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Batch metabase_table update migration (#70996)
Nicola Mometto4937134c3ef5
diff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yamlindex 526cb863c9f..867b8cc1729 100644--- a/resources/migrations/059_update_migrations.yaml+++ b/resources/migrations/059_update_migrations.yaml@@ -1225,34 +1225,6 @@ databaseChangeLog:tableName: python_libraryconstraintName: unique_python_library_entity_id- - changeSet:- id: v59.2026-01-27T13:00:00- author: wotbrew- comment: Drop medallion names for data_layer enum use (hidden, internal instead)- preConditions:- - onFail: MARK_RAN- - not:- changeSetExecuted:- id: v58.2026-01-27T13:00:00- author: wotbrew- changeLogFile: migrations/058_update_migrations.yaml- changes:- - sql:- sql: >-- UPDATE metabase_table- SET data_layer = CASE data_layer- WHEN 'copper' THEN 'hidden'- ELSE 'final'- END- rollback:- - sql:- sql: >-- UPDATE metabase_table- SET data_layer = CASE data_layer- WHEN 'hidden' THEN 'copper'- ELSE 'bronze'- END-- changeSet:id: v59.2026-01-28T10:00:00author: metamben@@ -3476,6 +3448,28 @@ databaseChangeLog:constraints:nullable: true+ - changeSet:+ id: v59.2026-03-17T13:00:00+ author: bronsa+ comment: Drop medallion names for data_layer enum use (hidden, internal instead)+ preConditions:+ - onFail: MARK_RAN+ - and:+ - not:+ changeSetExecuted:+ id: v58.2026-01-27T13:00:00+ author: wotbrew+ changeLogFile: migrations/058_update_migrations.yaml+ - not:+ changeSetExecuted:+ id: v59.2026-01-27T13:00:00+ author: wotbrew+ changeLogFile: migrations/059_update_migrations.yaml+ runInTransaction: false+ changes:+ - customChange:+ class: "metabase.app_db.custom_migrations.DropMedallionNamesForDataLayer"+# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<########################################################################################################################diff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.cljindex 7987c43ba24..1f3a2620086 100644--- a/src/metabase/app_db/custom_migrations.clj+++ b/src/metabase/app_db/custom_migrations.clj@@ -2052,6 +2052,60 @@:set {:source (json-in (dissoc parsed :source-incremental-strategy))}:where [:= :id id]}))))))+(defn- batched-data-layer-update!+ "Update `metabase_table.data_layer` in 500K ID-range chunks to avoid timeout on large instances.+ Computes min/max ID once and strides through the range. A final unbounded UPDATE catches any+ rows inserted during the migration."+ [from-values case-expr]+ (let [batch-size 500000+ {:keys [min-id max-id]} (t2/query-one+ {:select [[[:min :id] :min-id]+ [[:max :id] :max-id]]+ :from [:metabase_table]})]+ (when (and min-id max-id)+ (loop [start (long min-id)]+ (when (<= start (long max-id))+ (t2.execute/query-one+ {:update :metabase_table+ :set {:data_layer case-expr}+ :where [:and+ [:>= :id start]+ [:< :id (+ start batch-size)]+ [:not-in :data_layer from-values]]})+ (recur (+ start batch-size))))+ ;; catch any rows inserted above the original max-id during the migration+ (t2.execute/query-one+ {:update :metabase_table+ :set {:data_layer case-expr}+ :where [:and+ [:> :id max-id]+ [:not-in :data_layer from-values]]}))))++;; Intentionally not using define-reversible-migration here to avoid wrapping in a transaction.+;; A single long transaction on millions of rows can hit connection/transaction timeouts.+;; Each batch runs as its own implicit transaction instead.+;; Partial completion is safe: :model/Table's transform-data-layer handles on-read conversion+;; of any unconverted rows in both directions (58<->59).+(defrecord DropMedallionNamesForDataLayer []+ CustomTaskChange+ (execute [_ _database]+ (batched-data-layer-update!+ ["hidden" "internal" "final"]+ [:case [:= :data_layer "copper"] "hidden" :else "final"]))+ (getConfirmationMessage [_]+ "Custom migration: DropMedallionNamesForDataLayer")+ (setUp [_])+ (validate [_ _database]+ (ValidationErrors.))+ (setFileOpener [_ _resourceAccessor])++ CustomTaskRollback+ (rollback [_ _database]+ (when (should-execute-change?)+ (batched-data-layer-update!+ ["copper" "bronze" "silver" "gold"]+ [:case [:= :data_layer "hidden"] "copper" :else "bronze"]))))+(define-migration BackfillTransformTargetTables;; For each transform that has a target (database, schema, name), ensure a metabase_table row exists.;; If the table already exists (active or inactive), do nothing. Otherwise, insert a transform target row.