The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Data migration to fix clickhouse upload dbs with missing schema (#70363)
Riley Thompson1571e5818846
diff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yamlindex 2fdc07d8deb..12cce2e37c5 100644--- a/resources/migrations/059_update_migrations.yaml+++ b/resources/migrations/059_update_migrations.yaml@@ -3574,6 +3574,14 @@ databaseChangeLog:path: instance_analytics_views/query_log/v2/h2-query_log.sqlrelativeToChangelogFile: true+ - changeSet:+ id: v59.2026-03-04T00:00:00+ author: rileythompson+ comment: Fix ClickHouse upload DB schema name and models+ changes:+ - customChange:+ class: "metabase.app_db.custom_migrations.FixClickHouseUploadDBSchemaNames"+# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<########################################################################################################################diff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.cljindex a0ef4bfec3d..db8b2ca3911 100644--- a/src/metabase/app_db/custom_migrations.clj+++ b/src/metabase/app_db/custom_migrations.clj@@ -1854,3 +1854,76 @@(define-migration MoveExistingAtSymbolUserAttributes(reserve-at-symbol-user-attributes/migrate!))++(define-migration FixClickHouseUploadDBSchemaNames+ "This data migration is meant to fix the issues seen in #69667, #68298 and #65945.+ We made the driver feature `schemas` conditional on the `enable-multiple-db` DB connection setting.+ So for DBs with `enable-multiple-db` set to false (such as our cloud hosted upload DBs), the `schemas` feature was+ false. This meant that when a user disabled uploads, or changed their upload DBs, and then re-enabled uploads or+ changed their upload DB back, the `uploads_schema_name` field of the DB was set to null. So any uploads made after+ the `uploads_schema_name` field was set to null were created as tables with a null `schema`. For example:+ ;; metabase_database+ | id | engine | name | uploads_enabled | uploads_schema_name |+ | 2 | clickhouse | upload_db | true | null | ;; `uploads_schema_name` set to null after disabling and re-enabling uploads+ ;; metabase_table+ | id | db_id | name | schema | active | is_upload |+ | 1 | 2 | t1 | db_foo | true | true | ;; created before uploads_schema_name was set to null+ | 2 | 2 | t2 | null | true | true | ;; created after uploads_schema_name was set to null+ On the clickhouse DB these tables are still created in a schema, particularly the one specified in the `dbname`+ field of the DB `details`. This meant that the next time the DB sync ran, it would find this table under that+ schema, and would not find that table with a null schema. So then it would create a new table with the same name+ and a schema, and mark the old table with the null schema as inactive. For example:+ ;; metabase_table+ | id | db_id | name | schema | active | is_upload |+ | 1 | 2 | t1 | db_foo | true | true | ;; this table has a schema so it's synced correctly+ | 2 | 2 | t2 | null | false | true | ;; this table is now inactive since it has a null schema and isn't found by sync+ | 3 | 2 | t2 | db_foo | true | false | ;; this is the new table created by sync with the same name and a schema+ We create models based on upload tables, and since the upload tables got marked as inactive, attempting to access+ these models would give an inactive table error."+ ;; Look for a clickhouse DB with uploads_enabled+ (let [clickhouse-upload-db (t2/query {:select [:id :details :uploads_schema_name]+ :from [:metabase_database]+ :where [:and+ [:= :engine "clickhouse"]+ [:= :uploads_enabled true]]})+ ;; If this DB has a null `uploads_schema_name`, then set the `uploads_schema_name` to the value of the `dbname`+ set-uploads-schema-name! (fn [{:keys [id details uploads_schema_name]}]+ (let [decrypted-details (encrypted-json-out details)+ db-name (:dbname decrypted-details)]+ (when (and db-name (not uploads_schema_name))+ (t2/query {:update :metabase_database+ :set {:uploads_schema_name db-name}+ :where [:= :id id]}))))]+ (if (< 1 (count clickhouse-upload-db))+ (log/warn "FixClickHouseUploadDBSchemaNames: expected at most 1 ClickHouse upload database, found" (count clickhouse-upload-db))+ (do+ (run! set-uploads-schema-name! clickhouse-upload-db)+ ;; Look for any inactive upload tables with a null schema+ (let [inactive-upload-tables (t2/query {:select [:mt.id :mt.name :mt.db_id :md.uploads_schema_name]+ :from [[:metabase_table :mt]]+ :join [[:metabase_database :md] [:= :mt.db_id :md.id]]+ :where [:and+ [:= :md.engine "clickhouse"]+ [:= :md.uploads_enabled true]+ [:not= :md.uploads_schema_name nil]+ [:= :mt.schema nil]+ [:= :mt.active false]+ [:= :mt.is_upload true]]})+ retire-and-revive-upload-table! (fn [{:keys [id name db_id uploads_schema_name]}]+ ;; Look for an active non-upload table with the same name and the correct `uploads_schema_name`+ ;; Set it to be inactive and rename it to satisfy the (db_id, name, schema) unique key+ (t2/query {:update :metabase_table+ :set {:active false+ :name (str name "_retired_69667")}+ :where [:and+ [:= :name name]+ [:= :schema uploads_schema_name]+ [:= :active true]+ [:= :is_upload false]+ [:= :db_id db_id]]})+ ;; Set the inactive upload table to be active and set the schema to the correct `uploads_schema_name`+ (t2/query {:update :metabase_table+ :set {:active true+ :schema uploads_schema_name}+ :where [:= :id id]}))]+ (run! retire-and-revive-upload-table! inactive-upload-tables))))))diff --git a/test/metabase/app_db/custom_migrations_test.clj b/test/metabase/app_db/custom_migrations_test.cljindex ad11c2c060b..1d35b53ee2b 100644--- a/test/metabase/app_db/custom_migrations_test.clj+++ b/test/metabase/app_db/custom_migrations_test.clj@@ -2775,3 +2775,75 @@(is (= db-id (id->target target-id)))(is (nil? (id->target none-id)))(is (nil? (id->target deleted-id))))))))++(deftest fix-clickhouse-upload-db-schema-names-test+ (testing "FixClickHouseUploadDBSchemaNames, v59.2026-03-04T00:00:00: fix clickhouse upload db schema names"+ (encryption-test/with-secret-key "fake-secret-key"+ ;; Test when the upload db doesn't have an upload_schema_name set (both upload db and upload+ ;; tables are in a bad state) and when it does have an upload_schema_name set (upload db and+ ;; new upload tables are in a good state, but existing upload tables are in a bad state)+ (doseq [uploads-schema-name [nil "db_foo"]]+ (impl/test-migrations+ ["v59.2026-03-04T00:00:00"] [migrate!]+ (let [db-id (t2/insert-returning-pk! :metabase_database+ {:name "clickhouse cloud upload db"+ :engine "clickhouse"+ :created_at :%now+ :updated_at :%now+ :uploads_enabled true+ :uploads_schema_name uploads-schema-name+ :uploads_table_prefix "uploads_"+ :details (mi/encrypted-json-in {:dbname "db_foo"})})+ insert-table! (fn [db-id name schema active is-upload display-name]+ (t2/insert-returning-pk! :metabase_table+ {:db_id db-id+ :name name+ :schema schema+ :active active+ :is_upload is-upload+ :display_name display-name+ :created_at :%now+ :updated_at :%now}))+ ;; An uploads table in a good state, created before uploads_schema_name was set to null+ uploaded-0 (insert-table! db-id "uploads_test_table_0" "db_foo" true true "Test Table 0")+ ;; Two upload tables in a bad state, created after uploads_schema_name was set to null+ uploaded-1 (insert-table! db-id "uploads_test_table_1" nil false true "Test Table 1")+ uploaded-2 (insert-table! db-id "uploads_test_table_2" nil false true "Test Table 2")+ ;; The two non-upload versions of the above tables, created by the sync process+ synced-1 (insert-table! db-id "uploads_test_table_1" "db_foo" true false "Uploads Test Table 1")+ synced-2 (insert-table! db-id "uploads_test_table_2" "db_foo" true false "Uploads Test Table 2")+ ;; An unrelated non-upload table in the same schema that should be left alone+ unrelated (insert-table! db-id "unrelated_table" "db_foo" true false "Unrelated Table")]+ (migrate!)+ ;; The uploads db has the correct uploads_schema_name from the details+ (is (= "db_foo" (:uploads_schema_name (t2/select-one :metabase_database :id db-id))))+ (are [exp table-id] (= exp+ (t2/select-one [:metabase_table :name :schema :active :is_upload] :id table-id))+ ;; The upload table that was already in a good state remains unchanged+ {:name "uploads_test_table_0"+ :schema "db_foo"+ :active true+ :is_upload true} uploaded-0+ ;; The two upload tables in a bad state are updated to be active and have the correct schema+ {:name "uploads_test_table_1"+ :schema "db_foo"+ :active true+ :is_upload true} uploaded-1+ {:name "uploads_test_table_2"+ :schema "db_foo"+ :active true+ :is_upload true} uploaded-2+ ;; The two non-upload tables created by the sync have been renamed and set as inactive+ {:name "uploads_test_table_1_retired_69667"+ :schema "db_foo"+ :active false+ :is_upload false} synced-1+ {:name "uploads_test_table_2_retired_69667"+ :schema "db_foo"+ :active false+ :is_upload false} synced-2+ ;; The unrelated table remains unchanged+ {:name "unrelated_table"+ :schema "db_foo"+ :active true+ :is_upload false} unrelated)))))))