Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Data migration to fix clickhouse upload dbs with missing schema (#70363)

Riley Thompson1571e5818846
diff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yaml
index 2fdc07d8deb..12cce2e37c5 100644
--- a/resources/migrations/059_update_migrations.yaml
+++ b/resources/migrations/059_update_migrations.yaml
@@ -3574,6 +3574,14 @@ databaseChangeLog:
path: instance_analytics_views/query_log/v2/h2-query_log.sql
relativeToChangelogFile: true
+ - changeSet:
+ id: v59.2026-03-04T00:00:00
+ author: rileythompson
+ comment: Fix ClickHouse upload DB schema name and models
+ changes:
+ - customChange:
+ class: "metabase.app_db.custom_migrations.FixClickHouseUploadDBSchemaNames"
+
# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
########################################################################################################################
diff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.clj
index a0ef4bfec3d..db8b2ca3911 100644
--- a/src/metabase/app_db/custom_migrations.clj
+++ b/src/metabase/app_db/custom_migrations.clj
@@ -1854,3 +1854,76 @@
(define-migration MoveExistingAtSymbolUserAttributes
(reserve-at-symbol-user-attributes/migrate!))
+
+(define-migration FixClickHouseUploadDBSchemaNames
+ "This data migration is meant to fix the issues seen in #69667, #68298 and #65945.
+ We made the driver feature `schemas` conditional on the `enable-multiple-db` DB connection setting.
+ So for DBs with `enable-multiple-db` set to false (such as our cloud hosted upload DBs), the `schemas` feature was
+ false. This meant that when a user disabled uploads, or changed their upload DBs, and then re-enabled uploads or
+ changed their upload DB back, the `uploads_schema_name` field of the DB was set to null. So any uploads made after
+ the `uploads_schema_name` field was set to null were created as tables with a null `schema`. For example:
+ ;; metabase_database
+ | id | engine | name | uploads_enabled | uploads_schema_name |
+ | 2 | clickhouse | upload_db | true | null | ;; `uploads_schema_name` set to null after disabling and re-enabling uploads
+ ;; metabase_table
+ | id | db_id | name | schema | active | is_upload |
+ | 1 | 2 | t1 | db_foo | true | true | ;; created before uploads_schema_name was set to null
+ | 2 | 2 | t2 | null | true | true | ;; created after uploads_schema_name was set to null
+ On the clickhouse DB these tables are still created in a schema, particularly the one specified in the `dbname`
+ field of the DB `details`. This meant that the next time the DB sync ran, it would find this table under that
+ schema, and would not find that table with a null schema. So then it would create a new table with the same name
+ and a schema, and mark the old table with the null schema as inactive. For example:
+ ;; metabase_table
+ | id | db_id | name | schema | active | is_upload |
+ | 1 | 2 | t1 | db_foo | true | true | ;; this table has a schema so it's synced correctly
+ | 2 | 2 | t2 | null | false | true | ;; this table is now inactive since it has a null schema and isn't found by sync
+ | 3 | 2 | t2 | db_foo | true | false | ;; this is the new table created by sync with the same name and a schema
+ We create models based on upload tables, and since the upload tables got marked as inactive, attempting to access
+ these models would give an inactive table error."
+ ;; Look for a clickhouse DB with uploads_enabled
+ (let [clickhouse-upload-db (t2/query {:select [:id :details :uploads_schema_name]
+ :from [:metabase_database]
+ :where [:and
+ [:= :engine "clickhouse"]
+ [:= :uploads_enabled true]]})
+ ;; If this DB has a null `uploads_schema_name`, then set the `uploads_schema_name` to the value of the `dbname`
+ set-uploads-schema-name! (fn [{:keys [id details uploads_schema_name]}]
+ (let [decrypted-details (encrypted-json-out details)
+ db-name (:dbname decrypted-details)]
+ (when (and db-name (not uploads_schema_name))
+ (t2/query {:update :metabase_database
+ :set {:uploads_schema_name db-name}
+ :where [:= :id id]}))))]
+ (if (< 1 (count clickhouse-upload-db))
+ (log/warn "FixClickHouseUploadDBSchemaNames: expected at most 1 ClickHouse upload database, found" (count clickhouse-upload-db))
+ (do
+ (run! set-uploads-schema-name! clickhouse-upload-db)
+ ;; Look for any inactive upload tables with a null schema
+ (let [inactive-upload-tables (t2/query {:select [:mt.id :mt.name :mt.db_id :md.uploads_schema_name]
+ :from [[:metabase_table :mt]]
+ :join [[:metabase_database :md] [:= :mt.db_id :md.id]]
+ :where [:and
+ [:= :md.engine "clickhouse"]
+ [:= :md.uploads_enabled true]
+ [:not= :md.uploads_schema_name nil]
+ [:= :mt.schema nil]
+ [:= :mt.active false]
+ [:= :mt.is_upload true]]})
+ retire-and-revive-upload-table! (fn [{:keys [id name db_id uploads_schema_name]}]
+ ;; Look for an active non-upload table with the same name and the correct `uploads_schema_name`
+ ;; Set it to be inactive and rename it to satisfy the (db_id, name, schema) unique key
+ (t2/query {:update :metabase_table
+ :set {:active false
+ :name (str name "_retired_69667")}
+ :where [:and
+ [:= :name name]
+ [:= :schema uploads_schema_name]
+ [:= :active true]
+ [:= :is_upload false]
+ [:= :db_id db_id]]})
+ ;; Set the inactive upload table to be active and set the schema to the correct `uploads_schema_name`
+ (t2/query {:update :metabase_table
+ :set {:active true
+ :schema uploads_schema_name}
+ :where [:= :id id]}))]
+ (run! retire-and-revive-upload-table! inactive-upload-tables))))))
diff --git a/test/metabase/app_db/custom_migrations_test.clj b/test/metabase/app_db/custom_migrations_test.clj
index ad11c2c060b..1d35b53ee2b 100644
--- a/test/metabase/app_db/custom_migrations_test.clj
+++ b/test/metabase/app_db/custom_migrations_test.clj
@@ -2775,3 +2775,75 @@
(is (= db-id (id->target target-id)))
(is (nil? (id->target none-id)))
(is (nil? (id->target deleted-id))))))))
+
+(deftest fix-clickhouse-upload-db-schema-names-test
+ (testing "FixClickHouseUploadDBSchemaNames, v59.2026-03-04T00:00:00: fix clickhouse upload db schema names"
+ (encryption-test/with-secret-key "fake-secret-key"
+ ;; Test when the upload db doesn't have an upload_schema_name set (both upload db and upload
+ ;; tables are in a bad state) and when it does have an upload_schema_name set (upload db and
+ ;; new upload tables are in a good state, but existing upload tables are in a bad state)
+ (doseq [uploads-schema-name [nil "db_foo"]]
+ (impl/test-migrations
+ ["v59.2026-03-04T00:00:00"] [migrate!]
+ (let [db-id (t2/insert-returning-pk! :metabase_database
+ {:name "clickhouse cloud upload db"
+ :engine "clickhouse"
+ :created_at :%now
+ :updated_at :%now
+ :uploads_enabled true
+ :uploads_schema_name uploads-schema-name
+ :uploads_table_prefix "uploads_"
+ :details (mi/encrypted-json-in {:dbname "db_foo"})})
+ insert-table! (fn [db-id name schema active is-upload display-name]
+ (t2/insert-returning-pk! :metabase_table
+ {:db_id db-id
+ :name name
+ :schema schema
+ :active active
+ :is_upload is-upload
+ :display_name display-name
+ :created_at :%now
+ :updated_at :%now}))
+ ;; An uploads table in a good state, created before uploads_schema_name was set to null
+ uploaded-0 (insert-table! db-id "uploads_test_table_0" "db_foo" true true "Test Table 0")
+ ;; Two upload tables in a bad state, created after uploads_schema_name was set to null
+ uploaded-1 (insert-table! db-id "uploads_test_table_1" nil false true "Test Table 1")
+ uploaded-2 (insert-table! db-id "uploads_test_table_2" nil false true "Test Table 2")
+ ;; The two non-upload versions of the above tables, created by the sync process
+ synced-1 (insert-table! db-id "uploads_test_table_1" "db_foo" true false "Uploads Test Table 1")
+ synced-2 (insert-table! db-id "uploads_test_table_2" "db_foo" true false "Uploads Test Table 2")
+ ;; An unrelated non-upload table in the same schema that should be left alone
+ unrelated (insert-table! db-id "unrelated_table" "db_foo" true false "Unrelated Table")]
+ (migrate!)
+ ;; The uploads db has the correct uploads_schema_name from the details
+ (is (= "db_foo" (:uploads_schema_name (t2/select-one :metabase_database :id db-id))))
+ (are [exp table-id] (= exp
+ (t2/select-one [:metabase_table :name :schema :active :is_upload] :id table-id))
+ ;; The upload table that was already in a good state remains unchanged
+ {:name "uploads_test_table_0"
+ :schema "db_foo"
+ :active true
+ :is_upload true} uploaded-0
+ ;; The two upload tables in a bad state are updated to be active and have the correct schema
+ {:name "uploads_test_table_1"
+ :schema "db_foo"
+ :active true
+ :is_upload true} uploaded-1
+ {:name "uploads_test_table_2"
+ :schema "db_foo"
+ :active true
+ :is_upload true} uploaded-2
+ ;; The two non-upload tables created by the sync have been renamed and set as inactive
+ {:name "uploads_test_table_1_retired_69667"
+ :schema "db_foo"
+ :active false
+ :is_upload false} synced-1
+ {:name "uploads_test_table_2_retired_69667"
+ :schema "db_foo"
+ :active false
+ :is_upload false} synced-2
+ ;; The unrelated table remains unchanged
+ {:name "unrelated_table"
+ :schema "db_foo"
+ :active true
+ :is_upload false} unrelated)))))))