Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Continue sync even if a table fails (#66432)

Eric Normande883cd1a6624
diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_introspection_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_introspection_test.clj
index 32fde0e794d..47909658c67 100644
--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_introspection_test.clj
+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_introspection_test.clj
@@ -5,7 +5,11 @@
[metabase.driver :as driver]
[metabase.driver.common :as driver.common]
[metabase.query-processor :as qp]
+ [metabase.sync.core :as sync]
[metabase.test :as mt]
+ [metabase.test.data.clickhouse]
+ [metabase.util :as u]
+ [toucan2.core :as t2]
[toucan2.tools.with-temp :as t2.with-temp]))
(defn- desc-table!
@@ -542,3 +546,43 @@
;; tables from `information_schema`
(is (contains? tables tables-table))
(is (contains? tables columns-table)))))))
+
+(deftest ^:synchronized clickhouse-parameterized-view-sync-resilience-test
+ (testing "Sync should continue past parameterized views that cannot be introspected (#66395)"
+ (mt/test-driver :clickhouse
+ (mt/dataset metabase_db_scan_test
+ (let [db (mt/db)
+ details (mt/dbdef->connection-details :clickhouse :db {:database-name "metabase_db_scan_test"})
+ db-name "metabase_db_scan_test"]
+ ;; Create a parameterized view that will fail during sync
+ (metabase.test.data.clickhouse/exec-statements
+ [(format "CREATE OR REPLACE VIEW %s.parameterized_view AS SELECT {returnString:String} as result" db-name)
+ (format "CREATE TABLE %s.table_after_view (id Int32, name String) ENGINE = Memory" db-name)
+ (format "INSERT INTO %s.table_after_view VALUES (1, 'test')" db-name)]
+ details)
+ (try
+ ;; Sync the database - this should not crash despite the parameterized view
+ (is (not= ::thrown
+ (try
+ (sync/sync-database! db)
+ (catch Throwable _e
+ ::thrown)))
+ "Sync should not throw an exception when encountering a parameterized view")
+
+ ;; Verify that the table AFTER the problematic view was still synced
+ (let [table-after (t2/select-one :model/Table :db_id (u/the-id db) :name "table_after_view")]
+ (is (some? table-after)
+ "Table created after parameterized view should be synced")
+ (when table-after
+ (let [fields (t2/select :model/Field :table_id (:id table-after))]
+ (is (= #{"id" "name"}
+ (set (map :name fields)))
+ "Fields in table after parameterized view should be synced"))))
+ (finally
+ ;; Clean up
+ (metabase.test.data.clickhouse/exec-statements
+ [(format "DROP VIEW IF EXISTS %s.parameterized_view" db-name)
+ (format "DROP TABLE IF EXISTS %s.table_after_view" db-name)]
+ details))))))))
+
+
diff --git a/src/metabase/sync/fetch_metadata.clj b/src/metabase/sync/fetch_metadata.clj
index b85394b5b36..09f7940087a 100644
--- a/src/metabase/sync/fetch_metadata.clj
+++ b/src/metabase/sync/fetch_metadata.clj
@@ -59,16 +59,19 @@
instead. Also includes nested field column metadata."
[_driver database & {:keys [schema-names table-names]}]
;; Realize everything in a vector to close the connection. But only keep the ids because the maps can hog memory.
- (let [table-ids (into [] (map :id)
- (sync-util/reducible-sync-tables database :schema-names schema-names :table-names table-names))]
+ (let [table-ids (mapv :id (sync-util/reducible-sync-tables database :schema-names schema-names :table-names table-names))]
(eduction
(mapcat (fn [table-id]
- (let [table (t2/select-one :model/Table table-id)
- table-fields (table-fields-metadata database table)]
- ;; Realize the fields from this table (from `table-fields-metadata`) immediately to ensure the
- ;; connection is closed before moving to the next table.
- (mapv #(assoc % :table-schema (:schema table) :table-name (:name table))
- table-fields))))
+ (try
+ (let [table (t2/select-one :model/Table table-id)
+ table-fields (table-fields-metadata database table)]
+ ;; Realize the fields from this table (from `table-fields-metadata`) immediately to ensure the
+ ;; connection is closed before moving to the next table.
+ (mapv #(assoc % :table-schema (:schema table) :table-name (:name table))
+ table-fields))
+ (catch Throwable e
+ (log/warn e (str "Could not fetch fields from table " table-id))
+ nil))))
table-ids)))
(mu/defn fields-metadata