The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Continue sync even if a table fails (#66432)
Eric Normande883cd1a6624
diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_introspection_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_introspection_test.cljindex 32fde0e794d..47909658c67 100644--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_introspection_test.clj+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_introspection_test.clj@@ -5,7 +5,11 @@[metabase.driver :as driver][metabase.driver.common :as driver.common][metabase.query-processor :as qp]+ [metabase.sync.core :as sync][metabase.test :as mt]+ [metabase.test.data.clickhouse]+ [metabase.util :as u]+ [toucan2.core :as t2][toucan2.tools.with-temp :as t2.with-temp]))(defn- desc-table!@@ -542,3 +546,43 @@;; tables from `information_schema`(is (contains? tables tables-table))(is (contains? tables columns-table)))))))++(deftest ^:synchronized clickhouse-parameterized-view-sync-resilience-test+ (testing "Sync should continue past parameterized views that cannot be introspected (#66395)"+ (mt/test-driver :clickhouse+ (mt/dataset metabase_db_scan_test+ (let [db (mt/db)+ details (mt/dbdef->connection-details :clickhouse :db {:database-name "metabase_db_scan_test"})+ db-name "metabase_db_scan_test"]+ ;; Create a parameterized view that will fail during sync+ (metabase.test.data.clickhouse/exec-statements+ [(format "CREATE OR REPLACE VIEW %s.parameterized_view AS SELECT {returnString:String} as result" db-name)+ (format "CREATE TABLE %s.table_after_view (id Int32, name String) ENGINE = Memory" db-name)+ (format "INSERT INTO %s.table_after_view VALUES (1, 'test')" db-name)]+ details)+ (try+ ;; Sync the database - this should not crash despite the parameterized view+ (is (not= ::thrown+ (try+ (sync/sync-database! db)+ (catch Throwable _e+ ::thrown)))+ "Sync should not throw an exception when encountering a parameterized view")++ ;; Verify that the table AFTER the problematic view was still synced+ (let [table-after (t2/select-one :model/Table :db_id (u/the-id db) :name "table_after_view")]+ (is (some? table-after)+ "Table created after parameterized view should be synced")+ (when table-after+ (let [fields (t2/select :model/Field :table_id (:id table-after))]+ (is (= #{"id" "name"}+ (set (map :name fields)))+ "Fields in table after parameterized view should be synced"))))+ (finally+ ;; Clean up+ (metabase.test.data.clickhouse/exec-statements+ [(format "DROP VIEW IF EXISTS %s.parameterized_view" db-name)+ (format "DROP TABLE IF EXISTS %s.table_after_view" db-name)]+ details))))))))++diff --git a/src/metabase/sync/fetch_metadata.clj b/src/metabase/sync/fetch_metadata.cljindex b85394b5b36..09f7940087a 100644--- a/src/metabase/sync/fetch_metadata.clj+++ b/src/metabase/sync/fetch_metadata.clj@@ -59,16 +59,19 @@instead. Also includes nested field column metadata."[_driver database & {:keys [schema-names table-names]}];; Realize everything in a vector to close the connection. But only keep the ids because the maps can hog memory.- (let [table-ids (into [] (map :id)- (sync-util/reducible-sync-tables database :schema-names schema-names :table-names table-names))]+ (let [table-ids (mapv :id (sync-util/reducible-sync-tables database :schema-names schema-names :table-names table-names))](eduction(mapcat (fn [table-id]- (let [table (t2/select-one :model/Table table-id)- table-fields (table-fields-metadata database table)]- ;; Realize the fields from this table (from `table-fields-metadata`) immediately to ensure the- ;; connection is closed before moving to the next table.- (mapv #(assoc % :table-schema (:schema table) :table-name (:name table))- table-fields))))+ (try+ (let [table (t2/select-one :model/Table table-id)+ table-fields (table-fields-metadata database table)]+ ;; Realize the fields from this table (from `table-fields-metadata`) immediately to ensure the+ ;; connection is closed before moving to the next table.+ (mapv #(assoc % :table-schema (:schema table) :table-name (:name table))+ table-fields))+ (catch Throwable e+ (log/warn e (str "Could not fetch fields from table " table-id))+ nil))))table-ids)))(mu/defn fields-metadata