The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Surface essential-step sync failures instead of reporting initial sync complete (GHY-3943) (#76286)
Eric Normanda60bd4f293af
diff --git a/src/metabase/sync/sync_metadata.clj b/src/metabase/sync/sync_metadata.cljindex 2128a5d42b7..88d8dff3afc 100644--- a/src/metabase/sync/sync_metadata.clj+++ b/src/metabase/sync/sync_metadata.clj@@ -18,7 +18,6 @@[metabase.sync.sync-metadata.tables :as sync-tables][metabase.sync.util :as sync-util][metabase.tracing.core :as tracing]- [metabase.util :as u][metabase.util.malli :as mu][metabase.warehouse-schema.models.table :as table]))@@ -50,9 +49,9 @@[(sync-util/create-sync-step "sync-dbms-version" sync-dbms-ver/sync-dbms-version! sync-dbms-version-summary)(sync-util/create-sync-step "sync-timezone" sync-tz/sync-timezone! sync-timezone-summary);; Make sure the relevant table models are up-to-date- (sync-util/create-sync-step "sync-tables" #(sync-tables/sync-tables-and-database! % db-metadata) sync-tables-summary)+ (sync-util/create-sync-step "sync-tables" #(sync-tables/sync-tables-and-database! % db-metadata) sync-tables-summary true);; Now for each table, sync the fields- (sync-util/create-sync-step "sync-fields" sync-fields/sync-fields! sync-fields-summary)+ (sync-util/create-sync-step "sync-fields" sync-fields/sync-fields! sync-fields-summary true);; Now for each table, sync the FKS. This has to be done after syncing all the fields to make sure target fields exist(sync-util/create-sync-step "sync-fks" sync-fks/sync-fks! sync-fks-summary);; Sync index info if the database supports it@@ -64,16 +63,28 @@"Shared core of [[sync-db-metadata!]] and [[sync-db-metadata-explicit!]]: the metadata sync work,without the surrounding `*-sync-operation` wrapper (which is what applies the eligibility gating)."[database :- i/DatabaseInstance]- (let [db-metadata (try- (tracing/with-span :sync "sync.metadata.fetch-metadata" {:db/id (:id database)}- (fetch-metadata/db-metadata database))- (catch Throwable e- (sync-util/set-initial-database-sync-aborted! database)- (throw e)))]- (u/prog1 (sync-util/run-sync-operation "sync" database (make-sync-steps db-metadata))- (if (some sync-util/abandon-sync? (map second (:steps <>)))- (sync-util/set-initial-database-sync-aborted! database)- (sync-util/set-initial-database-sync-complete! database)))))+ (let [db-metadata (try+ (tracing/with-span :sync "sync.metadata.fetch-metadata" {:db/id (:id database)}+ (fetch-metadata/db-metadata database))+ (catch Throwable e+ (sync-util/set-initial-database-sync-aborted! database)+ (throw e)))+ steps (make-sync-steps db-metadata)+ essential-steps (into #{} (comp (filter :essential?) (map :step-name)) steps)+ results (sync-util/run-sync-operation "sync" database steps)]+ (cond+ (some sync-util/abandon-sync? (map second (:steps results)))+ (sync-util/set-initial-database-sync-aborted! database)++ (some (fn [[step-name step-results]]+ (and (contains? essential-steps step-name)+ (sync-util/step-failed? step-results)))+ (:steps results))+ (sync-util/set-initial-database-sync-aborted! database)++ :else+ (sync-util/set-initial-database-sync-complete! database))+ results))(mu/defn sync-db-metadata!"Sync the metadata for a Metabase `database`. This makes sure child Table & Field objects are synchronized.diff --git a/src/metabase/sync/util.clj b/src/metabase/sync/util.cljindex c4c27fc9ccd..bab1342056c 100644--- a/src/metabase/sync/util.clj+++ b/src/metabase/sync/util.clj@@ -600,21 +600,26 @@(def ^:private StepDefinition"Defines a step. `:sync-fn` runs the step, returns a map that contains step specific metadata. `log-summary-fn`- takes that metadata and turns it into a string for logging"+ takes that metadata and turns it into a string for logging. `:essential?` marks a step whose failure leaves the+ database unusable (e.g. field sync), so initial sync should be reported as failed rather than complete."[:map[:sync-fn [:=> [:cat StepRunMetadata] i/DatabaseInstance]][:step-name :string]- [:log-summary-fn [:maybe LogSummaryFunction]]])+ [:log-summary-fn [:maybe LogSummaryFunction]]+ [:essential? :boolean]])(defn create-sync-step"Creates and returns a step suitable for `run-step-with-metadata`. See `StepDefinition` for more info."([step-name sync-fn]- (create-sync-step step-name sync-fn nil))+ (create-sync-step step-name sync-fn nil false))([step-name sync-fn log-summary-fn]+ (create-sync-step step-name sync-fn log-summary-fn false))+ ([step-name sync-fn log-summary-fn essential?]{:sync-fn sync-fn:step-name step-name:log-summary-fn (when log-summary-fn- (comp str log-summary-fn))}))+ (comp str log-summary-fn))+ :essential? (boolean essential?)}))(mu/defn- run-step-with-metadata :- StepNameWithMetadata"Runs `step` on `database` returning metadata from the run"@@ -688,6 +693,13 @@(when-let [caught-exception (:throwable step-results)](transient-exception? caught-exception)))+(defn step-failed?+ "Given the results of a sync step, returns truthy if the step recorded an exception (transient or not). A failure of+ an essential step (e.g. field sync) leaves the database unusable, so initial sync should be reported as failed rather+ than complete."+ [step-results]+ (some? (:throwable step-results)))+(mu/defn run-sync-operation"Run `sync-steps` and log a summary message"[operation :- :stringdiff --git a/test/metabase/sync/sync_metadata_test.clj b/test/metabase/sync/sync_metadata_test.cljindex c17ce4000ac..14a46de6440 100644--- a/test/metabase/sync/sync_metadata_test.clj+++ b/test/metabase/sync/sync_metadata_test.clj@@ -12,6 +12,8 @@[metabase.test.sync :refer [sync-survives-crash?!]][toucan2.core :as t2]))+(set! *warn-on-reflection* true)+(deftest survive-metadata-errors(testing "Make sure we survive metadata sync failing"(sync-survives-crash?! metabase-metadata/sync-metabase-metadata!)))@@ -43,3 +45,18 @@(#'sync-metadata/sync-db-metadata!* db))))(is (= "aborted"(t2/select-one-fn :initial_sync_status :model/Database (:id db))))))))++(deftest field-sync-failure-aborts-initial-sync-test+ (testing "When an essential sync step fails non-transiently mid-initial-sync (e.g. MariaDB <10.2 whose"+ (testing "information_schema lacks generation_expression, so describe-fields throws, GHY-3943),"+ (testing "the initial sync status is marked aborted instead of falsely reporting \"complete\""+ (mt/with-temp-copy-of-db+ (t2/update! :model/Database (mt/id) {:initial_sync_status "incomplete"})+ (with-redefs [fetch-metadata/fields-metadata+ (fn [& _]+ (throw (doto (java.sql.SQLSyntaxErrorException.+ "Unknown column 'generation_expression' in 'field list'")+ (.setStackTrace (into-array StackTraceElement [])))))]+ (#'sync-metadata/sync-db-metadata!* (t2/select-one :model/Database :id (mt/id))))+ (is (= "aborted"+ (t2/select-one-fn :initial_sync_status :model/Database (mt/id)))))))))