The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Stop retrying permanently-failed fingerprinting every sync (#75640)
Eric Normandb47b12cf6b26
diff --git a/src/metabase/sync/analyze/fingerprint.clj b/src/metabase/sync/analyze/fingerprint.cljindex 62b3fdc17d5..c3567e1852d 100644--- a/src/metabase/sync/analyze/fingerprint.clj+++ b/src/metabase/sync/analyze/fingerprint.clj@@ -38,6 +38,15 @@(log/debugf "Saving fingerprint for %s" (sync-util/name-for-logging field))(t2/update! :model/Field (u/the-id field) (merge (incomplete-analysis-kvs) {:fingerprint fingerprint})))+(mu/defn- mark-fingerprinting-failed!+ "Advance `fingerprint_version` to the latest version for `fields` without saving a fingerprint. Called when+ fingerprinting fails for a non-transient reason, so that the Fields are not re-selected by [[fields-to-fingerprint]]+ and retried on every subsequent sync. Transient failures (see [[metabase.sync.util/transient-exception?]]) are+ left untouched so they are retried."+ [fields :- [:maybe [:sequential i/FieldInstance]]]+ (when-let [ids (seq (map u/the-id fields))]+ (t2/update! :model/Field :id [:in ids] {:fingerprint_version i/*latest-fingerprint-version*})))+(mr/def ::FingerprintStats[:map[:no-data-fingerprints ms/IntGreaterThanOrEqualToZero]@@ -69,7 +78,9 @@(reduce (fn [count-info [field fingerprint]](cond(instance? Throwable fingerprint)- (update count-info :failed-fingerprints inc)+ (do+ (mark-fingerprinting-failed! [field])+ (update count-info :failed-fingerprints inc))(some-> fingerprint :global :distinct-count zero?)(update count-info :no-data-fingerprints inc)@@ -202,8 +213,13 @@(let [stats(sync-util/with-returning-throwable (format "Error fingerprinting %s" (sync-util/name-for-logging table))(fingerprint-fields! table fields))]- (if (:throwable stats)- (merge (empty-stats-map 0) stats)+ (if-let [throwable (:throwable stats)]+ (do+ ;; transient connection/environment errors abort this run and retry on the next sync; for any other+ ;; (permanent) error, give up so we don't re-attempt every sync.+ (when-not (sync-util/transient-exception? throwable)+ (mark-fingerprinting-failed! fields))+ (merge (empty-stats-map 0) stats))stats)))(empty-stats-map 0))))diff --git a/src/metabase/sync/util.clj b/src/metabase/sync/util.cljindex df2560f393e..c4c27fc9ccd 100644--- a/src/metabase/sync/util.clj+++ b/src/metabase/sync/util.clj@@ -198,22 +198,36 @@(driver/sync-in-context (driver.u/database->driver database) databasef)))-;; TODO: future, expand this to `driver` level, where the drivers themselves can add to the-;; list of exception classes (like, driver-specific exceptions)+(defonce ^:private transient-exception-hierarchy+ (make-hierarchy))++(defn register-transient-exception+ "Register exception `klass` as a transient connection/environment-level failure — one that is expected to clear on a+ later sync. [[transient-exception?]] then returns true for `klass` and its subclasses, so sync aborts and retries+ the whole run later rather than treating the failure as permanent. Drivers may register driver-specific connection+ exceptions this way."+ [klass]+ (alter-var-root #'transient-exception-hierarchy derive klass ::transient-exception))+(doseq [klass [java.net.ConnectExceptionjava.net.NoRouteToHostExceptionjava.net.UnknownHostExceptioncom.mchange.v2.resourcepool.CannotAcquireResourceExceptionjavax.net.ssl.SSLHandshakeException]]- (derive klass ::exception-class-not-to-retry))+ (register-transient-exception klass))(def ^:dynamic *log-exceptions-and-continue?*"Whether to log exceptions during a sync step and proceed with the rest of the sync process. This is the defaultbehavior. You can disable this for debugging or test purposes."true)-(defn- do-not-retry-exception? [e]- (or (isa? (class e) ::exception-class-not-to-retry)+(defn transient-exception?+ "True if `e` (or any of its causes) is a connection/environment-level failure — database unreachable, connection+ pool exhausted, SSL handshake failure, and similar — rather than a failure that says anything about the data being+ synced. Such failures are expected to clear on a later sync and should not be treated as a permanent failure of the+ entity being synced. Register additional classes with [[register-transient-exception]]."+ [e]+ (or (isa? transient-exception-hierarchy (class e) ::transient-exception)(some-> (ex-cause e) recur)))(defn do-with-error-handling@@ -225,7 +239,7 @@(try(f)(catch Throwable e- (if (and *log-exceptions-and-continue?* (not (do-not-retry-exception? e)))+ (if (and *log-exceptions-and-continue?* (not (transient-exception? e)))(do(log/warn e message)e)@@ -235,9 +249,8 @@"Execute `body` in a way that catches and logs any Exceptions thrown, and returns the exception itself if they do so.Pass a `message` to help provide information about what failed for the log message.- The exception classes deriving from `:metabase.sync.util/exception-class-not-to-retry` are a list of classes tested- against exceptions thrown. If there is a match found, the sync is aborted as that error is not considered- recoverable for this sync run."+ Exceptions classified as transient (see [[transient-exception?]]) are not swallowed: the sync is aborted, as the+ database could not be reached and the failure is expected to clear on a later sync."{:style/indent 1}[message & body]`(do-with-error-handling ~message (^:once fn* [] ~@body)))@@ -673,7 +686,7 @@"Given the results of a sync step, returns truthy if a non-recoverable exception occurred"[step-results](when-let [caught-exception (:throwable step-results)]- (do-not-retry-exception? caught-exception)))+ (transient-exception? caught-exception)))(mu/defn run-sync-operation"Run `sync-steps` and log a summary message"diff --git a/test/metabase/sync/analyze/fingerprint_test.clj b/test/metabase/sync/analyze/fingerprint_test.cljindex b99a30b29e9..bea3c88975c 100644--- a/test/metabase/sync/analyze/fingerprint_test.clj+++ b/test/metabase/sync/analyze/fingerprint_test.clj@@ -11,7 +11,10 @@[metabase.test.data :as data][metabase.util :as u][toucan2.core :as t2])- (:import [com.mchange.v2.resourcepool CannotAcquireResourceException]))+ (:import+ [com.mchange.v2.resourcepool CannotAcquireResourceException]+ [java.net ConnectException]+ [java.sql SQLException SQLTimeoutException]));;; +----------------------------------------------------------------------------------------------------------------+;;; | TESTS FOR WHICH FIELDS NEED FINGERPRINTING |@@ -340,6 +343,47 @@:order-by [[:count :desc]]:limit 1}))))))))))+(defn- fingerprint-version-after-failure!+ "Create a fresh text Field at fingerprint_version 0 (eligible for fingerprinting up to version 5), run+ `fingerprint-table!` with the sample/fingerprint step failing as configured, and return the Field's+ `fingerprint_version` afterwards. A version of 0 means the Field will be re-selected and retried on the next+ sync; a version of 5 (the bound latest) means fingerprinting gave up and will not retry."+ [redefs]+ (mt/with-temp [:model/Table table {:db_id (mt/id)}+ :model/Field field {:table_id (u/the-id table)+ :active true+ :base_type :type/Text+ :fingerprint nil+ :fingerprint_version 0}]+ (binding [i/*latest-fingerprint-version* 5+ i/*fingerprint-version->types-that-should-be-re-fingerprinted* {5 #{:type/Text}}]+ (with-redefs-fn redefs+ (fn [] (sync.fingerprint/fingerprint-table! table))))+ (t2/select-one-fn :fingerprint_version :model/Field :id (u/the-id field))))++(defn- table-error-redefs+ "Sampling query throws `ex` — exercises the whole-table failure path in `fingerprint-table!`."+ [ex]+ {#'qp/process-query (fn [& _] (throw ex))})++(def ^:private field-error-redefs+ "Query returns rows but each Field's fingerprinter yields a Throwable — exercises the per-field failure path in+ `fingerprint-fields!`."+ {#'qp/process-query (fn [_query rff] (transduce identity (rff :metadata) [["a"] ["b"]]))+ #'fingerprinters/fingerprinter (constantly (fingerprinters/constant-fingerprinter (ex-info "fingerprinter boom" {})))})++(deftest retry-loop-on-failed-fingerprint-test+ (testing "GHY-3695: a Field whose fingerprint reliably fails must not be re-attempted on every sync"+ (testing "transient connection errors leave fingerprint_version untouched so the next sync retries"+ (is (= 0 (fingerprint-version-after-failure! (table-error-redefs (ConnectException. "connection refused")))))+ (is (= 0 (fingerprint-version-after-failure! (table-error-redefs (CannotAcquireResourceException. "pool exhausted"))))))+ (testing "query timeouts give up — a slow view stays slow, retrying just leaks another long query (GHY-3266)"+ (is (= 5 (fingerprint-version-after-failure! (table-error-redefs (SQLTimeoutException. "query timed out"))))))+ (testing "other query errors (permission denied, bad SQL, unsupported type) give up"+ (is (= 5 (fingerprint-version-after-failure! (table-error-redefs (SQLException. "permission denied"))))))+ (testing "per-field fingerprinter errors (unsupported coercion/dispatch) give up"+ (is (= 5 (fingerprint-version-after-failure! field-error-redefs))))))+(deftest abandon-failed-fingerprint-test(mt/test-drivers (mt/normal-drivers)(testing "fingerprinting stops after the first table if we can't connect to the db"