Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Stop retrying permanently-failed fingerprinting every sync (#75640)

Eric Normandb47b12cf6b26
diff --git a/src/metabase/sync/analyze/fingerprint.clj b/src/metabase/sync/analyze/fingerprint.clj
index 62b3fdc17d5..c3567e1852d 100644
--- a/src/metabase/sync/analyze/fingerprint.clj
+++ b/src/metabase/sync/analyze/fingerprint.clj
@@ -38,6 +38,15 @@
(log/debugf "Saving fingerprint for %s" (sync-util/name-for-logging field))
(t2/update! :model/Field (u/the-id field) (merge (incomplete-analysis-kvs) {:fingerprint fingerprint})))
+(mu/defn- mark-fingerprinting-failed!
+ "Advance `fingerprint_version` to the latest version for `fields` without saving a fingerprint. Called when
+ fingerprinting fails for a non-transient reason, so that the Fields are not re-selected by [[fields-to-fingerprint]]
+ and retried on every subsequent sync. Transient failures (see [[metabase.sync.util/transient-exception?]]) are
+ left untouched so they are retried."
+ [fields :- [:maybe [:sequential i/FieldInstance]]]
+ (when-let [ids (seq (map u/the-id fields))]
+ (t2/update! :model/Field :id [:in ids] {:fingerprint_version i/*latest-fingerprint-version*})))
+
(mr/def ::FingerprintStats
[:map
[:no-data-fingerprints ms/IntGreaterThanOrEqualToZero]
@@ -69,7 +78,9 @@
(reduce (fn [count-info [field fingerprint]]
(cond
(instance? Throwable fingerprint)
- (update count-info :failed-fingerprints inc)
+ (do
+ (mark-fingerprinting-failed! [field])
+ (update count-info :failed-fingerprints inc))
(some-> fingerprint :global :distinct-count zero?)
(update count-info :no-data-fingerprints inc)
@@ -202,8 +213,13 @@
(let [stats
(sync-util/with-returning-throwable (format "Error fingerprinting %s" (sync-util/name-for-logging table))
(fingerprint-fields! table fields))]
- (if (:throwable stats)
- (merge (empty-stats-map 0) stats)
+ (if-let [throwable (:throwable stats)]
+ (do
+ ;; transient connection/environment errors abort this run and retry on the next sync; for any other
+ ;; (permanent) error, give up so we don't re-attempt every sync.
+ (when-not (sync-util/transient-exception? throwable)
+ (mark-fingerprinting-failed! fields))
+ (merge (empty-stats-map 0) stats))
stats)))
(empty-stats-map 0))))
diff --git a/src/metabase/sync/util.clj b/src/metabase/sync/util.clj
index df2560f393e..c4c27fc9ccd 100644
--- a/src/metabase/sync/util.clj
+++ b/src/metabase/sync/util.clj
@@ -198,22 +198,36 @@
(driver/sync-in-context (driver.u/database->driver database) database
f)))
-;; TODO: future, expand this to `driver` level, where the drivers themselves can add to the
-;; list of exception classes (like, driver-specific exceptions)
+(defonce ^:private transient-exception-hierarchy
+ (make-hierarchy))
+
+(defn register-transient-exception
+ "Register exception `klass` as a transient connection/environment-level failure — one that is expected to clear on a
+ later sync. [[transient-exception?]] then returns true for `klass` and its subclasses, so sync aborts and retries
+ the whole run later rather than treating the failure as permanent. Drivers may register driver-specific connection
+ exceptions this way."
+ [klass]
+ (alter-var-root #'transient-exception-hierarchy derive klass ::transient-exception))
+
(doseq [klass [java.net.ConnectException
java.net.NoRouteToHostException
java.net.UnknownHostException
com.mchange.v2.resourcepool.CannotAcquireResourceException
javax.net.ssl.SSLHandshakeException]]
- (derive klass ::exception-class-not-to-retry))
+ (register-transient-exception klass))
(def ^:dynamic *log-exceptions-and-continue?*
"Whether to log exceptions during a sync step and proceed with the rest of the sync process. This is the default
behavior. You can disable this for debugging or test purposes."
true)
-(defn- do-not-retry-exception? [e]
- (or (isa? (class e) ::exception-class-not-to-retry)
+(defn transient-exception?
+ "True if `e` (or any of its causes) is a connection/environment-level failure — database unreachable, connection
+ pool exhausted, SSL handshake failure, and similar — rather than a failure that says anything about the data being
+ synced. Such failures are expected to clear on a later sync and should not be treated as a permanent failure of the
+ entity being synced. Register additional classes with [[register-transient-exception]]."
+ [e]
+ (or (isa? transient-exception-hierarchy (class e) ::transient-exception)
(some-> (ex-cause e) recur)))
(defn do-with-error-handling
@@ -225,7 +239,7 @@
(try
(f)
(catch Throwable e
- (if (and *log-exceptions-and-continue?* (not (do-not-retry-exception? e)))
+ (if (and *log-exceptions-and-continue?* (not (transient-exception? e)))
(do
(log/warn e message)
e)
@@ -235,9 +249,8 @@
"Execute `body` in a way that catches and logs any Exceptions thrown, and returns the exception itself if they do so.
Pass a `message` to help provide information about what failed for the log message.
- The exception classes deriving from `:metabase.sync.util/exception-class-not-to-retry` are a list of classes tested
- against exceptions thrown. If there is a match found, the sync is aborted as that error is not considered
- recoverable for this sync run."
+ Exceptions classified as transient (see [[transient-exception?]]) are not swallowed: the sync is aborted, as the
+ database could not be reached and the failure is expected to clear on a later sync."
{:style/indent 1}
[message & body]
`(do-with-error-handling ~message (^:once fn* [] ~@body)))
@@ -673,7 +686,7 @@
"Given the results of a sync step, returns truthy if a non-recoverable exception occurred"
[step-results]
(when-let [caught-exception (:throwable step-results)]
- (do-not-retry-exception? caught-exception)))
+ (transient-exception? caught-exception)))
(mu/defn run-sync-operation
"Run `sync-steps` and log a summary message"
diff --git a/test/metabase/sync/analyze/fingerprint_test.clj b/test/metabase/sync/analyze/fingerprint_test.clj
index b99a30b29e9..bea3c88975c 100644
--- a/test/metabase/sync/analyze/fingerprint_test.clj
+++ b/test/metabase/sync/analyze/fingerprint_test.clj
@@ -11,7 +11,10 @@
[metabase.test.data :as data]
[metabase.util :as u]
[toucan2.core :as t2])
- (:import [com.mchange.v2.resourcepool CannotAcquireResourceException]))
+ (:import
+ [com.mchange.v2.resourcepool CannotAcquireResourceException]
+ [java.net ConnectException]
+ [java.sql SQLException SQLTimeoutException]))
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | TESTS FOR WHICH FIELDS NEED FINGERPRINTING |
@@ -340,6 +343,47 @@
:order-by [[:count :desc]]
:limit 1}))))))))))
+(defn- fingerprint-version-after-failure!
+ "Create a fresh text Field at fingerprint_version 0 (eligible for fingerprinting up to version 5), run
+ `fingerprint-table!` with the sample/fingerprint step failing as configured, and return the Field's
+ `fingerprint_version` afterwards. A version of 0 means the Field will be re-selected and retried on the next
+ sync; a version of 5 (the bound latest) means fingerprinting gave up and will not retry."
+ [redefs]
+ (mt/with-temp [:model/Table table {:db_id (mt/id)}
+ :model/Field field {:table_id (u/the-id table)
+ :active true
+ :base_type :type/Text
+ :fingerprint nil
+ :fingerprint_version 0}]
+ (binding [i/*latest-fingerprint-version* 5
+ i/*fingerprint-version->types-that-should-be-re-fingerprinted* {5 #{:type/Text}}]
+ (with-redefs-fn redefs
+ (fn [] (sync.fingerprint/fingerprint-table! table))))
+ (t2/select-one-fn :fingerprint_version :model/Field :id (u/the-id field))))
+
+(defn- table-error-redefs
+ "Sampling query throws `ex` — exercises the whole-table failure path in `fingerprint-table!`."
+ [ex]
+ {#'qp/process-query (fn [& _] (throw ex))})
+
+(def ^:private field-error-redefs
+ "Query returns rows but each Field's fingerprinter yields a Throwable — exercises the per-field failure path in
+ `fingerprint-fields!`."
+ {#'qp/process-query (fn [_query rff] (transduce identity (rff :metadata) [["a"] ["b"]]))
+ #'fingerprinters/fingerprinter (constantly (fingerprinters/constant-fingerprinter (ex-info "fingerprinter boom" {})))})
+
+(deftest retry-loop-on-failed-fingerprint-test
+ (testing "GHY-3695: a Field whose fingerprint reliably fails must not be re-attempted on every sync"
+ (testing "transient connection errors leave fingerprint_version untouched so the next sync retries"
+ (is (= 0 (fingerprint-version-after-failure! (table-error-redefs (ConnectException. "connection refused")))))
+ (is (= 0 (fingerprint-version-after-failure! (table-error-redefs (CannotAcquireResourceException. "pool exhausted"))))))
+ (testing "query timeouts give up — a slow view stays slow, retrying just leaks another long query (GHY-3266)"
+ (is (= 5 (fingerprint-version-after-failure! (table-error-redefs (SQLTimeoutException. "query timed out"))))))
+ (testing "other query errors (permission denied, bad SQL, unsupported type) give up"
+ (is (= 5 (fingerprint-version-after-failure! (table-error-redefs (SQLException. "permission denied"))))))
+ (testing "per-field fingerprinter errors (unsupported coercion/dispatch) give up"
+ (is (= 5 (fingerprint-version-after-failure! field-error-redefs))))))
+
(deftest abandon-failed-fingerprint-test
(mt/test-drivers (mt/normal-drivers)
(testing "fingerprinting stops after the first table if we can't connect to the db"