Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

GDGT-2332: Retry cluster-lock writes on transient deadlocks (#75172)

Ngoc Khuata65b91edee26
diff --git a/src/metabase/app_db/cluster_lock.clj b/src/metabase/app_db/cluster_lock.clj
index 0ae5295bc07..8862a11e29b 100644
--- a/src/metabase/app_db/cluster_lock.clj
+++ b/src/metabase/app_db/cluster_lock.clj
@@ -21,6 +21,7 @@
[metabase.app-db.connection :as mdb.connection]
[metabase.app-db.query :as mdb.query]
[metabase.app-db.query-cancelation :as app-db.query-cancelation]
+ [metabase.app-db.transient-error :as transient-error]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.retry :as retry]
@@ -36,6 +37,7 @@
(def ^:private cluster-lock-timeout-seconds 1)
(defn- retryable?
+ "Errors that mean we failed to *acquire* the lock and should retry acquisition."
[^Throwable e]
;; We can retry getting the cluster lock if either we tried to concurrently insert the pk
;; for the lock resulting in a SQLIntegrityConstraintViolationException or if the query
@@ -46,6 +48,17 @@
(some-> (ex-message e) (str/includes? "duplicate key value violates unique constraint \"metabase_cluster_lock_pkey\""))
(app-db.query-cancelation/query-canceled-exception? (mdb.connection/db-type) e)))
+(defn- retry-if-error?
+ "Should we retry after exception `e`? Always retry lock-acquisition failures. When `retry-transient?`
+ is set, *also* retry transient db errors (deadlocks, lock timeouts, serialization failures) — on
+ multi-master appdbs like MariaDB Galera row locks aren't replicated across nodes, so the lock can't
+ serialize writers and the conflicting commit surfaces as a deadlock. Only callers whose locked body is
+ safe to re-run from scratch (idempotent, no side effects outside the appdb transaction) should opt in."
+ [retry-transient? ^Throwable e]
+ (or (retryable? e)
+ (and retry-transient?
+ (transient-error/transient-error? (mdb.connection/db-type) e))))
+
(def ^:private default-retry-config
{:max-retries 4
:delay-ms 1000 ;; Constant delay between retries.
@@ -165,17 +178,18 @@
{:locks [(normalize-lock-spec opts)]}
(map? opts)
- (let [{:keys [lock locks timeout-seconds retry-config]} opts]
+ (let [{:keys [lock locks timeout-seconds retry-config retry-transient?]} opts]
(when (and lock locks)
(throw (ex-info "Cluster-lock opts must specify exactly one of :lock or :locks"
{:opts opts})))
(when-not (or lock locks)
(throw (ex-info "Cluster-lock opts must specify :lock or :locks" {:opts opts})))
- (cond-> {:locks (if lock
- [(normalize-lock-spec (if (map? lock)
- lock
- {:lock lock :mode (or (:mode opts) :exclusive)}))]
- (mapv normalize-lock-spec locks))}
+ (cond-> {:locks (if lock
+ [(normalize-lock-spec (if (map? lock)
+ lock
+ {:lock lock :mode (or (:mode opts) :exclusive)}))]
+ (mapv normalize-lock-spec locks))
+ :retry-transient? (boolean retry-transient?)}
timeout-seconds (assoc :timeout-seconds timeout-seconds)
retry-config (assoc :retry-config retry-config)))
@@ -191,11 +205,12 @@
- a keyword `lock-name` — shorthand for exclusive lock on that name with default
timeout and retry config.
- a map with `:lock` (a keyword) or `:locks` (a seq of specs), plus optional
- `:mode`, `:timeout-seconds`, and `:retry-config`:
+ `:mode`, `:timeout-seconds`, `:retry-config`, and `:retry-transient?`:
{:lock ::foo} ; exclusive on ::foo
{:lock ::foo :mode :share} ; shared on ::foo
{:lock ::foo :timeout-seconds 5} ; with timeout override
+ {:lock ::foo :retry-transient? true} ; also retry deadlocks (see below)
{:locks [::foo ::bar]} ; two exclusive locks
{:locks [{:lock ::root :mode :share} ; intent-lock pattern:
{:lock ::leaf :mode :exclusive}] ; shared root + exclusive
@@ -204,21 +219,30 @@
In the `:locks` form, each element is either a bare keyword (exclusive) or a
map `{:lock, :mode}`. Per-element timeout/retry config is not supported —
those live on the top-level opts map. All locks are acquired in order inside
- a single transaction."
+ a single transaction.
+
+ `:retry-transient?` (default false) additionally retries transient db errors —
+ deadlocks, lock timeouts, serialization failures — that surface from inside the
+ locked body. On multi-master appdbs (e.g. MariaDB Galera) row locks aren't
+ replicated across nodes, so the lock can't serialize writers and the conflicting
+ commit comes back as a deadlock. Only opt in when the body is safe to re-run from
+ scratch — idempotent, with no side effects outside the appdb transaction (a
+ rolled-back deadlock undoes only the db writes, not external calls)."
[opts :- [:or
:keyword
[:map
- [:lock {:optional true} :keyword]
- [:locks {:optional true} [:sequential
- [:or :keyword
- [:map
- [:lock :keyword]
- [:mode {:optional true} [:enum :exclusive :share]]]]]]
- [:mode {:optional true} [:enum :exclusive :share]]
- [:timeout-seconds {:optional true} :int]
- [:retry-config {:optional true} [:ref ::retry/retry-overrides]]]]
+ [:lock {:optional true} :keyword]
+ [:locks {:optional true} [:sequential
+ [:or :keyword
+ [:map
+ [:lock :keyword]
+ [:mode {:optional true} [:enum :exclusive :share]]]]]]
+ [:mode {:optional true} [:enum :exclusive :share]]
+ [:timeout-seconds {:optional true} :int]
+ [:retry-config {:optional true} [:ref ::retry/retry-overrides]]
+ [:retry-transient? {:optional true} :boolean]]]
thunk :- ifn?]
- (let [{:keys [locks timeout-seconds retry-config]
+ (let [{:keys [locks timeout-seconds retry-config retry-transient?]
:or {timeout-seconds cluster-lock-timeout-seconds}} (parse-opts opts)]
(cond
;; h2 does not respect the query timeout when taking the lock and is not cross-process,
@@ -227,11 +251,14 @@
(do-with-h2-cluster-locks* locks thunk)
:else
- (let [config (merge default-retry-config retry-config)]
+ (let [config (assoc (merge default-retry-config retry-config)
+ :retry-if (fn [_ e] (retry-if-error? retry-transient? e)))]
(try
(retry/with-retry config
(do-with-cluster-locks* locks timeout-seconds thunk))
(catch Throwable e
+ ;; only a genuine lock-acquisition failure gets the "Failed to obtain cluster lock" wrapper;
+ ;; an exhausted transient body error (e.g. deadlock) propagates raw so the message stays truthful.
(if (retryable? e)
(throw (ex-info (str "Failed to obtain cluster lock: "
(str/join ", " (map :lock-name-str locks)))
@@ -244,9 +271,9 @@
"Run `body` in a transaction that tries to take a lock from the metabase_cluster_lock table of
the specified name to coordinate concurrency with other metabase instances sharing the appdb.
- `lock-options` may be a lock-name keyword, an options map
- `{:lock-name, :mode, :timeout-seconds, :retry-config}`, or a vector of such specs
- (all acquired in order inside one transaction)."
+ `lock-options` may be a lock-name keyword, or an options map
+ `{:lock, :locks, :mode, :timeout-seconds, :retry-config, :retry-transient?}` —
+ see [[do-with-cluster-lock]] for the full description of each."
([lock-options & body]
`(do-with-cluster-lock ~lock-options (fn [] ~@body))))
diff --git a/src/metabase/documents/view_log.clj b/src/metabase/documents/view_log.clj
index 6b4fe7d3b03..0442db30f97 100644
--- a/src/metabase/documents/view_log.clj
+++ b/src/metabase/documents/view_log.clj
@@ -25,15 +25,19 @@
(let [document-id->timestamp (update-vals (group-by :id document-id-timestamps)
(fn [xs] (apply t/max (map :timestamp xs))))]
(try
- (cluster-lock/with-cluster-lock document-statistics-lock
- ;; use `t2/table-name` to avoid triggering `after-update` hooks and creating a revision
- (t2/update! (t2/table-name :model/Document)
- :id [:in (keys document-id->timestamp)]
- {:last_viewed_at (into [:case]
- (mapcat (fn [[id timestamp]]
- [[:= :id id] [:greatest [:coalesce :last_viewed_at (t/offset-date-time 0)] timestamp]])
- document-id->timestamp))
- :updated_at :updated_at})) ;; setting last_viewed_at should not update the updated_at column
+ ;; :retry-transient? — the body is a single idempotent statement, safe to re-run on a
+ ;; multi-master deadlock (e.g. MariaDB Galera, where the cluster lock can't serialize writers).
+ (cluster-lock/with-cluster-lock {:lock document-statistics-lock :retry-transient? true}
+ ;; Use t2/query (raw SQL) instead of t2/update! so we don't trigger Toucan2 model hooks — the
+ ;; :model/Document after-update publishes :event/document-update and syncs card collections, which
+ ;; are side effects outside the tx and must not re-fire when :retry-transient? re-runs the body.
+ (t2/query {:update (t2/table-name :model/Document)
+ :set {:last_viewed_at (into [:case]
+ (mapcat (fn [[id timestamp]]
+ [[:= :id id] [:greatest [:coalesce :last_viewed_at (t/offset-date-time 0)] timestamp]])
+ document-id->timestamp))
+ :updated_at :updated_at} ;; setting last_viewed_at should not update the updated_at column
+ :where [:in :id (keys document-id->timestamp)]}))
(catch Exception e
(log/error e "Failed to update document last_viewed_at")))))
diff --git a/src/metabase/query_processor/middleware/update_used_cards.clj b/src/metabase/query_processor/middleware/update_used_cards.clj
index c2db6a30658..df50f79b192 100644
--- a/src/metabase/query_processor/middleware/update_used_cards.clj
+++ b/src/metabase/query_processor/middleware/update_used_cards.clj
@@ -26,8 +26,10 @@
(fn [xs] (apply t/max (map :timestamp xs))))]
(log/debugf "Update last_used_at of %d cards" (count card-id->timestamp))
(try
- ;; need to use a shared lock for all updates to the card table
- (cluster-lock/with-cluster-lock cluster-lock/card-statistics-lock
+ ;; need to use a shared lock for all updates to the card table.
+ ;; :retry-transient? — the body is a single idempotent statement, safe to re-run on a
+ ;; multi-master deadlock (e.g. MariaDB Galera, where the cluster lock can't serialize writers).
+ (cluster-lock/with-cluster-lock {:lock cluster-lock/card-statistics-lock :retry-transient? true}
(t2/query {:update [(t2/table-name :model/Card)]
:where [:in :id (keys card-id->timestamp)]
:set {:last_used_at (into [:case]
diff --git a/src/metabase/view_log/events/view_log.clj b/src/metabase/view_log/events/view_log.clj
index bb999527165..471232d6b5e 100644
--- a/src/metabase/view_log/events/view_log.clj
+++ b/src/metabase/view_log/events/view_log.clj
@@ -51,7 +51,9 @@
(let [cnt->ids (group-by-frequency ids)
lock-name (view-count-lock model)]
(log/debugf "Writing %d items to %s view counts with lock %s" (count ids) model lock-name)
- (cluster-lock/with-cluster-lock lock-name
+ ;; :retry-transient? — the body is a single idempotent statement, safe to re-run on a
+ ;; multi-master deadlock (e.g. MariaDB Galera, where the cluster lock can't serialize writers).
+ (cluster-lock/with-cluster-lock {:lock lock-name :retry-transient? true}
;; Using t2/query instead of t2/update! avoids triggering Toucan2 model hooks,
;; specifically search-index enqueues on after-update.
(t2/query {:update (t2/table-name model)
@@ -152,7 +154,9 @@
;; (specifically :hook/search-index after-update). The search index can tolerate staleness on this field: the
;; index will catch up on the next re-index cycle or when the dashboard is edited. This matches the pattern used
;; in increment-view-counts!*
- (cluster-lock/with-cluster-lock dashboard-statistics-lock
+ ;; :retry-transient? — the body is a single idempotent statement, safe to re-run on a
+ ;; multi-master deadlock (e.g. MariaDB Galera, where the cluster lock can't serialize writers).
+ (cluster-lock/with-cluster-lock {:lock dashboard-statistics-lock :retry-transient? true}
(t2/query {:update (t2/table-name :model/Dashboard)
:set {:last_viewed_at (into [:case]
(mapcat (fn [[id timestamp]]
diff --git a/test/metabase/app_db/cluster_lock_test.clj b/test/metabase/app_db/cluster_lock_test.clj
index bd006b9f582..bdc5728d012 100644
--- a/test/metabase/app_db/cluster_lock_test.clj
+++ b/test/metabase/app_db/cluster_lock_test.clj
@@ -4,16 +4,68 @@
[clojure.test :refer :all]
[metabase.app-db.cluster-lock :as sut]
[metabase.app-db.core :as mdb]
+ [metabase.app-db.transient-error :as transient-error]
[metabase.test.fixtures :as fixtures]
[metabase.util :as u]
[toucan2.core :as t2])
(:import
+ (java.sql SQLException)
(java.util.concurrent CountDownLatch TimeUnit)))
(set! *warn-on-reflection* true)
(use-fixtures :once (fixtures/initialize :db))
+(defn- deadlock-exception
+ "Build a deadlock SQLException for the current appdb type. Mirrors the codes in
+ [[metabase.app-db.transient-error]] so the test exercises the real db-type path."
+ []
+ (case (mdb/db-type)
+ :postgres (SQLException. "deadlock detected" "40P01")
+ :mysql (SQLException. "Deadlock found when trying to get lock; try restarting transaction" "40001" 1213)
+ :h2 (SQLException. "Deadlock" "40001" 40001)))
+
+(deftest retry-if-error?-test
+ (testing "transient deadlocks retry only when :retry-transient? is opted in"
+ ;; opted out (default) — a multi-master deadlock is NOT retried (preserves behavior for callers whose
+ ;; locked body has side effects outside the appdb tx, which aren't safe to re-run)
+ (is (not (#'sut/retry-if-error? false (deadlock-exception))))
+ ;; opted in (stats updates) — retried so the write survives the Galera/multi-master commit conflict
+ (is (#'sut/retry-if-error? true (deadlock-exception)))
+ (testing "even when wrapped in the rollback wrapper the driver adds"
+ (is (#'sut/retry-if-error? true (ex-info "Error rolling back after previous error" {} (deadlock-exception))))))
+ (testing "non-transient errors are never retried, regardless of :retry-transient?"
+ (is (not (#'sut/retry-if-error? true (ex-info "Bad Error" {}))))
+ (is (not (#'sut/retry-if-error? false (ex-info "Bad Error" {}))))))
+
+(deftest retry-transient-end-to-end-test
+ ;; h2 takes an in-process lock and bypasses the retry path entirely, so this only exercises real row locks.
+ (when (not= (mdb/db-type) :h2)
+ (let [retry-cfg {:delay-ms 1 :max-retries 3}]
+ (testing ":retry-transient? true re-runs the locked body after a transient deadlock"
+ (let [attempts (atom 0)
+ result (sut/with-cluster-lock {:lock ::retry-transient-test
+ :retry-transient? true
+ :retry-config retry-cfg}
+ (when (= 1 (swap! attempts inc))
+ (throw (deadlock-exception)))
+ :ok)]
+ (is (= :ok result))
+ (is (= 2 @attempts))))
+ (testing "without :retry-transient? the same deadlock is not retried and propagates"
+ (let [attempts (atom 0)
+ ;; toucan2 wraps a body exception thrown inside the tx in an ExceptionInfo, so the deadlock
+ ;; surfaces as that wrapper rather than a bare SQLException — but the SQLException is still in
+ ;; its cause chain.
+ e (is (thrown? Throwable
+ (sut/with-cluster-lock {:lock ::retry-transient-test
+ :retry-config retry-cfg}
+ (swap! attempts inc)
+ (throw (deadlock-exception)))))]
+ ;; it's a genuine transient error; it propagated unretried only because we didn't opt in.
+ (is (transient-error/transient-error? (mdb/db-type) e))
+ (is (= 1 @attempts)))))))
+
(deftest with-cluster-locking-test
(testing "works when used non-concurrently"
(is (nil? (sut/with-cluster-lock ::test-lock (Thread/sleep 1)))))