The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GDGT-2332: Retry cluster-lock writes on transient deadlocks (#75172)
Ngoc Khuata65b91edee26
diff --git a/src/metabase/app_db/cluster_lock.clj b/src/metabase/app_db/cluster_lock.cljindex 0ae5295bc07..8862a11e29b 100644--- a/src/metabase/app_db/cluster_lock.clj+++ b/src/metabase/app_db/cluster_lock.clj@@ -21,6 +21,7 @@[metabase.app-db.connection :as mdb.connection][metabase.app-db.query :as mdb.query][metabase.app-db.query-cancelation :as app-db.query-cancelation]+ [metabase.app-db.transient-error :as transient-error][metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.retry :as retry]@@ -36,6 +37,7 @@(def ^:private cluster-lock-timeout-seconds 1)(defn- retryable?+ "Errors that mean we failed to *acquire* the lock and should retry acquisition."[^Throwable e];; We can retry getting the cluster lock if either we tried to concurrently insert the pk;; for the lock resulting in a SQLIntegrityConstraintViolationException or if the query@@ -46,6 +48,17 @@(some-> (ex-message e) (str/includes? "duplicate key value violates unique constraint \"metabase_cluster_lock_pkey\""))(app-db.query-cancelation/query-canceled-exception? (mdb.connection/db-type) e)))+(defn- retry-if-error?+ "Should we retry after exception `e`? Always retry lock-acquisition failures. When `retry-transient?`+ is set, *also* retry transient db errors (deadlocks, lock timeouts, serialization failures) — on+ multi-master appdbs like MariaDB Galera row locks aren't replicated across nodes, so the lock can't+ serialize writers and the conflicting commit surfaces as a deadlock. Only callers whose locked body is+ safe to re-run from scratch (idempotent, no side effects outside the appdb transaction) should opt in."+ [retry-transient? ^Throwable e]+ (or (retryable? e)+ (and retry-transient?+ (transient-error/transient-error? (mdb.connection/db-type) e))))+(def ^:private default-retry-config{:max-retries 4:delay-ms 1000 ;; Constant delay between retries.@@ -165,17 +178,18 @@{:locks [(normalize-lock-spec opts)]}(map? opts)- (let [{:keys [lock locks timeout-seconds retry-config]} opts]+ (let [{:keys [lock locks timeout-seconds retry-config retry-transient?]} opts](when (and lock locks)(throw (ex-info "Cluster-lock opts must specify exactly one of :lock or :locks"{:opts opts})))(when-not (or lock locks)(throw (ex-info "Cluster-lock opts must specify :lock or :locks" {:opts opts})))- (cond-> {:locks (if lock- [(normalize-lock-spec (if (map? lock)- lock- {:lock lock :mode (or (:mode opts) :exclusive)}))]- (mapv normalize-lock-spec locks))}+ (cond-> {:locks (if lock+ [(normalize-lock-spec (if (map? lock)+ lock+ {:lock lock :mode (or (:mode opts) :exclusive)}))]+ (mapv normalize-lock-spec locks))+ :retry-transient? (boolean retry-transient?)}timeout-seconds (assoc :timeout-seconds timeout-seconds)retry-config (assoc :retry-config retry-config)))@@ -191,11 +205,12 @@- a keyword `lock-name` — shorthand for exclusive lock on that name with defaulttimeout and retry config.- a map with `:lock` (a keyword) or `:locks` (a seq of specs), plus optional- `:mode`, `:timeout-seconds`, and `:retry-config`:+ `:mode`, `:timeout-seconds`, `:retry-config`, and `:retry-transient?`:{:lock ::foo} ; exclusive on ::foo{:lock ::foo :mode :share} ; shared on ::foo{:lock ::foo :timeout-seconds 5} ; with timeout override+ {:lock ::foo :retry-transient? true} ; also retry deadlocks (see below){:locks [::foo ::bar]} ; two exclusive locks{:locks [{:lock ::root :mode :share} ; intent-lock pattern:{:lock ::leaf :mode :exclusive}] ; shared root + exclusive@@ -204,21 +219,30 @@In the `:locks` form, each element is either a bare keyword (exclusive) or amap `{:lock, :mode}`. Per-element timeout/retry config is not supported —those live on the top-level opts map. All locks are acquired in order inside- a single transaction."+ a single transaction.++ `:retry-transient?` (default false) additionally retries transient db errors —+ deadlocks, lock timeouts, serialization failures — that surface from inside the+ locked body. On multi-master appdbs (e.g. MariaDB Galera) row locks aren't+ replicated across nodes, so the lock can't serialize writers and the conflicting+ commit comes back as a deadlock. Only opt in when the body is safe to re-run from+ scratch — idempotent, with no side effects outside the appdb transaction (a+ rolled-back deadlock undoes only the db writes, not external calls)."[opts :- [:or:keyword[:map- [:lock {:optional true} :keyword]- [:locks {:optional true} [:sequential- [:or :keyword- [:map- [:lock :keyword]- [:mode {:optional true} [:enum :exclusive :share]]]]]]- [:mode {:optional true} [:enum :exclusive :share]]- [:timeout-seconds {:optional true} :int]- [:retry-config {:optional true} [:ref ::retry/retry-overrides]]]]+ [:lock {:optional true} :keyword]+ [:locks {:optional true} [:sequential+ [:or :keyword+ [:map+ [:lock :keyword]+ [:mode {:optional true} [:enum :exclusive :share]]]]]]+ [:mode {:optional true} [:enum :exclusive :share]]+ [:timeout-seconds {:optional true} :int]+ [:retry-config {:optional true} [:ref ::retry/retry-overrides]]+ [:retry-transient? {:optional true} :boolean]]]thunk :- ifn?]- (let [{:keys [locks timeout-seconds retry-config]+ (let [{:keys [locks timeout-seconds retry-config retry-transient?]:or {timeout-seconds cluster-lock-timeout-seconds}} (parse-opts opts)](cond;; h2 does not respect the query timeout when taking the lock and is not cross-process,@@ -227,11 +251,14 @@(do-with-h2-cluster-locks* locks thunk):else- (let [config (merge default-retry-config retry-config)]+ (let [config (assoc (merge default-retry-config retry-config)+ :retry-if (fn [_ e] (retry-if-error? retry-transient? e)))](try(retry/with-retry config(do-with-cluster-locks* locks timeout-seconds thunk))(catch Throwable e+ ;; only a genuine lock-acquisition failure gets the "Failed to obtain cluster lock" wrapper;+ ;; an exhausted transient body error (e.g. deadlock) propagates raw so the message stays truthful.(if (retryable? e)(throw (ex-info (str "Failed to obtain cluster lock: "(str/join ", " (map :lock-name-str locks)))@@ -244,9 +271,9 @@"Run `body` in a transaction that tries to take a lock from the metabase_cluster_lock table ofthe specified name to coordinate concurrency with other metabase instances sharing the appdb.- `lock-options` may be a lock-name keyword, an options map- `{:lock-name, :mode, :timeout-seconds, :retry-config}`, or a vector of such specs- (all acquired in order inside one transaction)."+ `lock-options` may be a lock-name keyword, or an options map+ `{:lock, :locks, :mode, :timeout-seconds, :retry-config, :retry-transient?}` —+ see [[do-with-cluster-lock]] for the full description of each."([lock-options & body]`(do-with-cluster-lock ~lock-options (fn [] ~@body))))diff --git a/src/metabase/documents/view_log.clj b/src/metabase/documents/view_log.cljindex 6b4fe7d3b03..0442db30f97 100644--- a/src/metabase/documents/view_log.clj+++ b/src/metabase/documents/view_log.clj@@ -25,15 +25,19 @@(let [document-id->timestamp (update-vals (group-by :id document-id-timestamps)(fn [xs] (apply t/max (map :timestamp xs))))](try- (cluster-lock/with-cluster-lock document-statistics-lock- ;; use `t2/table-name` to avoid triggering `after-update` hooks and creating a revision- (t2/update! (t2/table-name :model/Document)- :id [:in (keys document-id->timestamp)]- {:last_viewed_at (into [:case]- (mapcat (fn [[id timestamp]]- [[:= :id id] [:greatest [:coalesce :last_viewed_at (t/offset-date-time 0)] timestamp]])- document-id->timestamp))- :updated_at :updated_at})) ;; setting last_viewed_at should not update the updated_at column+ ;; :retry-transient? — the body is a single idempotent statement, safe to re-run on a+ ;; multi-master deadlock (e.g. MariaDB Galera, where the cluster lock can't serialize writers).+ (cluster-lock/with-cluster-lock {:lock document-statistics-lock :retry-transient? true}+ ;; Use t2/query (raw SQL) instead of t2/update! so we don't trigger Toucan2 model hooks — the+ ;; :model/Document after-update publishes :event/document-update and syncs card collections, which+ ;; are side effects outside the tx and must not re-fire when :retry-transient? re-runs the body.+ (t2/query {:update (t2/table-name :model/Document)+ :set {:last_viewed_at (into [:case]+ (mapcat (fn [[id timestamp]]+ [[:= :id id] [:greatest [:coalesce :last_viewed_at (t/offset-date-time 0)] timestamp]])+ document-id->timestamp))+ :updated_at :updated_at} ;; setting last_viewed_at should not update the updated_at column+ :where [:in :id (keys document-id->timestamp)]}))(catch Exception e(log/error e "Failed to update document last_viewed_at")))))diff --git a/src/metabase/query_processor/middleware/update_used_cards.clj b/src/metabase/query_processor/middleware/update_used_cards.cljindex c2db6a30658..df50f79b192 100644--- a/src/metabase/query_processor/middleware/update_used_cards.clj+++ b/src/metabase/query_processor/middleware/update_used_cards.clj@@ -26,8 +26,10 @@(fn [xs] (apply t/max (map :timestamp xs))))](log/debugf "Update last_used_at of %d cards" (count card-id->timestamp))(try- ;; need to use a shared lock for all updates to the card table- (cluster-lock/with-cluster-lock cluster-lock/card-statistics-lock+ ;; need to use a shared lock for all updates to the card table.+ ;; :retry-transient? — the body is a single idempotent statement, safe to re-run on a+ ;; multi-master deadlock (e.g. MariaDB Galera, where the cluster lock can't serialize writers).+ (cluster-lock/with-cluster-lock {:lock cluster-lock/card-statistics-lock :retry-transient? true}(t2/query {:update [(t2/table-name :model/Card)]:where [:in :id (keys card-id->timestamp)]:set {:last_used_at (into [:case]diff --git a/src/metabase/view_log/events/view_log.clj b/src/metabase/view_log/events/view_log.cljindex bb999527165..471232d6b5e 100644--- a/src/metabase/view_log/events/view_log.clj+++ b/src/metabase/view_log/events/view_log.clj@@ -51,7 +51,9 @@(let [cnt->ids (group-by-frequency ids)lock-name (view-count-lock model)](log/debugf "Writing %d items to %s view counts with lock %s" (count ids) model lock-name)- (cluster-lock/with-cluster-lock lock-name+ ;; :retry-transient? — the body is a single idempotent statement, safe to re-run on a+ ;; multi-master deadlock (e.g. MariaDB Galera, where the cluster lock can't serialize writers).+ (cluster-lock/with-cluster-lock {:lock lock-name :retry-transient? true};; Using t2/query instead of t2/update! avoids triggering Toucan2 model hooks,;; specifically search-index enqueues on after-update.(t2/query {:update (t2/table-name model)@@ -152,7 +154,9 @@;; (specifically :hook/search-index after-update). The search index can tolerate staleness on this field: the;; index will catch up on the next re-index cycle or when the dashboard is edited. This matches the pattern used;; in increment-view-counts!*- (cluster-lock/with-cluster-lock dashboard-statistics-lock+ ;; :retry-transient? — the body is a single idempotent statement, safe to re-run on a+ ;; multi-master deadlock (e.g. MariaDB Galera, where the cluster lock can't serialize writers).+ (cluster-lock/with-cluster-lock {:lock dashboard-statistics-lock :retry-transient? true}(t2/query {:update (t2/table-name :model/Dashboard):set {:last_viewed_at (into [:case](mapcat (fn [[id timestamp]]diff --git a/test/metabase/app_db/cluster_lock_test.clj b/test/metabase/app_db/cluster_lock_test.cljindex bd006b9f582..bdc5728d012 100644--- a/test/metabase/app_db/cluster_lock_test.clj+++ b/test/metabase/app_db/cluster_lock_test.clj@@ -4,16 +4,68 @@[clojure.test :refer :all][metabase.app-db.cluster-lock :as sut][metabase.app-db.core :as mdb]+ [metabase.app-db.transient-error :as transient-error][metabase.test.fixtures :as fixtures][metabase.util :as u][toucan2.core :as t2])(:import+ (java.sql SQLException)(java.util.concurrent CountDownLatch TimeUnit)))(set! *warn-on-reflection* true)(use-fixtures :once (fixtures/initialize :db))+(defn- deadlock-exception+ "Build a deadlock SQLException for the current appdb type. Mirrors the codes in+ [[metabase.app-db.transient-error]] so the test exercises the real db-type path."+ []+ (case (mdb/db-type)+ :postgres (SQLException. "deadlock detected" "40P01")+ :mysql (SQLException. "Deadlock found when trying to get lock; try restarting transaction" "40001" 1213)+ :h2 (SQLException. "Deadlock" "40001" 40001)))++(deftest retry-if-error?-test+ (testing "transient deadlocks retry only when :retry-transient? is opted in"+ ;; opted out (default) — a multi-master deadlock is NOT retried (preserves behavior for callers whose+ ;; locked body has side effects outside the appdb tx, which aren't safe to re-run)+ (is (not (#'sut/retry-if-error? false (deadlock-exception))))+ ;; opted in (stats updates) — retried so the write survives the Galera/multi-master commit conflict+ (is (#'sut/retry-if-error? true (deadlock-exception)))+ (testing "even when wrapped in the rollback wrapper the driver adds"+ (is (#'sut/retry-if-error? true (ex-info "Error rolling back after previous error" {} (deadlock-exception))))))+ (testing "non-transient errors are never retried, regardless of :retry-transient?"+ (is (not (#'sut/retry-if-error? true (ex-info "Bad Error" {}))))+ (is (not (#'sut/retry-if-error? false (ex-info "Bad Error" {}))))))++(deftest retry-transient-end-to-end-test+ ;; h2 takes an in-process lock and bypasses the retry path entirely, so this only exercises real row locks.+ (when (not= (mdb/db-type) :h2)+ (let [retry-cfg {:delay-ms 1 :max-retries 3}]+ (testing ":retry-transient? true re-runs the locked body after a transient deadlock"+ (let [attempts (atom 0)+ result (sut/with-cluster-lock {:lock ::retry-transient-test+ :retry-transient? true+ :retry-config retry-cfg}+ (when (= 1 (swap! attempts inc))+ (throw (deadlock-exception)))+ :ok)]+ (is (= :ok result))+ (is (= 2 @attempts))))+ (testing "without :retry-transient? the same deadlock is not retried and propagates"+ (let [attempts (atom 0)+ ;; toucan2 wraps a body exception thrown inside the tx in an ExceptionInfo, so the deadlock+ ;; surfaces as that wrapper rather than a bare SQLException — but the SQLException is still in+ ;; its cause chain.+ e (is (thrown? Throwable+ (sut/with-cluster-lock {:lock ::retry-transient-test+ :retry-config retry-cfg}+ (swap! attempts inc)+ (throw (deadlock-exception)))))]+ ;; it's a genuine transient error; it propagated unretried only because we didn't opt in.+ (is (transient-error/transient-error? (mdb/db-type) e))+ (is (= 1 @attempts)))))))+(deftest with-cluster-locking-test(testing "works when used non-concurrently"(is (nil? (sut/with-cluster-lock ::test-lock (Thread/sleep 1)))))