Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Reproduce and fix deadlock issue with remote sync (#74544)

Timothy S. Dean80950071f0e4
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index ab49279d886..0f982949f03 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -323,7 +323,8 @@
:api #{metabase.app-db.cluster-lock ; TODO FIXME
metabase.app-db.core
metabase.app-db.init
- metabase.app-db.setup}
+ metabase.app-db.setup
+ metabase.app-db.transient-error}
:uses #{auth-provider
classloader
config
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
index d2ab8f34ab6..09d5e3ca2e3 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
@@ -3,10 +3,13 @@
See the detailed breakdown of the (de)serialization processes in [[metabase.models.serialization]]."
(:require
[clojure.string :as str]
+ [diehard.core :as dh]
[medley.core :as m]
[metabase-enterprise.serialization.v2.backfill-ids :as serdes.backfill]
[metabase-enterprise.serialization.v2.ingest :as serdes.ingest]
[metabase-enterprise.serialization.v2.models :as serdes.models]
+ [metabase.app-db.core :as mdb]
+ [metabase.app-db.transient-error :as transient-error]
[metabase.config.core :as config]
[metabase.models.serialization :as serdes]
[metabase.search.core :as search]
@@ -15,8 +18,25 @@
[toucan2.core :as t2]
[toucan2.model :as t2.model]))
+(set! *warn-on-reflection* true)
+
(declare load-one!)
+(defn- with-retries
+ "Retries `f` up to `max-retries` times when it throws a transient DB error (deadlock, lock timeout, etc.).
+ Uses exponential backoff starting at `base-delay-ms`. Error classification is appdb-type-aware
+ via [[metabase.app-db.transient-error/transient-error?]]."
+ [max-retries base-delay-ms f]
+ (let [db-type (mdb/db-type)]
+ (dh/with-retry {:max-retries max-retries
+ :backoff-ms [base-delay-ms (* base-delay-ms (bit-shift-left 1 max-retries)) 2.0]
+ :retry-if (fn [_result exception]
+ (transient-error/transient-error? db-type exception))
+ :on-retry (fn [_result exception]
+ (log/warnf "Transient DB error, retrying: %s"
+ (ex-message exception)))}
+ (f))))
+
(def ^:private model->circular-dependency-keys
"Sometimes models have circular dependencies. For example, a card for a Dashboard Question has a `dashboard_id`
pointing to the dashboard it's in. But when we try to load that dashboard, we'll create all its dashcards, and one
@@ -66,8 +86,8 @@
(defn- safe-local-id
"Looks up the local primary key for `path`, swallowing any exception from the DB lookup.
- [[path-error-data]] is called from catch blocks inside the outer load transaction. When the original
- failure was a SQL error it will have poisoned the transaction, so any subsequent query (including
+ [[path-error-data]] is called from catch blocks. When the original failure was a SQL error it may
+ have poisoned the current transaction, so any subsequent query (including
[[serdes/load-find-local]]) will throw `current transaction is aborted, commands ignored until end of
transaction block` and shadow the real cause. We prefer losing `:local-id` over losing the exception chain."
[path]
@@ -192,7 +212,10 @@
local-or-nil (when-not require-new-entity (serdes/load-find-local rebuilt-path))]
(try
(warn-if-version-mismatch ingested path)
- (serdes/load-one! ingested local-or-nil)
+ (with-retries 3 200
+ (fn []
+ (t2/with-transaction [_tx]
+ (serdes/load-one! ingested local-or-nil))))
ctx
(catch Exception e
;; ugly mapv here to convert #ordered/map into normal map so it's readable in the logs
@@ -225,11 +248,14 @@
reindex? true}}]
(binding [*warned-version-mismatch* (atom false)]
(u/prog1
- (t2/with-transaction [_tx]
- ;; We proceed in the arbitrary order of ingest-list, deserializing all the files. Their declared dependencies
- ;; guide the import, and make sure all containers are imported before contents, etc.
+ ;; Each entity is loaded in its own transaction (inside load-one!), so a deadlock or transient
+ ;; failure on one entity doesn't abort the entire import. See #74412.
+ (do
(when backfill?
- (serdes.backfill/backfill-ids!))
+ (t2/with-transaction [_tx]
+ (serdes.backfill/backfill-ids!)))
+ ;; We proceed in the arbitrary order of ingest-list, deserializing all the files. Their declared
+ ;; dependencies guide the import, and make sure all containers are imported before contents, etc.
(let [contents (serdes.ingest/ingest-list ingestion)
ingest-errors (serdes.ingest/ingest-errors ingestion)
ctx (cond-> (new-context ingestion)
@@ -255,8 +281,6 @@
ctx
contents)))
(when reindex?
- ;; Hack: the transaction above typically takes much longer than our delay on the search indexing queue.
- ;; this means that the corresponding entries would have been missing or stale when we indexed them.
- ;; ideally, we would delay the indexing somehow, or only reindex what we've loaded.
- ;; while we're figuring that out, here's a crude stopgap.
+ ;; Reindex after all entities are loaded. Individual entity commits may have produced stale
+ ;; search index entries; this ensures the index reflects the final state.
(search/reindex!)))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
index a8d6b8a4c27..5dfe919c0c9 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
@@ -1517,18 +1517,92 @@
(is (= (:name card)
(t2/select-one-fn :name :model/Card :id (:id card)))))
- (testing "Partial load does not change the database"
+ (testing "Partial load commits successful entities; failed entity does not persist"
(t2/update! :model/Collection {:id (:id coll)} {:name (str "qwe_" (:name coll))})
+ (t2/update! :model/Card {:id (:id card)} {:name (str "qwe_" (:name card))})
(let [load-update! serdes/load-update!]
(with-redefs [serdes/load-update! (fn [model adjusted local]
- ;; Collection is loaded first
+ ;; Collection is loaded first, Card fails
(if (= model "Card")
(throw (ex-info "oops, error" {}))
(load-update! model adjusted local)))]
(is (thrown? clojure.lang.ExceptionInfo
(serdes.load/load-metabase! (ingestion-in-memory @serialized))))
- (is (= (str "qwe_" (:name coll))
- (t2/select-one-fn :name :model/Collection :id (:id coll))))))))))
+ ;; Collection loaded successfully in its own transaction — committed despite Card failure
+ (is (= (:name coll)
+ (t2/select-one-fn :name :model/Collection :id (:id coll))))
+ ;; Card failed — retains its pre-load value
+ (is (= (str "qwe_" (:name card))
+ (t2/select-one-fn :name :model/Card :id (:id card))))))))))
+
+(deftest transient-db-error-retry-test
+ (testing "Import survives a transient deadlock on a single entity (issue #74412)"
+ (mt/with-empty-h2-app-db!
+ (let [coll (ts/create! :model/Collection :name "coll")
+ card (ts/create! :model/Card :name "card" :collection_id (:id coll))
+ serialized (atom {})]
+ (reset! serialized (->> (serdes.extract/extract {:no-settings true
+ :no-data-model true
+ :targets [["Collection" (:id coll)]]})
+ vec))
+ (testing "A transient deadlock on one entity is retried and the import succeeds"
+ (t2/update! :model/Card {:id (:id card)} {:name "pre-retry"})
+ (let [call-count (atom 0)
+ load-update! serdes/load-update!]
+ (with-redefs [serdes/load-update! (fn [model adjusted local]
+ (when (= model "Card")
+ (swap! call-count inc)
+ (when (= 1 @call-count)
+ ;; Simulate a deadlock: H2 error code 40001, PG SQL state 40P01.
+ ;; Use H2 codes since tests run against H2 appdb.
+ (throw (ex-info "load-update! failed"
+ {}
+ (java.sql.SQLException.
+ "Deadlock detected"
+ "40001" ; SQL state
+ 40001))))) ; H2 error code
+ (load-update! model adjusted local))]
+ (is (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ "Import should succeed after retrying the deadlocked entity")
+ (is (= (:name card)
+ (t2/select-one-fn :name :model/Card :id (:id card)))
+ "Card should be updated to the serialized value after retry")
+ (is (= 2 @call-count)
+ "Card load-update! should have been called twice (first attempt deadlocked, second succeeded)"))))
+
+ (testing "Non-transient errors propagate immediately without retry"
+ (let [call-count (atom 0)
+ load-update! serdes/load-update!]
+ (with-redefs [serdes/load-update! (fn [model adjusted local]
+ (when (= model "Card")
+ (swap! call-count inc)
+ (throw (ex-info "constraint violation" {})))
+ (load-update! model adjusted local))]
+ (is (thrown? clojure.lang.ExceptionInfo
+ (serdes.load/load-metabase! (ingestion-in-memory @serialized)))
+ "Non-transient error should propagate")
+ (is (= 1 @call-count)
+ "Should not retry non-transient errors"))))
+
+ (testing "Successful entities are committed even when a later entity fails"
+ (t2/update! :model/Collection {:id (:id coll)} {:name "pre-import"})
+ (t2/update! :model/Card {:id (:id card)} {:name "pre-import"})
+ (let [load-update! serdes/load-update!]
+ (with-redefs [serdes/load-update! (fn [model adjusted local]
+ ;; Collection loads first and succeeds; Card fails
+ (if (= model "Card")
+ (throw (ex-info "oops" {}))
+ (load-update! model adjusted local)))]
+ (is (thrown? clojure.lang.ExceptionInfo
+ (serdes.load/load-metabase! (ingestion-in-memory @serialized))))
+ ;; With per-entity transactions, the Collection commit survives the Card failure.
+ ;; On master (single transaction), the Collection update is rolled back.
+ (is (= (:name coll)
+ (t2/select-one-fn :name :model/Collection :id (:id coll)))
+ "Collection should be committed despite later Card failure")
+ (is (= "pre-import"
+ (t2/select-one-fn :name :model/Card :id (:id card)))
+ "Card should retain its pre-import value"))))))))
(deftest path-error-data-handles-lookup-failure-test
(testing "path-error-data returns a well-formed map even when serdes/load-find-local throws
diff --git a/src/metabase/app_db/transient_error.clj b/src/metabase/app_db/transient_error.clj
new file mode 100644
index 00000000000..7671fe383a1
--- /dev/null
+++ b/src/metabase/app_db/transient_error.clj
@@ -0,0 +1,33 @@
+(ns metabase.app-db.transient-error
+ "Detection of transient application database errors (deadlocks, lock timeouts, serialization failures)
+ that may succeed on retry."
+ (:import
+ (java.sql SQLException)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private transient-error-codes
+ "SQL states (strings) and error codes (ints) that indicate transient, retryable database errors.
+ Keyed by appdb type. PostgreSQL uses SQL states; MySQL and H2 use numeric error codes."
+ {:postgres #{"40P01" ; deadlock detected
+ "40001" ; serialization failure
+ "55P03"} ; lock not available (lock timeout)
+ :mysql #{1213 ; ER_LOCK_DEADLOCK
+ 1205} ; ER_LOCK_WAIT_TIMEOUT
+ :h2 #{40001 ; DEADLOCK_1
+ 50200}}) ; LOCK_TIMEOUT_1
+
+(defn- transient-sql-exception?
+ [codes ^SQLException e]
+ (or (contains? codes (.getSQLState e))
+ (contains? codes (.getErrorCode e))))
+
+(defn transient-error?
+ "Whether exception `e` represents a transient database error (deadlock, lock timeout, serialization failure)
+ that may succeed on retry. Walks the full exception cause chain."
+ [db-type ^Throwable e]
+ (let [codes (get transient-error-codes (keyword db-type))]
+ (boolean
+ (some #(and (instance? SQLException %)
+ (transient-sql-exception? codes %))
+ (take-while some? (iterate ex-cause e))))))
diff --git a/test/metabase/app_db/transient_error_test.clj b/test/metabase/app_db/transient_error_test.clj
new file mode 100644
index 00000000000..4122d6c5858
--- /dev/null
+++ b/test/metabase/app_db/transient_error_test.clj
@@ -0,0 +1,44 @@
+(ns metabase.app-db.transient-error-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.app-db.transient-error :as transient-error])
+ (:import
+ (java.sql SQLException)))
+
+(deftest transient-error-h2-test
+ (testing "H2 deadlock (error code 40001)"
+ (is (transient-error/transient-error? :h2 (SQLException. "Deadlock" "40001" 40001))))
+ (testing "H2 lock timeout (error code 50200)"
+ (is (transient-error/transient-error? :h2 (SQLException. "Timeout" "HY000" 50200))))
+ (testing "H2 non-transient error"
+ (is (not (transient-error/transient-error? :h2 (SQLException. "Syntax error" "42000" 42000))))))
+
+(deftest transient-error-postgres-test
+ (testing "PostgreSQL deadlock (SQL state 40P01)"
+ (is (transient-error/transient-error? :postgres (SQLException. "deadlock detected" "40P01"))))
+ (testing "PostgreSQL serialization failure (SQL state 40001)"
+ (is (transient-error/transient-error? :postgres (SQLException. "serialization failure" "40001"))))
+ (testing "PostgreSQL lock timeout (SQL state 55P03)"
+ (is (transient-error/transient-error? :postgres (SQLException. "lock not available" "55P03"))))
+ (testing "PostgreSQL non-transient error"
+ (is (not (transient-error/transient-error? :postgres (SQLException. "unique violation" "23505"))))))
+
+(deftest transient-error-mysql-test
+ (testing "MySQL deadlock (error code 1213)"
+ (is (transient-error/transient-error? :mysql (SQLException. "Deadlock" "40001" 1213))))
+ (testing "MySQL lock wait timeout (error code 1205)"
+ (is (transient-error/transient-error? :mysql (SQLException. "Lock wait timeout" "HY000" 1205))))
+ (testing "MySQL non-transient error"
+ (is (not (transient-error/transient-error? :mysql (SQLException. "Duplicate entry" "23000" 1062))))))
+
+(deftest transient-error-cause-chain-test
+ (testing "Walks cause chain to find transient error"
+ (let [sql-ex (SQLException. "Deadlock" "40001" 40001)
+ wrapper (ex-info "wrapped" {} sql-ex)]
+ (is (transient-error/transient-error? :h2 wrapper))))
+ (testing "Non-transient cause chain"
+ (let [sql-ex (SQLException. "Syntax error" "42000" 42000)
+ wrapper (ex-info "wrapped" {} sql-ex)]
+ (is (not (transient-error/transient-error? :h2 wrapper)))))
+ (testing "No SQLException in chain"
+ (is (not (transient-error/transient-error? :h2 (ex-info "plain error" {}))))))