The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Reproduce and fix deadlock issue with remote sync (#74544)
Timothy S. Dean80950071f0e4
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex ab49279d886..0f982949f03 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -323,7 +323,8 @@:api #{metabase.app-db.cluster-lock ; TODO FIXMEmetabase.app-db.coremetabase.app-db.init- metabase.app-db.setup}+ metabase.app-db.setup+ metabase.app-db.transient-error}:uses #{auth-providerclassloaderconfigdiff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.cljindex d2ab8f34ab6..09d5e3ca2e3 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj@@ -3,10 +3,13 @@See the detailed breakdown of the (de)serialization processes in [[metabase.models.serialization]]."(:require[clojure.string :as str]+ [diehard.core :as dh][medley.core :as m][metabase-enterprise.serialization.v2.backfill-ids :as serdes.backfill][metabase-enterprise.serialization.v2.ingest :as serdes.ingest][metabase-enterprise.serialization.v2.models :as serdes.models]+ [metabase.app-db.core :as mdb]+ [metabase.app-db.transient-error :as transient-error][metabase.config.core :as config][metabase.models.serialization :as serdes][metabase.search.core :as search]@@ -15,8 +18,25 @@[toucan2.core :as t2][toucan2.model :as t2.model]))+(set! *warn-on-reflection* true)+(declare load-one!)+(defn- with-retries+ "Retries `f` up to `max-retries` times when it throws a transient DB error (deadlock, lock timeout, etc.).+ Uses exponential backoff starting at `base-delay-ms`. Error classification is appdb-type-aware+ via [[metabase.app-db.transient-error/transient-error?]]."+ [max-retries base-delay-ms f]+ (let [db-type (mdb/db-type)]+ (dh/with-retry {:max-retries max-retries+ :backoff-ms [base-delay-ms (* base-delay-ms (bit-shift-left 1 max-retries)) 2.0]+ :retry-if (fn [_result exception]+ (transient-error/transient-error? db-type exception))+ :on-retry (fn [_result exception]+ (log/warnf "Transient DB error, retrying: %s"+ (ex-message exception)))}+ (f))))+(def ^:private model->circular-dependency-keys"Sometimes models have circular dependencies. For example, a card for a Dashboard Question has a `dashboard_id`pointing to the dashboard it's in. But when we try to load that dashboard, we'll create all its dashcards, and one@@ -66,8 +86,8 @@(defn- safe-local-id"Looks up the local primary key for `path`, swallowing any exception from the DB lookup.- [[path-error-data]] is called from catch blocks inside the outer load transaction. When the original- failure was a SQL error it will have poisoned the transaction, so any subsequent query (including+ [[path-error-data]] is called from catch blocks. When the original failure was a SQL error it may+ have poisoned the current transaction, so any subsequent query (including[[serdes/load-find-local]]) will throw `current transaction is aborted, commands ignored until end oftransaction block` and shadow the real cause. We prefer losing `:local-id` over losing the exception chain."[path]@@ -192,7 +212,10 @@local-or-nil (when-not require-new-entity (serdes/load-find-local rebuilt-path))](try(warn-if-version-mismatch ingested path)- (serdes/load-one! ingested local-or-nil)+ (with-retries 3 200+ (fn []+ (t2/with-transaction [_tx]+ (serdes/load-one! ingested local-or-nil))))ctx(catch Exception e;; ugly mapv here to convert #ordered/map into normal map so it's readable in the logs@@ -225,11 +248,14 @@reindex? true}}](binding [*warned-version-mismatch* (atom false)](u/prog1- (t2/with-transaction [_tx]- ;; We proceed in the arbitrary order of ingest-list, deserializing all the files. Their declared dependencies- ;; guide the import, and make sure all containers are imported before contents, etc.+ ;; Each entity is loaded in its own transaction (inside load-one!), so a deadlock or transient+ ;; failure on one entity doesn't abort the entire import. See #74412.+ (do(when backfill?- (serdes.backfill/backfill-ids!))+ (t2/with-transaction [_tx]+ (serdes.backfill/backfill-ids!)))+ ;; We proceed in the arbitrary order of ingest-list, deserializing all the files. Their declared+ ;; dependencies guide the import, and make sure all containers are imported before contents, etc.(let [contents (serdes.ingest/ingest-list ingestion)ingest-errors (serdes.ingest/ingest-errors ingestion)ctx (cond-> (new-context ingestion)@@ -255,8 +281,6 @@ctxcontents)))(when reindex?- ;; Hack: the transaction above typically takes much longer than our delay on the search indexing queue.- ;; this means that the corresponding entries would have been missing or stale when we indexed them.- ;; ideally, we would delay the indexing somehow, or only reindex what we've loaded.- ;; while we're figuring that out, here's a crude stopgap.+ ;; Reindex after all entities are loaded. Individual entity commits may have produced stale+ ;; search index entries; this ensures the index reflects the final state.(search/reindex!)))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.cljindex a8d6b8a4c27..5dfe919c0c9 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj@@ -1517,18 +1517,92 @@(is (= (:name card)(t2/select-one-fn :name :model/Card :id (:id card)))))- (testing "Partial load does not change the database"+ (testing "Partial load commits successful entities; failed entity does not persist"(t2/update! :model/Collection {:id (:id coll)} {:name (str "qwe_" (:name coll))})+ (t2/update! :model/Card {:id (:id card)} {:name (str "qwe_" (:name card))})(let [load-update! serdes/load-update!](with-redefs [serdes/load-update! (fn [model adjusted local]- ;; Collection is loaded first+ ;; Collection is loaded first, Card fails(if (= model "Card")(throw (ex-info "oops, error" {}))(load-update! model adjusted local)))](is (thrown? clojure.lang.ExceptionInfo(serdes.load/load-metabase! (ingestion-in-memory @serialized))))- (is (= (str "qwe_" (:name coll))- (t2/select-one-fn :name :model/Collection :id (:id coll))))))))))+ ;; Collection loaded successfully in its own transaction — committed despite Card failure+ (is (= (:name coll)+ (t2/select-one-fn :name :model/Collection :id (:id coll))))+ ;; Card failed — retains its pre-load value+ (is (= (str "qwe_" (:name card))+ (t2/select-one-fn :name :model/Card :id (:id card))))))))))++(deftest transient-db-error-retry-test+ (testing "Import survives a transient deadlock on a single entity (issue #74412)"+ (mt/with-empty-h2-app-db!+ (let [coll (ts/create! :model/Collection :name "coll")+ card (ts/create! :model/Card :name "card" :collection_id (:id coll))+ serialized (atom {})]+ (reset! serialized (->> (serdes.extract/extract {:no-settings true+ :no-data-model true+ :targets [["Collection" (:id coll)]]})+ vec))+ (testing "A transient deadlock on one entity is retried and the import succeeds"+ (t2/update! :model/Card {:id (:id card)} {:name "pre-retry"})+ (let [call-count (atom 0)+ load-update! serdes/load-update!]+ (with-redefs [serdes/load-update! (fn [model adjusted local]+ (when (= model "Card")+ (swap! call-count inc)+ (when (= 1 @call-count)+ ;; Simulate a deadlock: H2 error code 40001, PG SQL state 40P01.+ ;; Use H2 codes since tests run against H2 appdb.+ (throw (ex-info "load-update! failed"+ {}+ (java.sql.SQLException.+ "Deadlock detected"+ "40001" ; SQL state+ 40001))))) ; H2 error code+ (load-update! model adjusted local))]+ (is (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ "Import should succeed after retrying the deadlocked entity")+ (is (= (:name card)+ (t2/select-one-fn :name :model/Card :id (:id card)))+ "Card should be updated to the serialized value after retry")+ (is (= 2 @call-count)+ "Card load-update! should have been called twice (first attempt deadlocked, second succeeded)"))))++ (testing "Non-transient errors propagate immediately without retry"+ (let [call-count (atom 0)+ load-update! serdes/load-update!]+ (with-redefs [serdes/load-update! (fn [model adjusted local]+ (when (= model "Card")+ (swap! call-count inc)+ (throw (ex-info "constraint violation" {})))+ (load-update! model adjusted local))]+ (is (thrown? clojure.lang.ExceptionInfo+ (serdes.load/load-metabase! (ingestion-in-memory @serialized)))+ "Non-transient error should propagate")+ (is (= 1 @call-count)+ "Should not retry non-transient errors"))))++ (testing "Successful entities are committed even when a later entity fails"+ (t2/update! :model/Collection {:id (:id coll)} {:name "pre-import"})+ (t2/update! :model/Card {:id (:id card)} {:name "pre-import"})+ (let [load-update! serdes/load-update!]+ (with-redefs [serdes/load-update! (fn [model adjusted local]+ ;; Collection loads first and succeeds; Card fails+ (if (= model "Card")+ (throw (ex-info "oops" {}))+ (load-update! model adjusted local)))]+ (is (thrown? clojure.lang.ExceptionInfo+ (serdes.load/load-metabase! (ingestion-in-memory @serialized))))+ ;; With per-entity transactions, the Collection commit survives the Card failure.+ ;; On master (single transaction), the Collection update is rolled back.+ (is (= (:name coll)+ (t2/select-one-fn :name :model/Collection :id (:id coll)))+ "Collection should be committed despite later Card failure")+ (is (= "pre-import"+ (t2/select-one-fn :name :model/Card :id (:id card)))+ "Card should retain its pre-import value"))))))))(deftest path-error-data-handles-lookup-failure-test(testing "path-error-data returns a well-formed map even when serdes/load-find-local throwsdiff --git a/src/metabase/app_db/transient_error.clj b/src/metabase/app_db/transient_error.cljnew file mode 100644index 00000000000..7671fe383a1--- /dev/null+++ b/src/metabase/app_db/transient_error.clj@@ -0,0 +1,33 @@+(ns metabase.app-db.transient-error+ "Detection of transient application database errors (deadlocks, lock timeouts, serialization failures)+ that may succeed on retry."+ (:import+ (java.sql SQLException)))++(set! *warn-on-reflection* true)++(def ^:private transient-error-codes+ "SQL states (strings) and error codes (ints) that indicate transient, retryable database errors.+ Keyed by appdb type. PostgreSQL uses SQL states; MySQL and H2 use numeric error codes."+ {:postgres #{"40P01" ; deadlock detected+ "40001" ; serialization failure+ "55P03"} ; lock not available (lock timeout)+ :mysql #{1213 ; ER_LOCK_DEADLOCK+ 1205} ; ER_LOCK_WAIT_TIMEOUT+ :h2 #{40001 ; DEADLOCK_1+ 50200}}) ; LOCK_TIMEOUT_1++(defn- transient-sql-exception?+ [codes ^SQLException e]+ (or (contains? codes (.getSQLState e))+ (contains? codes (.getErrorCode e))))++(defn transient-error?+ "Whether exception `e` represents a transient database error (deadlock, lock timeout, serialization failure)+ that may succeed on retry. Walks the full exception cause chain."+ [db-type ^Throwable e]+ (let [codes (get transient-error-codes (keyword db-type))]+ (boolean+ (some #(and (instance? SQLException %)+ (transient-sql-exception? codes %))+ (take-while some? (iterate ex-cause e))))))diff --git a/test/metabase/app_db/transient_error_test.clj b/test/metabase/app_db/transient_error_test.cljnew file mode 100644index 00000000000..4122d6c5858--- /dev/null+++ b/test/metabase/app_db/transient_error_test.clj@@ -0,0 +1,44 @@+(ns metabase.app-db.transient-error-test+ (:require+ [clojure.test :refer :all]+ [metabase.app-db.transient-error :as transient-error])+ (:import+ (java.sql SQLException)))++(deftest transient-error-h2-test+ (testing "H2 deadlock (error code 40001)"+ (is (transient-error/transient-error? :h2 (SQLException. "Deadlock" "40001" 40001))))+ (testing "H2 lock timeout (error code 50200)"+ (is (transient-error/transient-error? :h2 (SQLException. "Timeout" "HY000" 50200))))+ (testing "H2 non-transient error"+ (is (not (transient-error/transient-error? :h2 (SQLException. "Syntax error" "42000" 42000))))))++(deftest transient-error-postgres-test+ (testing "PostgreSQL deadlock (SQL state 40P01)"+ (is (transient-error/transient-error? :postgres (SQLException. "deadlock detected" "40P01"))))+ (testing "PostgreSQL serialization failure (SQL state 40001)"+ (is (transient-error/transient-error? :postgres (SQLException. "serialization failure" "40001"))))+ (testing "PostgreSQL lock timeout (SQL state 55P03)"+ (is (transient-error/transient-error? :postgres (SQLException. "lock not available" "55P03"))))+ (testing "PostgreSQL non-transient error"+ (is (not (transient-error/transient-error? :postgres (SQLException. "unique violation" "23505"))))))++(deftest transient-error-mysql-test+ (testing "MySQL deadlock (error code 1213)"+ (is (transient-error/transient-error? :mysql (SQLException. "Deadlock" "40001" 1213))))+ (testing "MySQL lock wait timeout (error code 1205)"+ (is (transient-error/transient-error? :mysql (SQLException. "Lock wait timeout" "HY000" 1205))))+ (testing "MySQL non-transient error"+ (is (not (transient-error/transient-error? :mysql (SQLException. "Duplicate entry" "23000" 1062))))))++(deftest transient-error-cause-chain-test+ (testing "Walks cause chain to find transient error"+ (let [sql-ex (SQLException. "Deadlock" "40001" 40001)+ wrapper (ex-info "wrapped" {} sql-ex)]+ (is (transient-error/transient-error? :h2 wrapper))))+ (testing "Non-transient cause chain"+ (let [sql-ex (SQLException. "Syntax error" "42000" 42000)+ wrapper (ex-info "wrapped" {} sql-ex)]+ (is (not (transient-error/transient-error? :h2 wrapper)))))+ (testing "No SQLException in chain"+ (is (not (transient-error/transient-error? :h2 (ex-info "plain error" {}))))))