Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[BOT-1838] Release JDBC savepoints when nested transactions OK (#77562)

Tyler Plude0c06e94821c8
diff --git a/src/metabase/app_db/connection.clj b/src/metabase/app_db/connection.clj
index 8052ff4d4b0..9e5bf2c2397 100644
--- a/src/metabase/app_db/connection.clj
+++ b/src/metabase/app_db/connection.clj
@@ -189,6 +189,57 @@
;; back to its own, later savepoint does not discard an earlier scope's writes.
(def ^:private ^:dynamic *rollback-required* nil)
+;; Open savepoints in creation order, each with whether its scope has finished. Shared by the tree, including threads
+;; that inherit its bindings. Releasing a savepoint destroys every later one, so a scope may only release once no
+;; later scope is still running; a sibling thread's early release would otherwise leave that scope nothing to release
+;; or roll back to, and on postgres the failed release aborts the transaction.
+(def ^:private ^:dynamic *open-savepoints* nil)
+
+(defn- set-savepoint! [^java.sql.Connection connection]
+ (let [open *open-savepoints*]
+ (locking open
+ (let [savepoint (.setSavepoint connection)]
+ (swap! open conj {:savepoint savepoint, :finished? false})
+ savepoint))))
+
+(defn- rollback-to-savepoint!
+ "Roll back to `savepoint` and forget it and everything after it, which no longer exists."
+ [^java.sql.Connection connection savepoint]
+ (let [open *open-savepoints*]
+ (locking open
+ (try
+ (.rollback connection savepoint)
+ (finally
+ (swap! open (fn [entries]
+ (into [] (take-while #(not (identical? (:savepoint %) savepoint)) entries)))))))))
+
+(defn- release-finished-savepoints!
+ "Mark `savepoint` finished and release the earliest savepoint with no unfinished successor, which frees the later
+ finished ones with it."
+ [^java.sql.Connection connection savepoint]
+ (let [open *open-savepoints*]
+ (locking open
+ (let [entries (swap! open (fn [entries]
+ (mapv #(cond-> % (identical? (:savepoint %) savepoint) (assoc :finished? true))
+ entries)))
+ first-releasable (loop [i (count entries)]
+ (cond
+ (zero? i) 0
+ (not (:finished? (nth entries (dec i)))) i
+ :else (recur (dec i))))]
+ (when (< first-releasable (count entries))
+ ;; copy rather than keep the subvec view of the discarded entries
+ (reset! open (into [] (subvec entries 0 first-releasable)))
+ (try
+ (.releaseSavepoint connection (:savepoint (nth entries first-releasable)))
+ (catch Throwable e
+ ;; Either the savepoint is already gone -- DDL commits implicitly on H2 and MySQL -- or, on postgres,
+ ;; the transaction is already aborted because a scope swallowed a SQL error. Nothing to undo either
+ ;; way, but the aborted case is worth surfacing: postgres silently turns the outermost commit into a
+ ;; rollback, so this is the only signal that the tree's writes and after-commit callbacks are about to
+ ;; diverge.
+ (log/warnf "Failed to release savepoint: %s" (ex-message e)))))))))
+
(def ^:dynamic *transaction-state*
"When non-nil, an atom holding a map of arbitrary per-transaction data, shared by the whole
nested-transaction tree and thrown away when the outermost transaction ends. Any subsystem can stash
@@ -268,7 +319,7 @@
(log/warnf "Failed to roll back transaction: %s"
(ex-message rollback-e)))))]
(letfn [(thunk []
- (let [savepoint (.setSavepoint connection)
+ (let [savepoint (set-savepoint! connection)
before-count (some-> *before-commit-callbacks* deref count)
after-count (some-> *after-commit-callbacks* deref count)
state-snapshot (when (and *transaction-state* (> *transaction-depth* 1))
@@ -280,7 +331,7 @@
(when before-count (discard-callbacks-after! *before-commit-callbacks* before-count))
;; Restore the state even if rollback throws. An outer scope must not see state from this one.
(try
- (.rollback connection savepoint)
+ (rollback-to-savepoint! connection savepoint)
(catch Throwable rollback-e
;; The writes remain pending. No enclosing scope may commit them.
;; A vanished savepoint still counts as a rollback failure. DDL committing implicitly,
@@ -366,7 +417,13 @@
(rollback-after-error! txn-e)))
:else
- [result false])))))]
+ (do
+ ;; The nested scope succeeded, so release its savepoint rather than leave it open until the
+ ;; outermost commit destroys it. On postgres each open savepoint is a live subtransaction that
+ ;; every row-visibility check walks, so a long pipeline of nested transactions slows to a
+ ;; crawl. Releasing is not committing: an enclosing rollback still undoes the released work.
+ (release-finished-savepoints! connection savepoint)
+ [result false]))))))]
;; Avoid toggling autocommit when the connection is already in a transaction.
(if (.getAutoCommit connection)
(try
@@ -455,6 +512,7 @@
*before-commit-callbacks* (if outermost? (atom []) *before-commit-callbacks*)
*transaction-state* (if outermost? (atom {}) *transaction-state*)
*rollback-required* (if outermost? (atom false) *rollback-required*)
+ *open-savepoints* (if outermost? (atom []) *open-savepoints*)
*after-commit-callbacks* callbacks]
(do-transaction connection rollback-only f))]
(when (and outermost? committed?)
diff --git a/test/metabase/app_db/connection_test.clj b/test/metabase/app_db/connection_test.clj
index 22934575314..1e8411f8301 100644
--- a/test/metabase/app_db/connection_test.clj
+++ b/test/metabase/app_db/connection_test.clj
@@ -11,7 +11,7 @@
[toucan2.jdbc.options :as t2.jdbc.options])
(:import
(com.mchange.v2.c3p0 DataSources PoolBackedDataSource WrapperConnectionPoolDataSource)
- (java.sql Connection SQLException)
+ (java.sql Connection SQLException Savepoint)
(java.util.concurrent Semaphore)
(java.util.concurrent.locks ReentrantReadWriteLock)))
@@ -437,6 +437,143 @@
(is (= msg (ex-message e))))
(is (true? @autocommit-reset-called))))))
+(defn- recording-connection
+ "Mock connection recording savepoint, rollback, and commit calls in `calls`."
+ [calls]
+ (let [sp-count (atom 0)
+ auto? (atom true)]
+ (reify Connection
+ (setAutoCommit [_ v] (reset! auto? v))
+ (getAutoCommit [_] @auto?)
+ (setSavepoint [_]
+ (let [n (swap! sp-count inc)]
+ (swap! calls conj [:set n])
+ (reify Savepoint
+ (getSavepointId [_] n))))
+ (releaseSavepoint [_ savepoint]
+ (swap! calls conj [:release (.getSavepointId ^Savepoint savepoint)]))
+ (rollback [_ savepoint]
+ (swap! calls conj [:rollback (.getSavepointId ^Savepoint savepoint)]))
+ (rollback [_]
+ (swap! calls conj [:rollback-connection]))
+ (commit [_]
+ (swap! calls conj [:commit])))))
+
+(deftest nested-transaction-savepoint-lifecycle-test
+ (let [calls (atom [])]
+ (binding [t2.connection/*current-connectable* (recording-connection calls)]
+ (testing "a successful nested transaction releases its savepoint; the outermost commits without releasing"
+ (t2/with-transaction [_]
+ (t2/with-transaction [_]))
+ (is (= [[:set 1] [:set 2] [:release 2] [:commit]]
+ @calls))))
+ (let [calls (atom [])]
+ (binding [t2.connection/*current-connectable* (recording-connection calls)]
+ (testing "a failed nested transaction rolls back to its savepoint without releasing it"
+ (t2/with-transaction [_]
+ (is (thrown-with-msg?
+ Exception #"boom"
+ (t2/with-transaction [_]
+ (throw (ex-info "boom" {}))))))
+ (is (= [[:set 1] [:set 2] [:rollback 2] [:commit]]
+ @calls)))))
+ (let [calls (atom [])]
+ (binding [t2.connection/*current-connectable* (recording-connection calls)]
+ (testing "deeper nesting releases innermost first"
+ (t2/with-transaction [_]
+ (t2/with-transaction [_]
+ (t2/with-transaction [_])))
+ (is (= [[:set 1] [:set 2] [:set 3] [:release 3] [:release 2] [:commit]]
+ @calls)))))))
+
+(defn- await-step
+ [p]
+ (let [v (deref p 10000 ::timeout)]
+ (is (not= ::timeout v) "sibling thread did not reach the expected step")
+ v))
+
+;; A `future` started inside a transaction inherits its bindings and runs nested scopes on the same connection.
+(deftest concurrent-nested-transaction-savepoint-test
+ (testing "a scope that finishes while a later sibling is still running leaves its savepoint until the sibling finishes"
+ (let [calls (atom [])
+ t1-in (promise)
+ t2-in (promise)
+ t1-out (promise)]
+ (binding [t2.connection/*current-connectable* (recording-connection calls)]
+ (t2/with-transaction [_]
+ (let [t1 (future
+ (t2/with-transaction [_]
+ (deliver t1-in true)
+ (await-step t2-in))
+ (deliver t1-out true))
+ t2 (future
+ (await-step t1-in)
+ (t2/with-transaction [_]
+ (deliver t2-in true)
+ (await-step t1-out)))]
+ (await-step t1)
+ (await-step t2))))
+ (is (= [[:set 1] [:set 2] [:set 3] [:release 2] [:commit]]
+ @calls))))
+ (testing "a sibling that rolls back destroys later savepoints, so their scopes no longer try to release them"
+ (let [calls (atom [])
+ t1-in (promise)
+ t2-in (promise)
+ t1-out (promise)]
+ (binding [t2.connection/*current-connectable* (recording-connection calls)]
+ (t2/with-transaction [_]
+ (let [t1 (future
+ (try
+ (t2/with-transaction [_]
+ (deliver t1-in true)
+ (await-step t2-in)
+ (throw (ex-info "boom" {})))
+ (catch Exception _)
+ (finally
+ (deliver t1-out true))))
+ t2 (future
+ (await-step t1-in)
+ (t2/with-transaction [_]
+ (deliver t2-in true)
+ (await-step t1-out)))]
+ (await-step t1)
+ (await-step t2))))
+ (is (= [[:set 1] [:set 2] [:set 3] [:rollback 2] [:commit]]
+ @calls)))))
+
+(deftest released-savepoint-preserves-rollback-semantics-test
+ (let [user-1 (mt/random-email)
+ user-2 (mt/random-email)
+ user-exists? (fn [email] (t2/exists? :model/User :email email))
+ create-user! (fn [email] (t2/insert! :model/User (assoc (mt/with-temp-defaults :model/User) :email email)))
+ ;; toucan2 rebuilds ExceptionInfos to attach its context trace (preserving ex-data but not identity
+ ;; or the cause chain), so detect our abort by an ex-data marker rather than identity
+ outer-abort (ex-info "(Abort the outer transaction)" {::outer-abort true})
+ outer-abort? (fn outer-abort? [e]
+ (boolean (or (::outer-abort (ex-data e))
+ (some-> (ex-cause e) outer-abort?))))]
+ (try
+ (t2/with-transaction [conn]
+ (t2/with-transaction [_ conn]
+ (create-user! user-1))
+ (testing "work from a successful nested transaction is visible in the outer transaction after release"
+ (is (user-exists? user-1)))
+ (is (thrown-with-msg?
+ Exception #"Abort the sibling"
+ (t2/with-transaction [_ conn]
+ (create-user! user-2)
+ (throw (ex-info "(Abort the sibling)" {})))))
+ (testing "a later sibling nested transaction rolls back only its own work"
+ (is (user-exists? user-1))
+ (is (not (user-exists? user-2))))
+ (throw outer-abort))
+ (catch Exception e
+ (when-not (outer-abort? e)
+ (throw e))))
+ (testing "rolling back the outer transaction undoes work from a released nested transaction"
+ (is (not (user-exists? user-1)))
+ (is (not (user-exists? user-2))))))
+
;;; ------------------------------ before-commit + transaction-state ------------------------------
;;; The mq transactional outbox relies on this machinery: it inserts its rows from a before-commit
;;; callback (so they commit atomically with the business transaction) and stashes the ids in