Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Handle callers that outlive their transaction (#84036)

Chris Truter31cf56c88820
diff --git a/src/metabase/app_db/connection.clj b/src/metabase/app_db/connection.clj
index 7833f0665f0..a61590945b9 100644
--- a/src/metabase/app_db/connection.clj
+++ b/src/metabase/app_db/connection.clj
@@ -172,19 +172,43 @@
#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}
(def ^:private ^:dynamic *transaction-depth* 0)
-(defn in-transaction?
- "Whether we are currently in a transaction."
- []
- (pos? *transaction-depth*))
-
-;; Accumulate 0-arity thunks to run just before / just after the outermost transaction commits. Each is
-;; bound to a fresh atom when the outermost transaction starts (see [[do-with-transaction]]) and shared by
-;; the whole nested-transaction tree; nil outside any transaction.
+;; Thunks to run just before and just after the outermost transaction commits.
+;; Each var is bound to a fresh atom when the outermost transaction starts, see [[do-with-transaction]].
+;; The whole nested-transaction tree shares that atom, and both vars are nil outside any transaction.
+;; When the transaction ends, the after-commit atom holds `::committed` or `::rolled-back` in place of its thunks.
+;; Read its value through [[pending-callbacks]].
#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}
(def ^:private ^:dynamic *before-commit-callbacks* nil)
#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}
(def ^:private ^:dynamic *after-commit-callbacks* nil)
+(defn- pending-callbacks
+ "Return `cbs`, the value of the after-commit callbacks atom, if they are still waiting to run, else nil.
+ Once the transaction has ended the atom holds `::committed` or `::rolled-back`, so read its value through this."
+ [cbs]
+ (when (vector? cbs)
+ cbs))
+
+(defn- inherited-transaction-ended?
+ "Whether this thread still has the bindings of a transaction that has since ended."
+ []
+ (when-let [callbacks *after-commit-callbacks*]
+ (nil? (pending-callbacks @callbacks))))
+
+(defn- live-transaction-depth
+ "Return the depth of the transaction this thread is in.
+ A thread that inherited a transaction which has since ended is not in one, so its depth is 0."
+ []
+ (if (inherited-transaction-ended?)
+ 0
+ *transaction-depth*))
+
+(defn in-transaction?
+ "Whether we are currently in a transaction.
+ A thread that outlived the transaction it started in is not."
+ []
+ (pos? (live-transaction-depth)))
+
;; Holds an atom set to true when a rollback fails and leaves behind writes that should have been discarded. The atom
;; is shared across the transaction tree so that the outermost scope cannot commit those writes, even if an
;; intermediate scope catches the rollback error.
@@ -258,7 +282,26 @@
(defn transaction-state
"Returns the current per-transaction [[*transaction-state*]] atom, or nil if not in a transaction."
[]
- *transaction-state*)
+ (when-not (inherited-transaction-ended?)
+ *transaction-state*))
+
+(defn- call-outside-transaction
+ "Call `thunk` with every transaction-scoped binding cleared, and return its result."
+ [thunk]
+ ;; Bind the transaction connection and callback accumulator to nil so they are not conveyed into async work
+ ;; (e.g. a reconcile `future`) a callback may start: that work must acquire its own connection rather than
+ ;; reuse this transaction's connection after it returns to the pool, and a do-after-commit it makes must run
+ ;; immediately rather than enqueue into this now-drained accumulator.
+ ;; A thunk registered late runs on a thread that still has every binding of the finished transaction, so clear
+ ;; the rest too: its before-commit callbacks have also run, and its state must not be mistaken for a live one.
+ (binding [t2.conn/*current-connectable* nil
+ *transaction-depth* 0
+ *after-commit-callbacks* nil
+ *before-commit-callbacks* nil
+ *transaction-state* nil
+ *rollback-required* nil
+ *open-savepoints* nil]
+ (thunk)))
(defn do-before-commit
"Run `thunk` just before the current outermost transaction commits — while the transaction is still
@@ -266,9 +309,19 @@
transaction back. Outside a transaction, runs `thunk` immediately. Mirror of [[do-after-commit]] for
work that must land *inside* the committing transaction."
[thunk]
- (if-let [callbacks *before-commit-callbacks*]
- (do (swap! callbacks conj thunk) nil)
- (thunk)))
+ (cond
+ ;; A thread that outlived its transaction still has the callbacks bound, but they have already run.
+ ;; Its connection has gone too, so the thunk must find its own.
+ (inherited-transaction-ended?) (call-outside-transaction thunk)
+ *before-commit-callbacks* (do (swap! *before-commit-callbacks* conj thunk) nil)
+ :else (thunk)))
+
+(defn- run-after-commit-callback! [thunk]
+ ;; the transaction already committed; a failing callback must not unwind it
+ (try
+ (call-outside-transaction thunk)
+ (catch Throwable t
+ (log/errorf "after-commit callback failed: %s" (ex-message t)))))
(defn do-after-commit
"Run `thunk` after the current outermost transaction commits successfully — never on rollback.
@@ -277,24 +330,21 @@
observe committed state (e.g. a reconcile that reads the row).
Do not do synchronous DB I/O in `thunk`: it runs while the transaction's connection is still checked out,
so a query here would hold a second connection and can deadlock a saturated pool. Hand DB work to the
- async job you schedule, which acquires its own connection."
+ async job you schedule, which acquires its own connection.
+ A thread started inside the transaction can call this after the commit, and `thunk` then runs immediately too.
+ An exception from `thunk` is logged, not thrown, whenever a transaction was involved."
[thunk]
(if-let [callbacks *after-commit-callbacks*]
- (do (swap! callbacks conj thunk) nil)
+ ;; A thread started inside the transaction keeps this binding after the transaction ends.
+ ;; Once the callbacks have run, a thunk added to them would never run.
+ (let [[before] (swap-vals! callbacks #(cond-> % (pending-callbacks %) (conj thunk)))]
+ (when (= ::committed before)
+ (run-after-commit-callback! thunk))
+ nil)
(thunk)))
(defn- run-after-commit-callbacks! [callbacks]
- ;; Bind the transaction connection and callback accumulator to nil so they are not conveyed into async work
- ;; (e.g. a reconcile `future`) a callback may start: that work must acquire its own connection rather than
- ;; reuse this transaction's connection after it returns to the pool, and a do-after-commit it makes must run
- ;; immediately rather than enqueue into this now-drained accumulator.
- (binding [t2.conn/*current-connectable* nil
- *transaction-depth* 0
- *after-commit-callbacks* nil]
- (doseq [thunk @callbacks]
- ;; the transaction already committed; a failing callback must not unwind it
- (try (thunk) (catch Throwable t (log/errorf "after-commit callback failed: %s" (ex-message t)))))
- (reset! callbacks [])))
+ (run! run-after-commit-callback! (first (reset-vals! callbacks ::committed))))
(defn- discard-callbacks-after!
"Truncate the `callbacks` atom back to its first `n` entries, dropping any that a now-rolling-back
@@ -305,7 +355,9 @@
(fn [cbs]
;; copy rather than return the subvec view, which would retain the discarded callbacks (and their
;; captured closures) through the backing array until the outer transaction finishes
- (into [] (subvec cbs 0 (min n (count cbs))))))))
+ (if (pending-callbacks cbs)
+ (into [] (subvec cbs 0 (min n (count cbs))))
+ cbs)))))
(defn- do-transaction [^java.sql.Connection connection rollback-only? f]
;; Set when a connection rollback fails and leaves pending writes. Restoring autocommit would commit those writes,
@@ -328,7 +380,7 @@
(letfn [(thunk []
(let [savepoint (set-savepoint! connection)
before-count (some-> *before-commit-callbacks* deref count)
- after-count (some-> *after-commit-callbacks* deref count)
+ after-count (some-> *after-commit-callbacks* deref pending-callbacks count)
state-snapshot (when (and *transaction-state* (> *transaction-depth* 1))
@*transaction-state*)]
(letfn [(rollback! []
@@ -500,31 +552,41 @@
(throw (ex-info (str "Cannot combine :rollback-only with :nested-transaction-rule :ignore -- an ignored "
"nested transaction has no savepoint to roll back to")
{:options options})))
- (cond
- (and (pos? *transaction-depth*)
- (= nested-transaction-rule :ignore))
- (f connection)
-
- (and (pos? *transaction-depth*)
- (= nested-transaction-rule :prohibit))
- (throw (ex-info "Attempted to create nested transaction with :nested-transaction-rule set to :prohibit"
- {:options options}))
-
- :else
- (let [outermost? (zero? *transaction-depth*)
- callbacks (if outermost? (atom []) *after-commit-callbacks*)
- [result committed?]
- (binding [*transaction-depth* (inc *transaction-depth*)
- ;; Create one set of callback accumulators and transaction state for the entire tree.
- *before-commit-callbacks* (if outermost? (atom []) *before-commit-callbacks*)
- *transaction-state* (if outermost? (atom {}) *transaction-state*)
- *rollback-required* (if outermost? (atom false) *rollback-required*)
- *open-savepoints* (if outermost? (atom []) *open-savepoints*)
- *after-commit-callbacks* callbacks]
- (do-transaction connection rollback-only f))]
- (when (and outermost? committed?)
- (run-after-commit-callbacks! callbacks))
- result)))
+ ;; Read the depth once: the outer transaction can commit on another thread between two reads.
+ (let [depth (live-transaction-depth)]
+ (cond
+ (and (pos? depth)
+ (= nested-transaction-rule :ignore))
+ (f connection)
+
+ (and (pos? depth)
+ (= nested-transaction-rule :prohibit))
+ (throw (ex-info "Attempted to create nested transaction with :nested-transaction-rule set to :prohibit"
+ {:options options}))
+
+ :else
+ (let [outermost? (zero? depth)
+ callbacks (if outermost? (atom []) *after-commit-callbacks*)
+ [result committed?]
+ (try
+ (binding [*transaction-depth* (inc depth)
+ ;; Create one set of callback accumulators and transaction state for the entire tree.
+ *before-commit-callbacks* (if outermost? (atom []) *before-commit-callbacks*)
+ *transaction-state* (if outermost? (atom {}) *transaction-state*)
+ *rollback-required* (if outermost? (atom false) *rollback-required*)
+ *open-savepoints* (if outermost? (atom []) *open-savepoints*)
+ *after-commit-callbacks* callbacks]
+ (do-transaction connection rollback-only f))
+ (catch Throwable e
+ (when outermost?
+ (reset! callbacks ::rolled-back))
+ (throw e)))]
+ ;; Mark the end either way, so a thread that inherited this transaction can tell it is over.
+ (when outermost?
+ (if committed?
+ (run-after-commit-callbacks! callbacks)
+ (reset! callbacks ::rolled-back)))
+ result))))
;;;; Unshared connections
;;;;
diff --git a/src/metabase/search/core.clj b/src/metabase/search/core.clj
index bff0a5019f6..5c059444090 100644
--- a/src/metabase/search/core.clj
+++ b/src/metabase/search/core.clj
@@ -244,7 +244,6 @@
(when-let [updates (->> (search.spec/search-models-to-update instance always?)
(remove (comp search.util/impossible-condition? second))
seq)]
- ;; We need to delay execution to handle deletes, which alert us *before* updating the database.
(search.ingestion/ingest-maybe-async! updates))))
(defn delete!
diff --git a/test/metabase/app_db/connection_test.clj b/test/metabase/app_db/connection_test.clj
index 1e8411f8301..15aa5848b24 100644
--- a/test/metabase/app_db/connection_test.clj
+++ b/test/metabase/app_db/connection_test.clj
@@ -215,6 +215,129 @@
(mdb.connection/do-after-commit (fn [] (swap! calls conj :b))))
(is (= [:a :nested-from-a :b] @calls))))
+(deftest after-commit-callback-registered-after-the-commit-runs-immediately-test
+ ;; A thread started inside a transaction keeps the transaction's bindings after it ends, so its do-after-commit
+ ;; still finds that transaction's callbacks. They have already run, and a thunk added to them would never run.
+ (let [committed (promise)
+ callback-state (promise)]
+ (t2/with-transaction [_conn]
+ (future
+ @committed
+ (mdb.connection/do-after-commit
+ #(deliver callback-state
+ {:current-connectable t2.connection/*current-connectable*
+ :in-transaction? (mdb.connection/in-transaction?)
+ :transaction-state (mdb.connection/transaction-state)
+ :before-commit-ran? (let [ran? (atom false)]
+ (mdb.connection/do-before-commit (fn [] (reset! ran? true)))
+ @ran?)}))))
+ (deliver committed true)
+ (is (= {:current-connectable nil
+ :in-transaction? false
+ :transaction-state nil
+ :before-commit-ran? true}
+ (deref callback-state 1000 ::timed-out)))))
+
+(deftest after-commit-callback-registered-after-a-rollback-never-runs-test
+ (let [rolled-back (promise)
+ calls (atom [])
+ worker (t2/with-transaction [_conn nil {:rollback-only true}]
+ (future
+ @rolled-back
+ (mdb.connection/do-after-commit (fn [] (swap! calls conj :late)))
+ :done))]
+ (deliver rolled-back true)
+ (is (= :done (deref worker 1000 ::timed-out)))
+ (is (= [] @calls))))
+
+(deftest after-commit-callback-registered-after-the-commit-does-not-throw-test
+ ;; `bound-fn` keeps the transaction's bindings the way a thread started inside it would.
+ (let [register-late (t2/with-transaction [_conn]
+ (bound-fn []
+ (mdb.connection/do-after-commit #(throw (ex-info "Callback error" {})))))]
+ (is (nil? (register-late)))))
+
+(defn- do-transaction-ending
+ "Call `f` in a transaction on `connectable` that commits, or rolls back if `rollback?`, and return its result."
+ [connectable rollback? f]
+ (if rollback?
+ (t2/with-transaction [_conn connectable {:rollback-only true}]
+ (f))
+ (t2/with-transaction [_conn connectable]
+ (f))))
+
+(deftest transaction-opened-after-the-outer-one-ends-is-its-own-transaction-test
+ ;; A caller that kept the bindings of a finished transaction is no longer inside it.
+ ;; A transaction it opens must commit and roll back on its own, not as a nested scope of the finished one.
+ ;; The enclosing `with-connection` keeps the connection open after the first transaction ends.
+ (letfn [(late-transaction-events [outer-rolls-back? body]
+ (let [events (atom [])]
+ (t2/with-connection [conn]
+ (let [open-late (do-transaction-ending
+ conn outer-rolls-back?
+ (fn []
+ (bound-fn []
+ (try
+ (t2/with-transaction [_conn]
+ (mdb.connection/do-after-commit #(swap! events conj :callback))
+ (swap! events conj :body)
+ (body))
+ (catch Exception _
+ (swap! events conj :rolled-back))))))]
+ (open-late)))
+ @events))]
+ (doseq [[outer-end outer-rolls-back?] {"committed" false, "rolled back" true}]
+ (testing (str "after the outer transaction " outer-end)
+ (testing "its after-commit callback waits for its own commit"
+ (is (= [:body :callback]
+ (late-transaction-events outer-rolls-back? (fn [])))))
+ (testing "its after-commit callback never runs if it rolls back"
+ (is (= [:body :rolled-back]
+ (late-transaction-events outer-rolls-back? #(throw (ex-info "Roll back" {}))))))))))
+
+(deftest caller-that-outlived-its-transaction-is-outside-it-test
+ (letfn [(transaction-view []
+ {:in-transaction? (mdb.connection/in-transaction?)
+ :transaction-state? (some? (mdb.connection/transaction-state))
+ :before-commit (let [outcome (atom :deferred)]
+ (mdb.connection/do-before-commit
+ #(reset! outcome (if t2.connection/*current-connectable*
+ :ran-on-the-old-connection
+ :ran-outside-a-transaction)))
+ @outcome)})]
+ (doseq [[outer-end rolls-back?] {"committed" false, "rolled back" true}]
+ (testing (str "after the transaction " outer-end)
+ (let [[inside view-afterwards] (do-transaction-ending
+ nil rolls-back?
+ (fn []
+ [(transaction-view) (bound-fn [] (transaction-view))]))]
+ (is (= {:inside {:in-transaction? true, :transaction-state? true, :before-commit :deferred}
+ :afterwards {:in-transaction? false
+ :transaction-state? false
+ :before-commit :ran-outside-a-transaction}}
+ {:inside inside
+ :afterwards (view-afterwards)})))))))
+
+(deftest nested-transaction-rolling-back-after-the-outer-commit-test
+ ;; The nested scope opens while there are still callbacks to count, and rolls back after they have run.
+ ;; The outer commit has destroyed its savepoint, so the rollback fails. That database error is the one to report.
+ (let [nested-open (promise)
+ committed (promise)]
+ (t2/with-connection [conn]
+ (let [worker (t2/with-transaction [_conn conn]
+ (let [nested (future
+ (try
+ (t2/with-transaction [_conn]
+ (deliver nested-open true)
+ @committed
+ (throw (ex-info "Roll back nested" {})))
+ (catch Exception e
+ (:rollback-error (ex-data e)))))]
+ (deref nested-open 1000 ::timed-out)
+ nested))]
+ (deliver committed true)
+ (is (instance? SQLException (deref worker 1000 ::timed-out)))))))
+
(deftest rollback-only-transaction-rolls-back-and-discards-callbacks-test
(let [email (mt/random-email)
calls (atom [])]