The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Handle callers that outlive their transaction (#84036)
Chris Truter31cf56c88820
diff --git a/src/metabase/app_db/connection.clj b/src/metabase/app_db/connection.cljindex 7833f0665f0..a61590945b9 100644--- a/src/metabase/app_db/connection.clj+++ b/src/metabase/app_db/connection.clj@@ -172,19 +172,43 @@#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}(def ^:private ^:dynamic *transaction-depth* 0)-(defn in-transaction?- "Whether we are currently in a transaction."- []- (pos? *transaction-depth*))--;; Accumulate 0-arity thunks to run just before / just after the outermost transaction commits. Each is-;; bound to a fresh atom when the outermost transaction starts (see [[do-with-transaction]]) and shared by-;; the whole nested-transaction tree; nil outside any transaction.+;; Thunks to run just before and just after the outermost transaction commits.+;; Each var is bound to a fresh atom when the outermost transaction starts, see [[do-with-transaction]].+;; The whole nested-transaction tree shares that atom, and both vars are nil outside any transaction.+;; When the transaction ends, the after-commit atom holds `::committed` or `::rolled-back` in place of its thunks.+;; Read its value through [[pending-callbacks]].#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}(def ^:private ^:dynamic *before-commit-callbacks* nil)#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}(def ^:private ^:dynamic *after-commit-callbacks* nil)+(defn- pending-callbacks+ "Return `cbs`, the value of the after-commit callbacks atom, if they are still waiting to run, else nil.+ Once the transaction has ended the atom holds `::committed` or `::rolled-back`, so read its value through this."+ [cbs]+ (when (vector? cbs)+ cbs))++(defn- inherited-transaction-ended?+ "Whether this thread still has the bindings of a transaction that has since ended."+ []+ (when-let [callbacks *after-commit-callbacks*]+ (nil? (pending-callbacks @callbacks))))++(defn- live-transaction-depth+ "Return the depth of the transaction this thread is in.+ A thread that inherited a transaction which has since ended is not in one, so its depth is 0."+ []+ (if (inherited-transaction-ended?)+ 0+ *transaction-depth*))++(defn in-transaction?+ "Whether we are currently in a transaction.+ A thread that outlived the transaction it started in is not."+ []+ (pos? (live-transaction-depth)))+;; Holds an atom set to true when a rollback fails and leaves behind writes that should have been discarded. The atom;; is shared across the transaction tree so that the outermost scope cannot commit those writes, even if an;; intermediate scope catches the rollback error.@@ -258,7 +282,26 @@(defn transaction-state"Returns the current per-transaction [[*transaction-state*]] atom, or nil if not in a transaction."[]- *transaction-state*)+ (when-not (inherited-transaction-ended?)+ *transaction-state*))++(defn- call-outside-transaction+ "Call `thunk` with every transaction-scoped binding cleared, and return its result."+ [thunk]+ ;; Bind the transaction connection and callback accumulator to nil so they are not conveyed into async work+ ;; (e.g. a reconcile `future`) a callback may start: that work must acquire its own connection rather than+ ;; reuse this transaction's connection after it returns to the pool, and a do-after-commit it makes must run+ ;; immediately rather than enqueue into this now-drained accumulator.+ ;; A thunk registered late runs on a thread that still has every binding of the finished transaction, so clear+ ;; the rest too: its before-commit callbacks have also run, and its state must not be mistaken for a live one.+ (binding [t2.conn/*current-connectable* nil+ *transaction-depth* 0+ *after-commit-callbacks* nil+ *before-commit-callbacks* nil+ *transaction-state* nil+ *rollback-required* nil+ *open-savepoints* nil]+ (thunk)))(defn do-before-commit"Run `thunk` just before the current outermost transaction commits — while the transaction is still@@ -266,9 +309,19 @@transaction back. Outside a transaction, runs `thunk` immediately. Mirror of [[do-after-commit]] forwork that must land *inside* the committing transaction."[thunk]- (if-let [callbacks *before-commit-callbacks*]- (do (swap! callbacks conj thunk) nil)- (thunk)))+ (cond+ ;; A thread that outlived its transaction still has the callbacks bound, but they have already run.+ ;; Its connection has gone too, so the thunk must find its own.+ (inherited-transaction-ended?) (call-outside-transaction thunk)+ *before-commit-callbacks* (do (swap! *before-commit-callbacks* conj thunk) nil)+ :else (thunk)))++(defn- run-after-commit-callback! [thunk]+ ;; the transaction already committed; a failing callback must not unwind it+ (try+ (call-outside-transaction thunk)+ (catch Throwable t+ (log/errorf "after-commit callback failed: %s" (ex-message t)))))(defn do-after-commit"Run `thunk` after the current outermost transaction commits successfully — never on rollback.@@ -277,24 +330,21 @@observe committed state (e.g. a reconcile that reads the row).Do not do synchronous DB I/O in `thunk`: it runs while the transaction's connection is still checked out,so a query here would hold a second connection and can deadlock a saturated pool. Hand DB work to the- async job you schedule, which acquires its own connection."+ async job you schedule, which acquires its own connection.+ A thread started inside the transaction can call this after the commit, and `thunk` then runs immediately too.+ An exception from `thunk` is logged, not thrown, whenever a transaction was involved."[thunk](if-let [callbacks *after-commit-callbacks*]- (do (swap! callbacks conj thunk) nil)+ ;; A thread started inside the transaction keeps this binding after the transaction ends.+ ;; Once the callbacks have run, a thunk added to them would never run.+ (let [[before] (swap-vals! callbacks #(cond-> % (pending-callbacks %) (conj thunk)))]+ (when (= ::committed before)+ (run-after-commit-callback! thunk))+ nil)(thunk)))(defn- run-after-commit-callbacks! [callbacks]- ;; Bind the transaction connection and callback accumulator to nil so they are not conveyed into async work- ;; (e.g. a reconcile `future`) a callback may start: that work must acquire its own connection rather than- ;; reuse this transaction's connection after it returns to the pool, and a do-after-commit it makes must run- ;; immediately rather than enqueue into this now-drained accumulator.- (binding [t2.conn/*current-connectable* nil- *transaction-depth* 0- *after-commit-callbacks* nil]- (doseq [thunk @callbacks]- ;; the transaction already committed; a failing callback must not unwind it- (try (thunk) (catch Throwable t (log/errorf "after-commit callback failed: %s" (ex-message t)))))- (reset! callbacks [])))+ (run! run-after-commit-callback! (first (reset-vals! callbacks ::committed))))(defn- discard-callbacks-after!"Truncate the `callbacks` atom back to its first `n` entries, dropping any that a now-rolling-back@@ -305,7 +355,9 @@(fn [cbs];; copy rather than return the subvec view, which would retain the discarded callbacks (and their;; captured closures) through the backing array until the outer transaction finishes- (into [] (subvec cbs 0 (min n (count cbs))))))))+ (if (pending-callbacks cbs)+ (into [] (subvec cbs 0 (min n (count cbs))))+ cbs)))))(defn- do-transaction [^java.sql.Connection connection rollback-only? f];; Set when a connection rollback fails and leaves pending writes. Restoring autocommit would commit those writes,@@ -328,7 +380,7 @@(letfn [(thunk [](let [savepoint (set-savepoint! connection)before-count (some-> *before-commit-callbacks* deref count)- after-count (some-> *after-commit-callbacks* deref count)+ after-count (some-> *after-commit-callbacks* deref pending-callbacks count)state-snapshot (when (and *transaction-state* (> *transaction-depth* 1))@*transaction-state*)](letfn [(rollback! []@@ -500,31 +552,41 @@(throw (ex-info (str "Cannot combine :rollback-only with :nested-transaction-rule :ignore -- an ignored ""nested transaction has no savepoint to roll back to"){:options options})))- (cond- (and (pos? *transaction-depth*)- (= nested-transaction-rule :ignore))- (f connection)-- (and (pos? *transaction-depth*)- (= nested-transaction-rule :prohibit))- (throw (ex-info "Attempted to create nested transaction with :nested-transaction-rule set to :prohibit"- {:options options}))-- :else- (let [outermost? (zero? *transaction-depth*)- callbacks (if outermost? (atom []) *after-commit-callbacks*)- [result committed?]- (binding [*transaction-depth* (inc *transaction-depth*)- ;; Create one set of callback accumulators and transaction state for the entire tree.- *before-commit-callbacks* (if outermost? (atom []) *before-commit-callbacks*)- *transaction-state* (if outermost? (atom {}) *transaction-state*)- *rollback-required* (if outermost? (atom false) *rollback-required*)- *open-savepoints* (if outermost? (atom []) *open-savepoints*)- *after-commit-callbacks* callbacks]- (do-transaction connection rollback-only f))]- (when (and outermost? committed?)- (run-after-commit-callbacks! callbacks))- result)))+ ;; Read the depth once: the outer transaction can commit on another thread between two reads.+ (let [depth (live-transaction-depth)]+ (cond+ (and (pos? depth)+ (= nested-transaction-rule :ignore))+ (f connection)++ (and (pos? depth)+ (= nested-transaction-rule :prohibit))+ (throw (ex-info "Attempted to create nested transaction with :nested-transaction-rule set to :prohibit"+ {:options options}))++ :else+ (let [outermost? (zero? depth)+ callbacks (if outermost? (atom []) *after-commit-callbacks*)+ [result committed?]+ (try+ (binding [*transaction-depth* (inc depth)+ ;; Create one set of callback accumulators and transaction state for the entire tree.+ *before-commit-callbacks* (if outermost? (atom []) *before-commit-callbacks*)+ *transaction-state* (if outermost? (atom {}) *transaction-state*)+ *rollback-required* (if outermost? (atom false) *rollback-required*)+ *open-savepoints* (if outermost? (atom []) *open-savepoints*)+ *after-commit-callbacks* callbacks]+ (do-transaction connection rollback-only f))+ (catch Throwable e+ (when outermost?+ (reset! callbacks ::rolled-back))+ (throw e)))]+ ;; Mark the end either way, so a thread that inherited this transaction can tell it is over.+ (when outermost?+ (if committed?+ (run-after-commit-callbacks! callbacks)+ (reset! callbacks ::rolled-back)))+ result))));;;; Unshared connections;;;;diff --git a/src/metabase/search/core.clj b/src/metabase/search/core.cljindex bff0a5019f6..5c059444090 100644--- a/src/metabase/search/core.clj+++ b/src/metabase/search/core.clj@@ -244,7 +244,6 @@(when-let [updates (->> (search.spec/search-models-to-update instance always?)(remove (comp search.util/impossible-condition? second))seq)]- ;; We need to delay execution to handle deletes, which alert us *before* updating the database.(search.ingestion/ingest-maybe-async! updates))))(defn delete!diff --git a/test/metabase/app_db/connection_test.clj b/test/metabase/app_db/connection_test.cljindex 1e8411f8301..15aa5848b24 100644--- a/test/metabase/app_db/connection_test.clj+++ b/test/metabase/app_db/connection_test.clj@@ -215,6 +215,129 @@(mdb.connection/do-after-commit (fn [] (swap! calls conj :b))))(is (= [:a :nested-from-a :b] @calls))))+(deftest after-commit-callback-registered-after-the-commit-runs-immediately-test+ ;; A thread started inside a transaction keeps the transaction's bindings after it ends, so its do-after-commit+ ;; still finds that transaction's callbacks. They have already run, and a thunk added to them would never run.+ (let [committed (promise)+ callback-state (promise)]+ (t2/with-transaction [_conn]+ (future+ @committed+ (mdb.connection/do-after-commit+ #(deliver callback-state+ {:current-connectable t2.connection/*current-connectable*+ :in-transaction? (mdb.connection/in-transaction?)+ :transaction-state (mdb.connection/transaction-state)+ :before-commit-ran? (let [ran? (atom false)]+ (mdb.connection/do-before-commit (fn [] (reset! ran? true)))+ @ran?)}))))+ (deliver committed true)+ (is (= {:current-connectable nil+ :in-transaction? false+ :transaction-state nil+ :before-commit-ran? true}+ (deref callback-state 1000 ::timed-out)))))++(deftest after-commit-callback-registered-after-a-rollback-never-runs-test+ (let [rolled-back (promise)+ calls (atom [])+ worker (t2/with-transaction [_conn nil {:rollback-only true}]+ (future+ @rolled-back+ (mdb.connection/do-after-commit (fn [] (swap! calls conj :late)))+ :done))]+ (deliver rolled-back true)+ (is (= :done (deref worker 1000 ::timed-out)))+ (is (= [] @calls))))++(deftest after-commit-callback-registered-after-the-commit-does-not-throw-test+ ;; `bound-fn` keeps the transaction's bindings the way a thread started inside it would.+ (let [register-late (t2/with-transaction [_conn]+ (bound-fn []+ (mdb.connection/do-after-commit #(throw (ex-info "Callback error" {})))))]+ (is (nil? (register-late)))))++(defn- do-transaction-ending+ "Call `f` in a transaction on `connectable` that commits, or rolls back if `rollback?`, and return its result."+ [connectable rollback? f]+ (if rollback?+ (t2/with-transaction [_conn connectable {:rollback-only true}]+ (f))+ (t2/with-transaction [_conn connectable]+ (f))))++(deftest transaction-opened-after-the-outer-one-ends-is-its-own-transaction-test+ ;; A caller that kept the bindings of a finished transaction is no longer inside it.+ ;; A transaction it opens must commit and roll back on its own, not as a nested scope of the finished one.+ ;; The enclosing `with-connection` keeps the connection open after the first transaction ends.+ (letfn [(late-transaction-events [outer-rolls-back? body]+ (let [events (atom [])]+ (t2/with-connection [conn]+ (let [open-late (do-transaction-ending+ conn outer-rolls-back?+ (fn []+ (bound-fn []+ (try+ (t2/with-transaction [_conn]+ (mdb.connection/do-after-commit #(swap! events conj :callback))+ (swap! events conj :body)+ (body))+ (catch Exception _+ (swap! events conj :rolled-back))))))]+ (open-late)))+ @events))]+ (doseq [[outer-end outer-rolls-back?] {"committed" false, "rolled back" true}]+ (testing (str "after the outer transaction " outer-end)+ (testing "its after-commit callback waits for its own commit"+ (is (= [:body :callback]+ (late-transaction-events outer-rolls-back? (fn [])))))+ (testing "its after-commit callback never runs if it rolls back"+ (is (= [:body :rolled-back]+ (late-transaction-events outer-rolls-back? #(throw (ex-info "Roll back" {}))))))))))++(deftest caller-that-outlived-its-transaction-is-outside-it-test+ (letfn [(transaction-view []+ {:in-transaction? (mdb.connection/in-transaction?)+ :transaction-state? (some? (mdb.connection/transaction-state))+ :before-commit (let [outcome (atom :deferred)]+ (mdb.connection/do-before-commit+ #(reset! outcome (if t2.connection/*current-connectable*+ :ran-on-the-old-connection+ :ran-outside-a-transaction)))+ @outcome)})]+ (doseq [[outer-end rolls-back?] {"committed" false, "rolled back" true}]+ (testing (str "after the transaction " outer-end)+ (let [[inside view-afterwards] (do-transaction-ending+ nil rolls-back?+ (fn []+ [(transaction-view) (bound-fn [] (transaction-view))]))]+ (is (= {:inside {:in-transaction? true, :transaction-state? true, :before-commit :deferred}+ :afterwards {:in-transaction? false+ :transaction-state? false+ :before-commit :ran-outside-a-transaction}}+ {:inside inside+ :afterwards (view-afterwards)})))))))++(deftest nested-transaction-rolling-back-after-the-outer-commit-test+ ;; The nested scope opens while there are still callbacks to count, and rolls back after they have run.+ ;; The outer commit has destroyed its savepoint, so the rollback fails. That database error is the one to report.+ (let [nested-open (promise)+ committed (promise)]+ (t2/with-connection [conn]+ (let [worker (t2/with-transaction [_conn conn]+ (let [nested (future+ (try+ (t2/with-transaction [_conn]+ (deliver nested-open true)+ @committed+ (throw (ex-info "Roll back nested" {})))+ (catch Exception e+ (:rollback-error (ex-data e)))))]+ (deref nested-open 1000 ::timed-out)+ nested))]+ (deliver committed true)+ (is (instance? SQLException (deref worker 1000 ::timed-out)))))))+(deftest rollback-only-transaction-rolls-back-and-discards-callbacks-test(let [email (mt/random-email)calls (atom [])]