Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Unshared appdb connections + detached audit cluster lock (fix first-boot stall) (#78276)

dpsutton0ed95e05f66e
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn
index bc4ebdbcb70..6c1f7a52e9b 100644
--- a/.clj-kondo/config.edn
+++ b/.clj-kondo/config.edn
@@ -467,6 +467,8 @@
metabase.actions.test-util/with-actions clojure.core/let
metabase.transforms.test-util/with-transform-cleanup! clojure.core/let
metabase.api.common/let-404 clojure.core/let
+ metabase.app-db.connection/with-unshared-connection clojure.core/fn
+ metabase.app-db.core/with-unshared-connection clojure.core/fn
metabase.app-db.custom-migrations.util/with-temp-scheduler! clojure.core/fn
metabase.app-db.custom-migrations/define-migration clj-kondo.lint-as/def-catch-all
metabase.app-db.custom-migrations/define-reversible-migration clj-kondo.lint-as/def-catch-all
diff --git a/deps.edn b/deps.edn
index 154b1cbf18f..b4ca745dbfd 100644
--- a/deps.edn
+++ b/deps.edn
@@ -80,7 +80,7 @@
hiccup/hiccup {:mvn/version "2.0.0"} ; HTML templating
inflections/inflections {:mvn/version "0.15.0"} ; Clojure/Script library used for prularizing words
instaparse/instaparse {:mvn/version "1.5.0"} ; Make your own parser
- io.github.camsaul/toucan2 {:mvn/version "1.0.571"}
+ io.github.camsaul/toucan2 {:mvn/version "1.0.572"}
io.github.eerohele/pp {#_#_:git/tag "2024-11-13.77" ; super fast pretty-printing library
:git/sha "ffc4edc482c057cd9412c62f018a41e9140c618b"
:git/url "https://github.com/eerohele/pp"}
diff --git a/enterprise/backend/src/metabase_enterprise/audit_app/audit.clj b/enterprise/backend/src/metabase_enterprise/audit_app/audit.clj
index 1a7ece59b34..f33602a116c 100644
--- a/enterprise/backend/src/metabase_enterprise/audit_app/audit.clj
+++ b/enterprise/backend/src/metabase_enterprise/audit_app/audit.clj
@@ -63,21 +63,28 @@
- This uses a weird ID because some tests were hardcoded to look for database with ID = 2, and inserting an extra db
throws that off since these IDs are sequential."
[engine id]
- (t2/insert! :model/Database {:is_audit true
- :id id
- :name default-db-name
- :description "Internal Audit DB used to power metabase analytics."
- :engine engine
- :is_full_sync true
- :is_on_demand false
- :creator_id nil
- :auto_run_queries true})
- ;; guard against someone manually deleting the audit-db entry, but not removing the audit-db permissions.
- (t2/delete! :model/Permissions {:where [:like :object (str "%/db/" id "/%")]}))
+ ;; one transaction so a crash can't commit the database row yet skip the permissions guard below —
+ ;; the next boot would see the database and no-op, leaving the stale permissions with no re-run path
+ (t2/with-transaction [_conn]
+ (t2/insert! :model/Database {:is_audit true
+ :id id
+ :name default-db-name
+ :description "Internal Audit DB used to power metabase analytics."
+ :engine engine
+ :is_full_sync true
+ :is_on_demand false
+ :creator_id nil
+ :auto_run_queries true})
+ ;; guard against someone manually deleting the audit-db entry, but not removing the audit-db permissions.
+ (t2/delete! :model/Permissions {:where [:like :object (str "%/db/" id "/%")]})))
(defn- adjust-audit-db-to-source!
[{audit-db-id :id}]
;; We need to move back to a schema that matches the serialized data
+ ;;
+ ;; The flip commits, so on a multi-node mysql/h2 cluster other nodes can see engine="postgres" while
+ ;; the load runs — the longstanding behavior of this pipeline (only #76551's single-transaction era
+ ;; briefly hid it). Resolving serialized names against an overlay instead would avoid even that.
(t2/update! :model/Database audit-db-id {:engine "postgres"})
;; do a separate select and update of table ids that are not downcased
;; we don't want to try to downcase audit db tables that may already have a downcased version
@@ -332,25 +339,31 @@
(directory-content-checksum views-dir ".sql"))))
(defn- maybe-sync-audit-db!
- "One-shot synchronous `:scan :schema` sync of the audit DB. Fires when either trigger is true:
+ "One-shot synchronous `:scan :schema` sync of the audit DB. Fires when any trigger is true:
- `engine-changed?` — `maybe-load-analytics-content!` swapped the audit DB engine and
field metadata needs to be refreshed for the new dialect.
- the `instance_analytics_views` SQL files have changed since the last successful sync,
meaning a migration may have added a new view that isn't yet in `metabase_table`.
- The two triggers share one sync because they both want the same operation. Runs synchronously
- (not in a background future) so it stays inside the caller's cross-node cluster lock and
- transaction — a sync on another thread would escape the lock and, on a transactional appdb,
- deadlock against the caller's uncommitted `metabase_table` writes."
+ - a previous engine-changed sync failed or was interrupted, recorded durably in
+ [[audit-app.settings/audit-db-dialect-sync-pending]].
+ The triggers share one sync because they all want the same operation. Runs synchronously
+ (not in a background future) so it stays serialized under the caller's cross-node cluster lock."
[audit-db engine-changed?]
- (let [current (views-checksum)
- views-stale? (and current (not= current (audit-app.settings/last-analytics-views-checksum)))]
- (when (or engine-changed? views-stale?)
- (log/infof "Syncing Audit DB schema (engine-changed? %s, views-stale? %s)"
- engine-changed? views-stale?)
+ (let [current (views-checksum)
+ views-stale? (and current (not= current (audit-app.settings/last-analytics-views-checksum)))
+ sync-pending? (audit-app.settings/audit-db-dialect-sync-pending)]
+ ;; record the debt durably BEFORE attempting the sync: the content checksum has already advanced, so
+ ;; a failed or interrupted sync would otherwise never be retried — no other path syncs the audit DB
+ (when engine-changed?
+ (audit-app.settings/audit-db-dialect-sync-pending! true))
+ (when (or engine-changed? views-stale? sync-pending?)
+ (log/infof "Syncing Audit DB schema (engine-changed? %s, views-stale? %s, sync-pending? %s)"
+ engine-changed? views-stale? sync-pending?)
(try
(log/with-no-logs (sync/sync-database! audit-db {:scan :schema}))
(when current
(audit-app.settings/last-analytics-views-checksum! current))
+ (audit-app.settings/audit-db-dialect-sync-pending! false)
(log/info "Audit DB sync complete.")
(catch Exception e
(log/error e "Audit DB sync failed."))))))
@@ -435,27 +448,30 @@
(group-by (comp u/lower-case-en :name))
(filter (fn [[_ rows]] (> (count rows) 1))))]
(doseq [[_ rows] groups]
- (let [referenced-ids (into #{}
- (map :table_id)
- (t2/query {:select-distinct [:table_id]
- :from [(t2/table-name :model/Card)]
- :where [:in :table_id (map :id rows)]}))
- survivor (or (first (filter (comp referenced-ids :id) rows))
- (first (filter :active rows))
- (first rows))
- [c-name c-schema] (host-canonical-table (:name survivor))
- orphans (remove #(= (:id %) (:id survivor)) rows)]
- ;; move content off each orphan, then delete it before re-canonicalizing the survivor so the survivor's
- ;; new (name, schema) can't collide with an orphan still sitting at that slot on idx_unique_table
- (doseq [{orphan-id :id} orphans]
- (repoint-cards-to-survivor! orphan-id (:id survivor))
- (t2/delete! :model/Table orphan-id))
- ;; clear is_defective_duplicate so the survivor re-enters idx_unique_table (its unique_table_helper is
- ;; NULL — and thus excluded from the index — while the flag is set)
- (t2/update! :model/Table (:id survivor)
- {:active true :name c-name :schema c-schema :is_defective_duplicate false})
- (log/infof "Reconciled %d duplicate audit view row(s) onto table id %s"
- (count orphans) (:id survivor))))))
+ ;; one transaction per group so a crash can't delete the orphans yet skip the survivor update —
+ ;; the shrunken group would no longer register as duplicates, stranding the survivor's flags
+ (t2/with-transaction [_conn]
+ (let [referenced-ids (into #{}
+ (map :table_id)
+ (t2/query {:select-distinct [:table_id]
+ :from [(t2/table-name :model/Card)]
+ :where [:in :table_id (map :id rows)]}))
+ survivor (or (first (filter (comp referenced-ids :id) rows))
+ (first (filter :active rows))
+ (first rows))
+ [c-name c-schema] (host-canonical-table (:name survivor))
+ orphans (remove #(= (:id %) (:id survivor)) rows)]
+ ;; move content off each orphan, then delete it before re-canonicalizing the survivor so the survivor's
+ ;; new (name, schema) can't collide with an orphan still sitting at that slot on idx_unique_table
+ (doseq [{orphan-id :id} orphans]
+ (repoint-cards-to-survivor! orphan-id (:id survivor))
+ (t2/delete! :model/Table orphan-id))
+ ;; clear is_defective_duplicate so the survivor re-enters idx_unique_table (its unique_table_helper is
+ ;; NULL — and thus excluded from the index — while the flag is set)
+ (t2/update! :model/Table (:id survivor)
+ {:active true :name c-name :schema c-schema :is_defective_duplicate false})
+ (log/infof "Reconciled %d duplicate audit view row(s) onto table id %s"
+ (count orphans) (:id survivor)))))))
(def ^:private audit-db-cluster-lock
"Cluster lock serializing the audit DB load/adjust/sync/reconcile across nodes (GHY-3974 1b)."
@@ -478,11 +494,15 @@
;; serialize install+adjust+load+sync+reconcile across nodes so a rolling upgrade can't run an adjust or sync
;; against a half-adjusted schema (`with-duplicate-ops-prevented` is per-process only). The install runs inside the
;; lock too, so a node that acquires it after the installer committed sees the DB already exists and falls through
- ;; to a no-op rather than colliding on the audit DB primary key. The sync runs synchronously inside the lock so it
- ;; stays in the lock's transaction. If another node already holds the lock it is doing this same work, so we skip
- ;; rather than fail the boot.
+ ;; to a no-op rather than colliding on the audit DB primary key. The sync runs synchronously so it stays serialized
+ ;; under the lock. If another node already holds the lock it is doing this same work, so we skip rather than fail
+ ;; the boot.
+ ;;
+ ;; the detached lock keeps the pipeline off the lock's transaction — its work commits incrementally in
+ ;; small transactions (#78238). That is safe here because the pipeline self-heals: the checksum only
+ ;; advances on completion and reconcile runs every boot, so a crash mid-pipeline is repaired next boot.
(try
- (cluster-lock/with-cluster-lock {:lock audit-db-cluster-lock :timeout-seconds 5 :retry-config {:max-retries 2}}
+ (cluster-lock/with-detached-cluster-lock {:lock audit-db-cluster-lock :timeout-seconds 5 :retry-config {:max-retries 2}}
(u/prog1 (maybe-install-audit-db!)
(when-let [audit-db (t2/select-one :model/Database :is_audit true)]
((sync-util/with-duplicate-ops-prevented
diff --git a/enterprise/backend/src/metabase_enterprise/audit_app/settings.clj b/enterprise/backend/src/metabase_enterprise/audit_app/settings.clj
index 5b2309d3bd4..ccf71faa3a6 100644
--- a/enterprise/backend/src/metabase_enterprise/audit_app/settings.clj
+++ b/enterprise/backend/src/metabase_enterprise/audit_app/settings.clj
@@ -34,3 +34,15 @@
:audit :never
:doc false
:export? false)
+
+(defsetting audit-db-dialect-sync-pending
+ "Whether the audit DB still owes a schema sync for the host dialect. Set (durably) before an
+ engine-changed sync is attempted and cleared only when one succeeds, so a sync that fails or is
+ interrupted after the analytics content checksum has already advanced is retried on the next boot
+ instead of waiting for the next release to change the checksum."
+ :type :boolean
+ :default false
+ :visibility :internal
+ :audit :never
+ :doc false
+ :export? false)
diff --git a/enterprise/backend/test/metabase_enterprise/audit_app/audit_test.clj b/enterprise/backend/test/metabase_enterprise/audit_app/audit_test.clj
index 2d19cbe9560..5d4a7be0c4e 100644
--- a/enterprise/backend/test/metabase_enterprise/audit_app/audit_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/audit_app/audit_test.clj
@@ -13,12 +13,14 @@
[metabase.permissions-rest.data-permissions.graph :as data-perms.graph]
[metabase.permissions.models.permissions-group :as perms-group]
[metabase.plugins.core :as plugins]
+ [metabase.sync.core :as sync.core]
[metabase.sync.sync :as sync]
[metabase.sync.task.sync-databases :as task.sync-databases]
[metabase.task.core :as task]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[metabase.util :as u]
+ [toucan2.connection :as t2.connection]
[toucan2.core :as t2])
(:import
(java.nio.charset StandardCharsets)
@@ -431,3 +433,85 @@
(t2/delete! :model/Database :is_audit true)
(audit/last-analytics-checksum! 0)
(mbc/ensure-audit-db-installed!)))))
+
+(deftest interrupted-engine-changed-sync-repaired-on-next-boot-test
+ ;; A failed or interrupted post-load dialect sync must be repaired by the next boot.
+ ;;
+ ;; `engine-changed? true` is the mysql/h2-host state: `maybe-load-analytics-content!` flips the
+ ;; audit DB engine to postgres for the serdes load and back to the host dialect after, and the
+ ;; returned boolean triggers a one-shot schema sync so field metadata gets re-scanned for the
+ ;; host dialect. (On a postgres appdb the engine never flips, so the boolean is injected here.)
+ ;; By the time that sync runs, the content checksum has already advanced -- so if the sync dies
+ ;; (or merely fails: `maybe-sync-audit-db!` catches and logs), the next boot sees engine
+ ;; matching, checksum matching, and views not stale, and must still notice the audit DB never
+ ;; got its dialect sync. There is no other repair path: the scheduled sync task and the sync
+ ;; API both refuse the audit DB.
+ (with-audit-db-restoration!
+ (let [audit-db (t2/select-one :model/Database :is_audit true)]
+ ;; "Boot 1": steady state (checksums current), the engine-changed sync dies mid-flight.
+ (with-redefs [sync.core/sync-database! (fn [& _] (throw (ex-info "killed mid-sync" {})))]
+ (#'ee-audit/maybe-sync-audit-db! audit-db true))
+ ;; "Boot 2": the full, un-stubbed pipeline, spying on audit-DB syncs.
+ (let [audit-db-syncs (atom 0)
+ real-sync sync.core/sync-database!]
+ (with-redefs [sync.core/sync-database! (fn [db & [opts]]
+ (when (:is_audit db)
+ (swap! audit-db-syncs inc))
+ (real-sync db opts))]
+ (mbc/ensure-audit-db-installed!))
+ (testing "the boot after an interrupted engine-changed sync re-syncs the audit DB"
+ (is (pos? @audit-db-syncs)))))))
+
+(deftest audit-pipeline-commits-incrementally-test
+ ;; Pins the `:detached? true` on `ensure-audit-db-installed!`'s cluster lock behaviorally: work
+ ;; done inside the pipeline must be visible to other sessions while the pipeline is still
+ ;; running. Under the pre-detached design the whole pipeline rode the lock's transaction, so
+ ;; nothing was visible until the final commit -- if a refactor drops the flag, the marker row
+ ;; below stays invisible to the probing thread and this test fails. (The probe binds
+ ;; *current-connectable* to nil because future conveys bindings: without it, a transactional
+ ;; pipeline would hand the probe its own connection and it would see uncommitted work.)
+ (with-audit-db-restoration!
+ (let [email (mt/random-email)
+ visible (atom ::not-checked)]
+ (try
+ ;; stale the content checksum so the (stubbed) load runs inside the locked pipeline
+ (audit/last-analytics-checksum! 0)
+ (with-redefs [serialization.cmd/v2-load-internal!
+ (fn [& _]
+ (t2/insert! :model/User (assoc (mt/with-temp-defaults :model/User) :email email))
+ (reset! visible
+ (deref (future
+ (binding [t2.connection/*current-connectable* nil]
+ (t2/exists? :model/User :email email)))
+ 5000 ::timeout))
+ ;; report no errors: content is already loaded from the steady state, so
+ ;; letting the checksum advance leaves a consistent end state
+ {:errors [] :seen []})]
+ (mbc/ensure-audit-db-installed!))
+ (testing "work committed inside the audit pipeline is visible mid-pipeline from another connection"
+ (is (true? @visible)))
+ (finally
+ (t2/delete! :model/User :email email))))))
+
+(deftest crash-mid-load-repaired-on-next-boot-test
+ ;; Pins the self-healing premise the detached lock's safety argument leans on: a crash in the
+ ;; middle of the serdes load must be repaired by the next boot. Two halves: the content
+ ;; checksum must not advance past a load that did not complete (so the next boot re-enters the
+ ;; load), and the next boot's un-stubbed pipeline must actually converge back to the healthy
+ ;; state.
+ (with-audit-db-restoration!
+ (let [healthy-checksum (audit/last-analytics-checksum)]
+ ;; make the content look new again, as an upgrade would
+ (audit/last-analytics-checksum! 0)
+ ;; "Boot 1": the serdes load dies partway through.
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"killed mid-load"
+ (with-redefs [serialization.cmd/v2-load-internal!
+ (fn [& _] (throw (ex-info "killed mid-load" {})))]
+ (mbc/ensure-audit-db-installed!))))
+ (testing "the content checksum does not advance past an incomplete load"
+ (is (= 0 (audit/last-analytics-checksum))))
+ ;; "Boot 2": the full, un-stubbed pipeline.
+ (mbc/ensure-audit-db-installed!)
+ (testing "the next boot re-runs the load and converges to the healthy state"
+ (is (= healthy-checksum (audit/last-analytics-checksum)))
+ (is (pos? (t2/count :model/Card :database_id audit/audit-db-id)))))))
diff --git a/src/metabase/app_db/cluster_lock.clj b/src/metabase/app_db/cluster_lock.clj
index 8862a11e29b..23778d75756 100644
--- a/src/metabase/app_db/cluster_lock.clj
+++ b/src/metabase/app_db/cluster_lock.clj
@@ -104,17 +104,35 @@
(with-open [stmt (prepare-statement conn lock-name-str timeout mode)
result-set (.executeQuery stmt)]
(when-not (.next result-set)
- ;; this record will not be visible until the tx commits, so there's no need to lock it
- ;; we instead rely on concurrent threads having constraint violation trying to insert their own record
- (t2/query-one {:insert-into [:metabase_cluster_lock]
- :columns [:lock_name]
- :values [[lock-name-str]]})))
+ ;; this record will not be visible until the tx commits, so there's no need to lock it; concurrent
+ ;; inserters get a constraint violation and retry. Raw JDBC because the insert must run on `conn`
+ ;; (under a detached lock ambient resolution would hand it a different connection) and needs the
+ ;; same query timeout as the SELECT — concurrent first-time inserters block on the winner's
+ ;; uncommitted unique-index entry
+ (let [[sql] (mdb.query/compile {:insert-into [:metabase_cluster_lock]
+ :columns [:lock_name]
+ :values [[[:raw "?"]]]})]
+ (with-open [insert-stmt (.prepareStatement conn ^String sql)]
+ (doto insert-stmt
+ (.setQueryTimeout timeout)
+ (.setString 1 lock-name-str))
+ (.executeUpdate insert-stmt)))))
(log/debugf "Obtained cluster lock: %s (%s)" lock-name-str mode))
+(def ^:private ^:dynamic *detached-locks-held*
+ "Lock-name strings currently held by [[do-with-detached-cluster-lock]] in this dynamic scope. A detached
+ hold lives on a dedicated connection, so re-acquiring the same name from this scope — detached or
+ transactional — would block against our own row lock; acquisitions fail fast instead."
+ #{})
+
(defn- do-with-cluster-locks*
"Acquire all `locks` (each a `{:lock-name-str, :mode}` map) inside a single
transaction, then run `thunk`."
[locks timeout-seconds thunk]
+ (doseq [{:keys [lock-name-str]} locks]
+ (when (*detached-locks-held* lock-name-str)
+ (throw (ex-info "Cluster lock is already held detached in this scope"
+ {:lock-name lock-name-str}))))
(t2/with-transaction [conn]
(doseq [{:keys [lock-name-str mode]} locks]
(acquire-lock-row! conn lock-name-str timeout-seconds mode))
@@ -270,13 +288,79 @@
(defmacro with-cluster-lock
"Run `body` in a transaction that tries to take a lock from the metabase_cluster_lock table of
the specified name to coordinate concurrency with other metabase instances sharing the appdb.
+ (On an h2 appdb the lock is an in-process read-write lock and no transaction is involved.)
`lock-options` may be a lock-name keyword, or an options map
`{:lock, :locks, :mode, :timeout-seconds, :retry-config, :retry-transient?}` —
- see [[do-with-cluster-lock]] for the full description of each."
+ see [[do-with-cluster-lock]] for the full description of each.
+
+ For long-running work whose appdb writes should commit incrementally instead of riding the lock's
+ transaction, see [[with-detached-cluster-lock]]."
([lock-options & body]
`(do-with-cluster-lock ~lock-options (fn [] ~@body))))
+(mu/defn do-with-detached-cluster-lock
+ "Impl for [[with-detached-cluster-lock]]."
+ [{:keys [lock timeout-seconds retry-config]
+ :or {timeout-seconds cluster-lock-timeout-seconds}}
+ :- [:map
+ [:lock :keyword]
+ [:timeout-seconds {:optional true} :int]
+ [:retry-config {:optional true} [:ref ::retry/retry-overrides]]]
+ thunk :- ifn?]
+ (let [lock-name-str (keyword->lock-name-str lock)]
+ (when (*detached-locks-held* lock-name-str)
+ (throw (ex-info "Cluster lock is already held detached in this scope"
+ {:lock-name lock-name-str})))
+ (if (= (mdb.connection/db-type) :h2)
+ ;; the h2 in-process lock never holds a transaction, so it is already 'detached'
+ (do-with-h2-cluster-locks* [{:lock-name-str lock-name-str :mode :exclusive}]
+ #(binding [*detached-locks-held* (conj *detached-locks-held* lock-name-str)]
+ (thunk)))
+ (mdb.connection/with-unshared-connection [conn]
+ (.setAutoCommit ^Connection conn false)
+ (let [config (merge default-retry-config retry-config)]
+ (try
+ (retry/with-retry config
+ ;; clear the aborted transaction a failed previous attempt leaves behind
+ (.rollback ^Connection conn)
+ (acquire-lock-row! conn lock-name-str timeout-seconds :exclusive))
+ (catch Throwable e
+ (if (retryable? e)
+ (throw (ex-info (str "Failed to obtain cluster lock: " lock-name-str)
+ {:lock-names [lock-name-str]
+ :retries (:max-retries config)}
+ e))
+ (throw e)))))
+ ;; the body runs outside the retry above, so its errors are never retried (its work has already
+ ;; committed) and never mistaken for acquisition failure
+ (let [result (binding [*detached-locks-held* (conj *detached-locks-held* lock-name-str)]
+ (thunk))]
+ ;; releases the row lock; on a body throw the pool's check-in rollback releases it instead
+ (.commit ^Connection conn)
+ result)))))
+
+(defmacro with-detached-cluster-lock
+ "Like [[with-cluster-lock]], but holds the (exclusive) lock row on a dedicated connection
+ (see [[metabase.app-db.connection/with-unshared-connection]]) while `body` runs on ordinary pooled
+ connections: `body`'s appdb work commits incrementally in its own short transactions instead of riding
+ one long transaction that holds the lock. The lock is released when `body` completes (commit) or
+ throws (pool check-in rollback) — but `body`'s already-committed work is NOT rolled back by a throw,
+ so only use this for long-running work that is idempotent/self-healing (the audit boot pipeline).
+
+ Not reentrant: re-acquiring a lock this scope already holds detached — in either detached or
+ transactional form — throws instead of self-deadlocking against the dedicated connection. (The
+ reverse — requesting a detached hold on a lock this scope holds transactionally — is not detected
+ and times out; don't.) Taking *different* locks inside `body` works normally. Hold duration is
+ bounded by the connection pool's limits: c3p0's `unreturnedConnectionTimeout`, when configured,
+ destroys the connection and silently releases the lock, and with the appdb checkout timeout set to
+ 0 (wait forever) a body blocked on pool checkout holds the lock indefinitely.
+
+ `opts` is `{:lock, :timeout-seconds, :retry-config}` as in [[with-cluster-lock]]; retries apply to
+ lock acquisition only, never to `body`."
+ [opts & body]
+ `(do-with-detached-cluster-lock ~opts (fn [] ~@body)))
+
(def card-statistics-lock
"A shared keyword that any method doing a batch update of card statistics can use for the cluster lock"
::statistics-lock)
diff --git a/src/metabase/app_db/connection.clj b/src/metabase/app_db/connection.clj
index 575696f8f88..e6496400b0a 100644
--- a/src/metabase/app_db/connection.clj
+++ b/src/metabase/app_db/connection.clj
@@ -300,6 +300,44 @@
(run-after-commit-callbacks! callbacks))
result)))
+;;;; Unshared connections
+;;;;
+;;;; A connection that holds fragile long-lived state -- a streaming result set (postgres portal), a row
+;;;; lock held while other work runs -- must not be picked up by ambient connection reuse: a borrower that
+;;;; inherits it via [[toucan2.connection/*current-connectable*]] and commits destroys that state (see
+;;;; #78238 and the revert of #76645). [[toucan2.connection/unshared-connection!]] makes toucan use such a
+;;;; connection only when passed it explicitly, and never advertise it ambiently -- ambient work inside
+;;;; `body` resolves to the default connectable and runs on other pooled connections.
+
+(defn do-with-unshared-connection
+ "Impl for [[with-unshared-connection]]."
+ [f]
+ ;; through *application-db* (not the raw data source) so its connection read-lock gate applies
+ (with-open [conn (.getConnection *application-db*)]
+ (f (t2.conn/unshared-connection! conn))))
+
+(defmacro with-unshared-connection
+ "Execute `body` with `conn-binding` bound to a fresh app-db connection that ambient connection reuse can
+ never pick up: toucan runs queries and transactions on it when `body` passes it explicitly, but never
+ binds it as [[toucan2.connection/*current-connectable*]], so toucan calls that resolve their connection
+ ambiently -- including mid-reduction of a long-running query on this connection -- run (and commit) on
+ other pooled connections.
+
+ In every other respect this is an ordinary JDBC connection, configured by `body`: e.g. call
+ `.setAutoCommit false` for a streaming result set (postgres portal/cursor) or a held row lock. It is
+ closed when `body` ends; the pool resets its state (rolling back any unresolved transaction) on check-in.
+
+ Caveat: do not pass the unshared connection to an explicit `t2/with-transaction` while an ambient
+ appdb transaction is open on this thread. Nesting depth is tracked per thread, not per connection, so
+ the inner call would take a savepoint on this connection that no commit ever follows, and its work
+ would silently roll back at check-in.
+
+ (mdb/with-unshared-connection [conn]
+ (.setAutoCommit conn false)
+ (reduce rf init (t2/reducible-query conn a-huge-query)))"
+ [[conn-binding] & body]
+ `(do-with-unshared-connection (fn [~conn-binding] ~@body)))
+
(methodical/defmethod t2.pipeline/transduce-query :before :default
"Make sure application database calls are not done inside core.async dispatch pool threads. This is done relatively
early in the pipeline so the stacktrace when this fails isn't super enormous."
diff --git a/src/metabase/app_db/core.clj b/src/metabase/app_db/core.clj
index 57d32570b3e..2809ef4242b 100644
--- a/src/metabase/app_db/core.clj
+++ b/src/metabase/app_db/core.clj
@@ -42,7 +42,8 @@
in-transaction?
quoting-style
unique-identifier
- transaction-state]
+ transaction-state
+ with-unshared-connection]
[mdb.connection-pool-setup
recent-activity?]
[mdb.data-source
diff --git a/test/metabase/app_db/cluster_lock_test.clj b/test/metabase/app_db/cluster_lock_test.clj
index bdc5728d012..78eb13439aa 100644
--- a/test/metabase/app_db/cluster_lock_test.clj
+++ b/test/metabase/app_db/cluster_lock_test.clj
@@ -5,6 +5,7 @@
[metabase.app-db.cluster-lock :as sut]
[metabase.app-db.core :as mdb]
[metabase.app-db.transient-error :as transient-error]
+ [metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[metabase.util :as u]
[toucan2.core :as t2])
@@ -92,6 +93,13 @@
(future (Thread/sleep 500) (a/>!! fin-chan :done))
(is (nil? (sut/with-cluster-lock ::test-lock (Thread/sleep 1))))))))
+(defn- do-with-lock-opts
+ "Route on the test-only :detached? key so every helper call site can exercise either entry point."
+ [lock-opts thunk]
+ (if (:detached? lock-opts)
+ (sut/do-with-detached-cluster-lock (dissoc lock-opts :detached?) thunk)
+ (sut/do-with-cluster-lock lock-opts thunk)))
+
(defn- run-with-lock
"Run `thunk` with the given lock opts on a future. Returns a map of
`{:entered, :release, :done}` latches/promise the caller can use to
@@ -102,9 +110,10 @@
done (promise)]
(future
(try
- (sut/with-cluster-lock lock-opts
- (.countDown entered)
- (.await release))
+ (do-with-lock-opts lock-opts
+ (fn []
+ (.countDown entered)
+ (.await release)))
(deliver done :ok)
(catch Throwable e
(deliver done [:err (ex-message e)]))))
@@ -116,8 +125,7 @@
(let [entered (promise)
acquired (future
(try
- (sut/with-cluster-lock lock-opts
- (deliver entered :yes))
+ (do-with-lock-opts lock-opts (fn [] (deliver entered :yes)))
(catch Throwable _
(deliver entered :err))))
result (deref entered timeout-ms :timeout)]
@@ -226,3 +234,133 @@
(is (= [[:a :failed :timeout]
[:b :done]]
@results))))))
+
+(deftest detached-lock-basic-test
+ (testing "returns the body's value and works non-concurrently"
+ (is (= :ok (sut/with-detached-cluster-lock {:lock ::detached-basic} :ok)))))
+
+(deftest detached-lock-body-commits-incrementally-test
+ (let [email (mt/random-email)]
+ (try
+ (testing "body work commits as it goes and survives a later throw in the body"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"
+ (sut/with-detached-cluster-lock {:lock ::detached-commit}
+ (t2/insert! :model/User (assoc (mt/with-temp-defaults :model/User) :email email))
+ (throw (ex-info "boom" {})))))
+ (is (t2/exists? :model/User :email email)))
+ (finally
+ (t2/delete! :model/User :email email)))))
+
+(deftest detached-lock-reentry-throws-test
+ (testing "re-acquiring a held detached lock throws instead of self-deadlocking"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"already held detached"
+ (sut/with-detached-cluster-lock {:lock ::detached-reentry}
+ (sut/with-detached-cluster-lock {:lock ::detached-reentry}
+ :nope)))))
+ ;; the transactional-path guard lives in the row-lock impl; h2 takes its in-process reentrant lock
+ (when (not= (mdb/db-type) :h2)
+ (testing "a transactional acquisition of a lock this scope holds detached also throws"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"already held detached"
+ (sut/with-detached-cluster-lock {:lock ::detached-reentry}
+ (sut/with-cluster-lock ::detached-reentry
+ :nope)))))
+ (testing "taking a different lock inside a detached body works normally"
+ (is (= :ok (sut/with-detached-cluster-lock {:lock ::detached-reentry}
+ (sut/with-cluster-lock ::detached-reentry-other
+ :ok)))))))
+
+(deftest detached-lock-mutual-exclusion-test
+ ;; h2 uses the in-process rw-lock path; real row-lock semantics only.
+ (when (not= (mdb/db-type) :h2)
+ ;; warm up the row first so we're not racing on the initial INSERT
+ (sut/with-detached-cluster-lock {:lock ::detached-mutex} :warm)
+ (let [held (run-with-lock {:lock ::detached-mutex :detached? true})]
+ (is (.await ^CountDownLatch (:entered held) 3 TimeUnit/SECONDS))
+ (testing "a detached holder blocks another detached acquirer"
+ (is (not (acquirable-within? {:lock ::detached-mutex :detached? true
+ :timeout-seconds 1 :retry-config {:max-retries 0}}
+ 3000))))
+ (testing "a detached holder blocks a transactional acquirer"
+ (is (not (acquirable-within? {:lock ::detached-mutex
+ :timeout-seconds 1 :retry-config {:max-retries 0}}
+ 3000))))
+ (.countDown ^CountDownLatch (:release held))
+ (is (= :ok (deref (:done held) 3000 :timeout)))
+ (testing "released once the body completes"
+ (is (acquirable-within? {:lock ::detached-mutex :detached? true} 3000))))))
+
+(defn- query-canceled-exception
+ "Build a query-canceled SQLException for the current appdb type. Mirrors the codes in
+ [[metabase.app-db.query-cancelation]] so the test exercises the real db-type path."
+ []
+ (case (mdb/db-type)
+ :postgres (SQLException. "ERROR: canceling statement due to user request" "57014")
+ :mysql (SQLException. "Statement cancelled due to timeout or client request" "70100" 1317)))
+
+(deftest detached-lock-body-error-not-retried-test
+ ;; h2 takes an in-process lock and bypasses the retry path entirely, so this only exercises real row locks.
+ (when (not= (mdb/db-type) :h2)
+ ;; warm up the row first so we're not racing on the initial INSERT
+ (sut/with-detached-cluster-lock {:lock ::detached-body-error} :warm)
+ (let [attempts (atom 0)
+ e (is (thrown? Throwable
+ (sut/with-detached-cluster-lock {:lock ::detached-body-error
+ :retry-config {:max-retries 2 :delay-ms 1}}
+ (swap! attempts inc)
+ (throw (query-canceled-exception)))))]
+ (testing "an acquisition-retryable error raised by the body must not re-run the body: its work commits
+ incrementally, so a re-run double-applies it"
+ (is (= 1 @attempts)))
+ (testing "a body error must not be rewrapped as an acquisition failure — the lock WAS obtained"
+ (is (not (re-find #"Failed to obtain cluster lock" (str (ex-message e)))))))))
+
+(deftest detached-lock-nested-lock-timeout-not-retried-test
+ ;; The audit-pipeline shape: a detached body that takes another cluster lock inside (serdes takes the
+ ;; permissions lock when inserting a Database). When the inner acquisition times out, the canceled
+ ;; SELECT stays in the cause chain of the inner "Failed to obtain cluster lock" wrapper, so the outer
+ ;; machinery must not read it as a failure to acquire the OUTER lock: re-running the body would
+ ;; double-apply its committed work, and attributing the error to the outer lock makes callers that
+ ;; treat outer-lock contention as benign (ensure-audit-db-installed!) swallow a genuine body failure.
+ (when (not= (mdb/db-type) :h2)
+ (sut/with-cluster-lock ::nested-timeout-inner :warm)
+ (sut/with-detached-cluster-lock {:lock ::nested-timeout-outer} :warm)
+ (let [held (run-with-lock {:lock ::nested-timeout-inner})]
+ (try
+ (is (.await ^CountDownLatch (:entered held) 3 TimeUnit/SECONDS))
+ (let [attempts (atom 0)
+ e (is (thrown? Throwable
+ (sut/with-detached-cluster-lock {:lock ::nested-timeout-outer
+ :retry-config {:max-retries 2 :delay-ms 1}}
+ (swap! attempts inc)
+ (sut/with-cluster-lock {:lock ::nested-timeout-inner
+ :timeout-seconds 1 :retry-config {:max-retries 0}}
+ :never))))]
+ (testing "the incrementally-committing outer body must run exactly once"
+ (is (= 1 @attempts)))
+ (testing "the failure must not be attributed to the outer lock the body already holds"
+ (is (not= [(u/qualified-name ::nested-timeout-outer)]
+ (:lock-names (ex-data e))))))
+ (finally
+ (.countDown ^CountDownLatch (:release held))
+ (is (= :ok (deref (:done held) 3000 :timeout))))))))
+
+(deftest detached-lock-released-after-body-throw-test
+ ;; The throw path has no explicit rollback: it relies on with-open closing the dedicated
+ ;; connection and the pool rolling back the unresolved transaction on check-in. If that ever
+ ;; stopped happening (pool swap or misconfiguration), a boot-time audit failure would leave the
+ ;; row lock held until the connection ages out, silently stalling other nodes' boots — the class
+ ;; of bug the detached lock exists to fix. Release after *normal* completion is covered by
+ ;; detached-lock-mutual-exclusion-test; this pins the throw path.
+ (when (not= (mdb/db-type) :h2)
+ ;; warm up the row first so we're not racing on the initial INSERT
+ (sut/with-detached-cluster-lock {:lock ::detached-throw-release} :warm)
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"
+ (sut/with-detached-cluster-lock {:lock ::detached-throw-release}
+ (throw (ex-info "boom" {})))))
+ (testing "the row lock is released once the throwing body's connection is checked back in"
+ (is (acquirable-within? {:lock ::detached-throw-release :detached? true
+ :timeout-seconds 1 :retry-config {:max-retries 0}}
+ 3000))
+ (is (acquirable-within? {:lock ::detached-throw-release
+ :timeout-seconds 1 :retry-config {:max-retries 0}}
+ 3000)))))
diff --git a/test/metabase/app_db/connection_test.clj b/test/metabase/app_db/connection_test.clj
index e6d5eee3e1c..ad576573450 100644
--- a/test/metabase/app_db/connection_test.clj
+++ b/test/metabase/app_db/connection_test.clj
@@ -5,7 +5,8 @@
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[toucan2.connection :as t2.connection]
- [toucan2.core :as t2])
+ [toucan2.core :as t2]
+ [toucan2.jdbc.options :as t2.jdbc.options])
(:import
(java.sql Connection)
(java.util.concurrent Semaphore)))
@@ -339,3 +340,90 @@
(throw (ex-info "force savepoint rollback" {})))))
(is (= {:outer-key "outer-val"} @mdb.connection/*transaction-state*)
"data stashed by a rolled-back nested transaction is discarded from transaction-state")))
+
+(deftest unshared-connection-not-bound-during-use-test
+ (mdb.connection/with-unshared-connection [conn]
+ (testing "explicitly passing the unshared connection works, but it is never bound as *current-connectable*"
+ (let [rows (reduce (fn [acc row]
+ ;; the contract is nil, not merely "not this connection": while an unshared
+ ;; connection is in use there is no ambient connection at all
+ (is (nil? t2.connection/*current-connectable*)
+ "ambient resolution must be suppressed mid-reduction")
+ (conj acc (into {} row)))
+ []
+ (t2/reducible-query conn ["select 1 as x"]))]
+ (is (= 1 (count rows)))))
+ (testing "a transaction opened while the unshared connection is in use gets its own connection"
+ (reduce (fn [_ _row]
+ (t2/with-transaction [tx-conn]
+ (is (not (identical? tx-conn conn)))))
+ nil
+ (t2/reducible-query conn ["select 1 as x"])))))
+
+(deftest unshared-connection-borrower-work-commits-independently-test
+ (let [email (mt/random-email)]
+ (try
+ (testing "toucan work while the unshared connection is in use commits on its own connection"
+ (mdb.connection/with-unshared-connection [conn]
+ ;; open a transaction on the unshared connection that is never committed: if the insert below
+ ;; wrongly rode this connection, it would be rolled back at pool check-in and the user would not exist
+ (.setAutoCommit ^Connection conn false)
+ (reduce (fn [_ _row]
+ (t2/insert! :model/User (assoc (mt/with-temp-defaults :model/User) :email email)))
+ nil
+ (t2/reducible-query conn ["select 1 as x"])))
+ (is (t2/exists? :model/User :email email)))
+ (finally
+ (t2/delete! :model/User :email email)))))
+
+(deftest unshared-connection-postgres-portal-survival-test
+ ;; postgres streams a result set through a portal that dies if anything commits on the connection
+ ;; mid-iteration -- the failure mode that forced the revert of #76645. Other appdbs buffer, so there is
+ ;; nothing to kill.
+ (when (= (mdb.connection/db-type) :postgres)
+ (let [portal-death? (fn [e]
+ (loop [e e]
+ (cond
+ (nil? e) false
+ (re-find #"portal" (str (ex-message e))) true
+ :else (recur (ex-cause e)))))
+ stream! (fn [^Connection conn]
+ ;; fetch-size + autocommit off = pg streams through a portal
+ (.setAutoCommit conn false)
+ (binding [t2.jdbc.options/*options* {:fetch-size 50}]
+ (reduce (fn [n _row]
+ ;; unrelated toucan work mid-stream, resolved ambiently, that commits
+ (when (= n 30)
+ (t2/with-transaction [_]
+ (t2/query ["select 1"])))
+ (inc n))
+ 0
+ (t2/reducible-query conn ["select g from generate_series(1, 1000) g"]))))]
+ (testing "control: an ordinary connection is ambiently visible mid-reduction; the borrower commit kills the portal"
+ (with-open [^Connection raw (.getConnection (mdb.connection/data-source))]
+ (let [e (try (stream! raw) nil (catch Throwable e e))]
+ (is (portal-death? e)))
+ (.rollback raw)
+ (.setAutoCommit raw true)))
+ (testing "an unshared connection is invisible to the borrower; the portal survives"
+ (mdb.connection/with-unshared-connection [conn]
+ (is (= 1000 (stream! conn))))))))
+
+(deftest unshared-connection-uncommitted-work-rolls-back-on-close-test
+ ;; the with-unshared-connection docstring promises the pool resets state on check-in, rolling
+ ;; back any unresolved transaction — the same mechanism the detached cluster lock relies on to
+ ;; release its row locks after a body throw. Pin the rollback itself.
+ (let [lock-name "connection-test/unshared-rollback"]
+ (try
+ (mdb.connection/with-unshared-connection [conn]
+ (.setAutoCommit ^Connection conn false)
+ (t2/query-one conn {:insert-into :metabase_cluster_lock
+ :columns [:lock_name]
+ :values [[lock-name]]})
+ (testing "the write is visible on the unshared connection before close"
+ (is (= 1 (count (t2/query conn ["select * from metabase_cluster_lock where lock_name = ?"
+ lock-name]))))))
+ (testing "the uncommitted write vanished at pool check-in"
+ (is (not (t2/exists? :metabase_cluster_lock :lock_name lock-name))))
+ (finally
+ (t2/delete! :metabase_cluster_lock :lock_name lock-name)))))