The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Unshared appdb connections + detached audit cluster lock (fix first-boot stall) (#78276)
dpsutton0ed95e05f66e
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.ednindex bc4ebdbcb70..6c1f7a52e9b 100644--- a/.clj-kondo/config.edn+++ b/.clj-kondo/config.edn@@ -467,6 +467,8 @@metabase.actions.test-util/with-actions clojure.core/letmetabase.transforms.test-util/with-transform-cleanup! clojure.core/letmetabase.api.common/let-404 clojure.core/let+ metabase.app-db.connection/with-unshared-connection clojure.core/fn+ metabase.app-db.core/with-unshared-connection clojure.core/fnmetabase.app-db.custom-migrations.util/with-temp-scheduler! clojure.core/fnmetabase.app-db.custom-migrations/define-migration clj-kondo.lint-as/def-catch-allmetabase.app-db.custom-migrations/define-reversible-migration clj-kondo.lint-as/def-catch-alldiff --git a/deps.edn b/deps.ednindex 154b1cbf18f..b4ca745dbfd 100644--- a/deps.edn+++ b/deps.edn@@ -80,7 +80,7 @@hiccup/hiccup {:mvn/version "2.0.0"} ; HTML templatinginflections/inflections {:mvn/version "0.15.0"} ; Clojure/Script library used for prularizing wordsinstaparse/instaparse {:mvn/version "1.5.0"} ; Make your own parser- io.github.camsaul/toucan2 {:mvn/version "1.0.571"}+ io.github.camsaul/toucan2 {:mvn/version "1.0.572"}io.github.eerohele/pp {#_#_:git/tag "2024-11-13.77" ; super fast pretty-printing library:git/sha "ffc4edc482c057cd9412c62f018a41e9140c618b":git/url "https://github.com/eerohele/pp"}diff --git a/enterprise/backend/src/metabase_enterprise/audit_app/audit.clj b/enterprise/backend/src/metabase_enterprise/audit_app/audit.cljindex 1a7ece59b34..f33602a116c 100644--- a/enterprise/backend/src/metabase_enterprise/audit_app/audit.clj+++ b/enterprise/backend/src/metabase_enterprise/audit_app/audit.clj@@ -63,21 +63,28 @@- This uses a weird ID because some tests were hardcoded to look for database with ID = 2, and inserting an extra dbthrows that off since these IDs are sequential."[engine id]- (t2/insert! :model/Database {:is_audit true- :id id- :name default-db-name- :description "Internal Audit DB used to power metabase analytics."- :engine engine- :is_full_sync true- :is_on_demand false- :creator_id nil- :auto_run_queries true})- ;; guard against someone manually deleting the audit-db entry, but not removing the audit-db permissions.- (t2/delete! :model/Permissions {:where [:like :object (str "%/db/" id "/%")]}))+ ;; one transaction so a crash can't commit the database row yet skip the permissions guard below —+ ;; the next boot would see the database and no-op, leaving the stale permissions with no re-run path+ (t2/with-transaction [_conn]+ (t2/insert! :model/Database {:is_audit true+ :id id+ :name default-db-name+ :description "Internal Audit DB used to power metabase analytics."+ :engine engine+ :is_full_sync true+ :is_on_demand false+ :creator_id nil+ :auto_run_queries true})+ ;; guard against someone manually deleting the audit-db entry, but not removing the audit-db permissions.+ (t2/delete! :model/Permissions {:where [:like :object (str "%/db/" id "/%")]})))(defn- adjust-audit-db-to-source![{audit-db-id :id}];; We need to move back to a schema that matches the serialized data+ ;;+ ;; The flip commits, so on a multi-node mysql/h2 cluster other nodes can see engine="postgres" while+ ;; the load runs — the longstanding behavior of this pipeline (only #76551's single-transaction era+ ;; briefly hid it). Resolving serialized names against an overlay instead would avoid even that.(t2/update! :model/Database audit-db-id {:engine "postgres"});; do a separate select and update of table ids that are not downcased;; we don't want to try to downcase audit db tables that may already have a downcased version@@ -332,25 +339,31 @@(directory-content-checksum views-dir ".sql"))))(defn- maybe-sync-audit-db!- "One-shot synchronous `:scan :schema` sync of the audit DB. Fires when either trigger is true:+ "One-shot synchronous `:scan :schema` sync of the audit DB. Fires when any trigger is true:- `engine-changed?` — `maybe-load-analytics-content!` swapped the audit DB engine andfield metadata needs to be refreshed for the new dialect.- the `instance_analytics_views` SQL files have changed since the last successful sync,meaning a migration may have added a new view that isn't yet in `metabase_table`.- The two triggers share one sync because they both want the same operation. Runs synchronously- (not in a background future) so it stays inside the caller's cross-node cluster lock and- transaction — a sync on another thread would escape the lock and, on a transactional appdb,- deadlock against the caller's uncommitted `metabase_table` writes."+ - a previous engine-changed sync failed or was interrupted, recorded durably in+ [[audit-app.settings/audit-db-dialect-sync-pending]].+ The triggers share one sync because they all want the same operation. Runs synchronously+ (not in a background future) so it stays serialized under the caller's cross-node cluster lock."[audit-db engine-changed?]- (let [current (views-checksum)- views-stale? (and current (not= current (audit-app.settings/last-analytics-views-checksum)))]- (when (or engine-changed? views-stale?)- (log/infof "Syncing Audit DB schema (engine-changed? %s, views-stale? %s)"- engine-changed? views-stale?)+ (let [current (views-checksum)+ views-stale? (and current (not= current (audit-app.settings/last-analytics-views-checksum)))+ sync-pending? (audit-app.settings/audit-db-dialect-sync-pending)]+ ;; record the debt durably BEFORE attempting the sync: the content checksum has already advanced, so+ ;; a failed or interrupted sync would otherwise never be retried — no other path syncs the audit DB+ (when engine-changed?+ (audit-app.settings/audit-db-dialect-sync-pending! true))+ (when (or engine-changed? views-stale? sync-pending?)+ (log/infof "Syncing Audit DB schema (engine-changed? %s, views-stale? %s, sync-pending? %s)"+ engine-changed? views-stale? sync-pending?)(try(log/with-no-logs (sync/sync-database! audit-db {:scan :schema}))(when current(audit-app.settings/last-analytics-views-checksum! current))+ (audit-app.settings/audit-db-dialect-sync-pending! false)(log/info "Audit DB sync complete.")(catch Exception e(log/error e "Audit DB sync failed."))))))@@ -435,27 +448,30 @@(group-by (comp u/lower-case-en :name))(filter (fn [[_ rows]] (> (count rows) 1))))](doseq [[_ rows] groups]- (let [referenced-ids (into #{}- (map :table_id)- (t2/query {:select-distinct [:table_id]- :from [(t2/table-name :model/Card)]- :where [:in :table_id (map :id rows)]}))- survivor (or (first (filter (comp referenced-ids :id) rows))- (first (filter :active rows))- (first rows))- [c-name c-schema] (host-canonical-table (:name survivor))- orphans (remove #(= (:id %) (:id survivor)) rows)]- ;; move content off each orphan, then delete it before re-canonicalizing the survivor so the survivor's- ;; new (name, schema) can't collide with an orphan still sitting at that slot on idx_unique_table- (doseq [{orphan-id :id} orphans]- (repoint-cards-to-survivor! orphan-id (:id survivor))- (t2/delete! :model/Table orphan-id))- ;; clear is_defective_duplicate so the survivor re-enters idx_unique_table (its unique_table_helper is- ;; NULL — and thus excluded from the index — while the flag is set)- (t2/update! :model/Table (:id survivor)- {:active true :name c-name :schema c-schema :is_defective_duplicate false})- (log/infof "Reconciled %d duplicate audit view row(s) onto table id %s"- (count orphans) (:id survivor))))))+ ;; one transaction per group so a crash can't delete the orphans yet skip the survivor update —+ ;; the shrunken group would no longer register as duplicates, stranding the survivor's flags+ (t2/with-transaction [_conn]+ (let [referenced-ids (into #{}+ (map :table_id)+ (t2/query {:select-distinct [:table_id]+ :from [(t2/table-name :model/Card)]+ :where [:in :table_id (map :id rows)]}))+ survivor (or (first (filter (comp referenced-ids :id) rows))+ (first (filter :active rows))+ (first rows))+ [c-name c-schema] (host-canonical-table (:name survivor))+ orphans (remove #(= (:id %) (:id survivor)) rows)]+ ;; move content off each orphan, then delete it before re-canonicalizing the survivor so the survivor's+ ;; new (name, schema) can't collide with an orphan still sitting at that slot on idx_unique_table+ (doseq [{orphan-id :id} orphans]+ (repoint-cards-to-survivor! orphan-id (:id survivor))+ (t2/delete! :model/Table orphan-id))+ ;; clear is_defective_duplicate so the survivor re-enters idx_unique_table (its unique_table_helper is+ ;; NULL — and thus excluded from the index — while the flag is set)+ (t2/update! :model/Table (:id survivor)+ {:active true :name c-name :schema c-schema :is_defective_duplicate false})+ (log/infof "Reconciled %d duplicate audit view row(s) onto table id %s"+ (count orphans) (:id survivor)))))))(def ^:private audit-db-cluster-lock"Cluster lock serializing the audit DB load/adjust/sync/reconcile across nodes (GHY-3974 1b)."@@ -478,11 +494,15 @@;; serialize install+adjust+load+sync+reconcile across nodes so a rolling upgrade can't run an adjust or sync;; against a half-adjusted schema (`with-duplicate-ops-prevented` is per-process only). The install runs inside the;; lock too, so a node that acquires it after the installer committed sees the DB already exists and falls through- ;; to a no-op rather than colliding on the audit DB primary key. The sync runs synchronously inside the lock so it- ;; stays in the lock's transaction. If another node already holds the lock it is doing this same work, so we skip- ;; rather than fail the boot.+ ;; to a no-op rather than colliding on the audit DB primary key. The sync runs synchronously so it stays serialized+ ;; under the lock. If another node already holds the lock it is doing this same work, so we skip rather than fail+ ;; the boot.+ ;;+ ;; the detached lock keeps the pipeline off the lock's transaction — its work commits incrementally in+ ;; small transactions (#78238). That is safe here because the pipeline self-heals: the checksum only+ ;; advances on completion and reconcile runs every boot, so a crash mid-pipeline is repaired next boot.(try- (cluster-lock/with-cluster-lock {:lock audit-db-cluster-lock :timeout-seconds 5 :retry-config {:max-retries 2}}+ (cluster-lock/with-detached-cluster-lock {:lock audit-db-cluster-lock :timeout-seconds 5 :retry-config {:max-retries 2}}(u/prog1 (maybe-install-audit-db!)(when-let [audit-db (t2/select-one :model/Database :is_audit true)]((sync-util/with-duplicate-ops-preventeddiff --git a/enterprise/backend/src/metabase_enterprise/audit_app/settings.clj b/enterprise/backend/src/metabase_enterprise/audit_app/settings.cljindex 5b2309d3bd4..ccf71faa3a6 100644--- a/enterprise/backend/src/metabase_enterprise/audit_app/settings.clj+++ b/enterprise/backend/src/metabase_enterprise/audit_app/settings.clj@@ -34,3 +34,15 @@:audit :never:doc false:export? false)++(defsetting audit-db-dialect-sync-pending+ "Whether the audit DB still owes a schema sync for the host dialect. Set (durably) before an+ engine-changed sync is attempted and cleared only when one succeeds, so a sync that fails or is+ interrupted after the analytics content checksum has already advanced is retried on the next boot+ instead of waiting for the next release to change the checksum."+ :type :boolean+ :default false+ :visibility :internal+ :audit :never+ :doc false+ :export? false)diff --git a/enterprise/backend/test/metabase_enterprise/audit_app/audit_test.clj b/enterprise/backend/test/metabase_enterprise/audit_app/audit_test.cljindex 2d19cbe9560..5d4a7be0c4e 100644--- a/enterprise/backend/test/metabase_enterprise/audit_app/audit_test.clj+++ b/enterprise/backend/test/metabase_enterprise/audit_app/audit_test.clj@@ -13,12 +13,14 @@[metabase.permissions-rest.data-permissions.graph :as data-perms.graph][metabase.permissions.models.permissions-group :as perms-group][metabase.plugins.core :as plugins]+ [metabase.sync.core :as sync.core][metabase.sync.sync :as sync][metabase.sync.task.sync-databases :as task.sync-databases][metabase.task.core :as task][metabase.test :as mt][metabase.test.fixtures :as fixtures][metabase.util :as u]+ [toucan2.connection :as t2.connection][toucan2.core :as t2])(:import(java.nio.charset StandardCharsets)@@ -431,3 +433,85 @@(t2/delete! :model/Database :is_audit true)(audit/last-analytics-checksum! 0)(mbc/ensure-audit-db-installed!)))))++(deftest interrupted-engine-changed-sync-repaired-on-next-boot-test+ ;; A failed or interrupted post-load dialect sync must be repaired by the next boot.+ ;;+ ;; `engine-changed? true` is the mysql/h2-host state: `maybe-load-analytics-content!` flips the+ ;; audit DB engine to postgres for the serdes load and back to the host dialect after, and the+ ;; returned boolean triggers a one-shot schema sync so field metadata gets re-scanned for the+ ;; host dialect. (On a postgres appdb the engine never flips, so the boolean is injected here.)+ ;; By the time that sync runs, the content checksum has already advanced -- so if the sync dies+ ;; (or merely fails: `maybe-sync-audit-db!` catches and logs), the next boot sees engine+ ;; matching, checksum matching, and views not stale, and must still notice the audit DB never+ ;; got its dialect sync. There is no other repair path: the scheduled sync task and the sync+ ;; API both refuse the audit DB.+ (with-audit-db-restoration!+ (let [audit-db (t2/select-one :model/Database :is_audit true)]+ ;; "Boot 1": steady state (checksums current), the engine-changed sync dies mid-flight.+ (with-redefs [sync.core/sync-database! (fn [& _] (throw (ex-info "killed mid-sync" {})))]+ (#'ee-audit/maybe-sync-audit-db! audit-db true))+ ;; "Boot 2": the full, un-stubbed pipeline, spying on audit-DB syncs.+ (let [audit-db-syncs (atom 0)+ real-sync sync.core/sync-database!]+ (with-redefs [sync.core/sync-database! (fn [db & [opts]]+ (when (:is_audit db)+ (swap! audit-db-syncs inc))+ (real-sync db opts))]+ (mbc/ensure-audit-db-installed!))+ (testing "the boot after an interrupted engine-changed sync re-syncs the audit DB"+ (is (pos? @audit-db-syncs)))))))++(deftest audit-pipeline-commits-incrementally-test+ ;; Pins the `:detached? true` on `ensure-audit-db-installed!`'s cluster lock behaviorally: work+ ;; done inside the pipeline must be visible to other sessions while the pipeline is still+ ;; running. Under the pre-detached design the whole pipeline rode the lock's transaction, so+ ;; nothing was visible until the final commit -- if a refactor drops the flag, the marker row+ ;; below stays invisible to the probing thread and this test fails. (The probe binds+ ;; *current-connectable* to nil because future conveys bindings: without it, a transactional+ ;; pipeline would hand the probe its own connection and it would see uncommitted work.)+ (with-audit-db-restoration!+ (let [email (mt/random-email)+ visible (atom ::not-checked)]+ (try+ ;; stale the content checksum so the (stubbed) load runs inside the locked pipeline+ (audit/last-analytics-checksum! 0)+ (with-redefs [serialization.cmd/v2-load-internal!+ (fn [& _]+ (t2/insert! :model/User (assoc (mt/with-temp-defaults :model/User) :email email))+ (reset! visible+ (deref (future+ (binding [t2.connection/*current-connectable* nil]+ (t2/exists? :model/User :email email)))+ 5000 ::timeout))+ ;; report no errors: content is already loaded from the steady state, so+ ;; letting the checksum advance leaves a consistent end state+ {:errors [] :seen []})]+ (mbc/ensure-audit-db-installed!))+ (testing "work committed inside the audit pipeline is visible mid-pipeline from another connection"+ (is (true? @visible)))+ (finally+ (t2/delete! :model/User :email email))))))++(deftest crash-mid-load-repaired-on-next-boot-test+ ;; Pins the self-healing premise the detached lock's safety argument leans on: a crash in the+ ;; middle of the serdes load must be repaired by the next boot. Two halves: the content+ ;; checksum must not advance past a load that did not complete (so the next boot re-enters the+ ;; load), and the next boot's un-stubbed pipeline must actually converge back to the healthy+ ;; state.+ (with-audit-db-restoration!+ (let [healthy-checksum (audit/last-analytics-checksum)]+ ;; make the content look new again, as an upgrade would+ (audit/last-analytics-checksum! 0)+ ;; "Boot 1": the serdes load dies partway through.+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"killed mid-load"+ (with-redefs [serialization.cmd/v2-load-internal!+ (fn [& _] (throw (ex-info "killed mid-load" {})))]+ (mbc/ensure-audit-db-installed!))))+ (testing "the content checksum does not advance past an incomplete load"+ (is (= 0 (audit/last-analytics-checksum))))+ ;; "Boot 2": the full, un-stubbed pipeline.+ (mbc/ensure-audit-db-installed!)+ (testing "the next boot re-runs the load and converges to the healthy state"+ (is (= healthy-checksum (audit/last-analytics-checksum)))+ (is (pos? (t2/count :model/Card :database_id audit/audit-db-id)))))))diff --git a/src/metabase/app_db/cluster_lock.clj b/src/metabase/app_db/cluster_lock.cljindex 8862a11e29b..23778d75756 100644--- a/src/metabase/app_db/cluster_lock.clj+++ b/src/metabase/app_db/cluster_lock.clj@@ -104,17 +104,35 @@(with-open [stmt (prepare-statement conn lock-name-str timeout mode)result-set (.executeQuery stmt)](when-not (.next result-set)- ;; this record will not be visible until the tx commits, so there's no need to lock it- ;; we instead rely on concurrent threads having constraint violation trying to insert their own record- (t2/query-one {:insert-into [:metabase_cluster_lock]- :columns [:lock_name]- :values [[lock-name-str]]})))+ ;; this record will not be visible until the tx commits, so there's no need to lock it; concurrent+ ;; inserters get a constraint violation and retry. Raw JDBC because the insert must run on `conn`+ ;; (under a detached lock ambient resolution would hand it a different connection) and needs the+ ;; same query timeout as the SELECT — concurrent first-time inserters block on the winner's+ ;; uncommitted unique-index entry+ (let [[sql] (mdb.query/compile {:insert-into [:metabase_cluster_lock]+ :columns [:lock_name]+ :values [[[:raw "?"]]]})]+ (with-open [insert-stmt (.prepareStatement conn ^String sql)]+ (doto insert-stmt+ (.setQueryTimeout timeout)+ (.setString 1 lock-name-str))+ (.executeUpdate insert-stmt)))))(log/debugf "Obtained cluster lock: %s (%s)" lock-name-str mode))+(def ^:private ^:dynamic *detached-locks-held*+ "Lock-name strings currently held by [[do-with-detached-cluster-lock]] in this dynamic scope. A detached+ hold lives on a dedicated connection, so re-acquiring the same name from this scope — detached or+ transactional — would block against our own row lock; acquisitions fail fast instead."+ #{})+(defn- do-with-cluster-locks*"Acquire all `locks` (each a `{:lock-name-str, :mode}` map) inside a singletransaction, then run `thunk`."[locks timeout-seconds thunk]+ (doseq [{:keys [lock-name-str]} locks]+ (when (*detached-locks-held* lock-name-str)+ (throw (ex-info "Cluster lock is already held detached in this scope"+ {:lock-name lock-name-str}))))(t2/with-transaction [conn](doseq [{:keys [lock-name-str mode]} locks](acquire-lock-row! conn lock-name-str timeout-seconds mode))@@ -270,13 +288,79 @@(defmacro with-cluster-lock"Run `body` in a transaction that tries to take a lock from the metabase_cluster_lock table ofthe specified name to coordinate concurrency with other metabase instances sharing the appdb.+ (On an h2 appdb the lock is an in-process read-write lock and no transaction is involved.)`lock-options` may be a lock-name keyword, or an options map`{:lock, :locks, :mode, :timeout-seconds, :retry-config, :retry-transient?}` —- see [[do-with-cluster-lock]] for the full description of each."+ see [[do-with-cluster-lock]] for the full description of each.++ For long-running work whose appdb writes should commit incrementally instead of riding the lock's+ transaction, see [[with-detached-cluster-lock]]."([lock-options & body]`(do-with-cluster-lock ~lock-options (fn [] ~@body))))+(mu/defn do-with-detached-cluster-lock+ "Impl for [[with-detached-cluster-lock]]."+ [{:keys [lock timeout-seconds retry-config]+ :or {timeout-seconds cluster-lock-timeout-seconds}}+ :- [:map+ [:lock :keyword]+ [:timeout-seconds {:optional true} :int]+ [:retry-config {:optional true} [:ref ::retry/retry-overrides]]]+ thunk :- ifn?]+ (let [lock-name-str (keyword->lock-name-str lock)]+ (when (*detached-locks-held* lock-name-str)+ (throw (ex-info "Cluster lock is already held detached in this scope"+ {:lock-name lock-name-str})))+ (if (= (mdb.connection/db-type) :h2)+ ;; the h2 in-process lock never holds a transaction, so it is already 'detached'+ (do-with-h2-cluster-locks* [{:lock-name-str lock-name-str :mode :exclusive}]+ #(binding [*detached-locks-held* (conj *detached-locks-held* lock-name-str)]+ (thunk)))+ (mdb.connection/with-unshared-connection [conn]+ (.setAutoCommit ^Connection conn false)+ (let [config (merge default-retry-config retry-config)]+ (try+ (retry/with-retry config+ ;; clear the aborted transaction a failed previous attempt leaves behind+ (.rollback ^Connection conn)+ (acquire-lock-row! conn lock-name-str timeout-seconds :exclusive))+ (catch Throwable e+ (if (retryable? e)+ (throw (ex-info (str "Failed to obtain cluster lock: " lock-name-str)+ {:lock-names [lock-name-str]+ :retries (:max-retries config)}+ e))+ (throw e)))))+ ;; the body runs outside the retry above, so its errors are never retried (its work has already+ ;; committed) and never mistaken for acquisition failure+ (let [result (binding [*detached-locks-held* (conj *detached-locks-held* lock-name-str)]+ (thunk))]+ ;; releases the row lock; on a body throw the pool's check-in rollback releases it instead+ (.commit ^Connection conn)+ result)))))++(defmacro with-detached-cluster-lock+ "Like [[with-cluster-lock]], but holds the (exclusive) lock row on a dedicated connection+ (see [[metabase.app-db.connection/with-unshared-connection]]) while `body` runs on ordinary pooled+ connections: `body`'s appdb work commits incrementally in its own short transactions instead of riding+ one long transaction that holds the lock. The lock is released when `body` completes (commit) or+ throws (pool check-in rollback) — but `body`'s already-committed work is NOT rolled back by a throw,+ so only use this for long-running work that is idempotent/self-healing (the audit boot pipeline).++ Not reentrant: re-acquiring a lock this scope already holds detached — in either detached or+ transactional form — throws instead of self-deadlocking against the dedicated connection. (The+ reverse — requesting a detached hold on a lock this scope holds transactionally — is not detected+ and times out; don't.) Taking *different* locks inside `body` works normally. Hold duration is+ bounded by the connection pool's limits: c3p0's `unreturnedConnectionTimeout`, when configured,+ destroys the connection and silently releases the lock, and with the appdb checkout timeout set to+ 0 (wait forever) a body blocked on pool checkout holds the lock indefinitely.++ `opts` is `{:lock, :timeout-seconds, :retry-config}` as in [[with-cluster-lock]]; retries apply to+ lock acquisition only, never to `body`."+ [opts & body]+ `(do-with-detached-cluster-lock ~opts (fn [] ~@body)))+(def card-statistics-lock"A shared keyword that any method doing a batch update of card statistics can use for the cluster lock"::statistics-lock)diff --git a/src/metabase/app_db/connection.clj b/src/metabase/app_db/connection.cljindex 575696f8f88..e6496400b0a 100644--- a/src/metabase/app_db/connection.clj+++ b/src/metabase/app_db/connection.clj@@ -300,6 +300,44 @@(run-after-commit-callbacks! callbacks))result)))+;;;; Unshared connections+;;;;+;;;; A connection that holds fragile long-lived state -- a streaming result set (postgres portal), a row+;;;; lock held while other work runs -- must not be picked up by ambient connection reuse: a borrower that+;;;; inherits it via [[toucan2.connection/*current-connectable*]] and commits destroys that state (see+;;;; #78238 and the revert of #76645). [[toucan2.connection/unshared-connection!]] makes toucan use such a+;;;; connection only when passed it explicitly, and never advertise it ambiently -- ambient work inside+;;;; `body` resolves to the default connectable and runs on other pooled connections.++(defn do-with-unshared-connection+ "Impl for [[with-unshared-connection]]."+ [f]+ ;; through *application-db* (not the raw data source) so its connection read-lock gate applies+ (with-open [conn (.getConnection *application-db*)]+ (f (t2.conn/unshared-connection! conn))))++(defmacro with-unshared-connection+ "Execute `body` with `conn-binding` bound to a fresh app-db connection that ambient connection reuse can+ never pick up: toucan runs queries and transactions on it when `body` passes it explicitly, but never+ binds it as [[toucan2.connection/*current-connectable*]], so toucan calls that resolve their connection+ ambiently -- including mid-reduction of a long-running query on this connection -- run (and commit) on+ other pooled connections.++ In every other respect this is an ordinary JDBC connection, configured by `body`: e.g. call+ `.setAutoCommit false` for a streaming result set (postgres portal/cursor) or a held row lock. It is+ closed when `body` ends; the pool resets its state (rolling back any unresolved transaction) on check-in.++ Caveat: do not pass the unshared connection to an explicit `t2/with-transaction` while an ambient+ appdb transaction is open on this thread. Nesting depth is tracked per thread, not per connection, so+ the inner call would take a savepoint on this connection that no commit ever follows, and its work+ would silently roll back at check-in.++ (mdb/with-unshared-connection [conn]+ (.setAutoCommit conn false)+ (reduce rf init (t2/reducible-query conn a-huge-query)))"+ [[conn-binding] & body]+ `(do-with-unshared-connection (fn [~conn-binding] ~@body)))+(methodical/defmethod t2.pipeline/transduce-query :before :default"Make sure application database calls are not done inside core.async dispatch pool threads. This is done relativelyearly in the pipeline so the stacktrace when this fails isn't super enormous."diff --git a/src/metabase/app_db/core.clj b/src/metabase/app_db/core.cljindex 57d32570b3e..2809ef4242b 100644--- a/src/metabase/app_db/core.clj+++ b/src/metabase/app_db/core.clj@@ -42,7 +42,8 @@in-transaction?quoting-styleunique-identifier- transaction-state]+ transaction-state+ with-unshared-connection][mdb.connection-pool-setuprecent-activity?][mdb.data-sourcediff --git a/test/metabase/app_db/cluster_lock_test.clj b/test/metabase/app_db/cluster_lock_test.cljindex bdc5728d012..78eb13439aa 100644--- a/test/metabase/app_db/cluster_lock_test.clj+++ b/test/metabase/app_db/cluster_lock_test.clj@@ -5,6 +5,7 @@[metabase.app-db.cluster-lock :as sut][metabase.app-db.core :as mdb][metabase.app-db.transient-error :as transient-error]+ [metabase.test :as mt][metabase.test.fixtures :as fixtures][metabase.util :as u][toucan2.core :as t2])@@ -92,6 +93,13 @@(future (Thread/sleep 500) (a/>!! fin-chan :done))(is (nil? (sut/with-cluster-lock ::test-lock (Thread/sleep 1))))))))+(defn- do-with-lock-opts+ "Route on the test-only :detached? key so every helper call site can exercise either entry point."+ [lock-opts thunk]+ (if (:detached? lock-opts)+ (sut/do-with-detached-cluster-lock (dissoc lock-opts :detached?) thunk)+ (sut/do-with-cluster-lock lock-opts thunk)))+(defn- run-with-lock"Run `thunk` with the given lock opts on a future. Returns a map of`{:entered, :release, :done}` latches/promise the caller can use to@@ -102,9 +110,10 @@done (promise)](future(try- (sut/with-cluster-lock lock-opts- (.countDown entered)- (.await release))+ (do-with-lock-opts lock-opts+ (fn []+ (.countDown entered)+ (.await release)))(deliver done :ok)(catch Throwable e(deliver done [:err (ex-message e)]))))@@ -116,8 +125,7 @@(let [entered (promise)acquired (future(try- (sut/with-cluster-lock lock-opts- (deliver entered :yes))+ (do-with-lock-opts lock-opts (fn [] (deliver entered :yes)))(catch Throwable _(deliver entered :err))))result (deref entered timeout-ms :timeout)]@@ -226,3 +234,133 @@(is (= [[:a :failed :timeout][:b :done]]@results))))))++(deftest detached-lock-basic-test+ (testing "returns the body's value and works non-concurrently"+ (is (= :ok (sut/with-detached-cluster-lock {:lock ::detached-basic} :ok)))))++(deftest detached-lock-body-commits-incrementally-test+ (let [email (mt/random-email)]+ (try+ (testing "body work commits as it goes and survives a later throw in the body"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"+ (sut/with-detached-cluster-lock {:lock ::detached-commit}+ (t2/insert! :model/User (assoc (mt/with-temp-defaults :model/User) :email email))+ (throw (ex-info "boom" {})))))+ (is (t2/exists? :model/User :email email)))+ (finally+ (t2/delete! :model/User :email email)))))++(deftest detached-lock-reentry-throws-test+ (testing "re-acquiring a held detached lock throws instead of self-deadlocking"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"already held detached"+ (sut/with-detached-cluster-lock {:lock ::detached-reentry}+ (sut/with-detached-cluster-lock {:lock ::detached-reentry}+ :nope)))))+ ;; the transactional-path guard lives in the row-lock impl; h2 takes its in-process reentrant lock+ (when (not= (mdb/db-type) :h2)+ (testing "a transactional acquisition of a lock this scope holds detached also throws"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"already held detached"+ (sut/with-detached-cluster-lock {:lock ::detached-reentry}+ (sut/with-cluster-lock ::detached-reentry+ :nope)))))+ (testing "taking a different lock inside a detached body works normally"+ (is (= :ok (sut/with-detached-cluster-lock {:lock ::detached-reentry}+ (sut/with-cluster-lock ::detached-reentry-other+ :ok)))))))++(deftest detached-lock-mutual-exclusion-test+ ;; h2 uses the in-process rw-lock path; real row-lock semantics only.+ (when (not= (mdb/db-type) :h2)+ ;; warm up the row first so we're not racing on the initial INSERT+ (sut/with-detached-cluster-lock {:lock ::detached-mutex} :warm)+ (let [held (run-with-lock {:lock ::detached-mutex :detached? true})]+ (is (.await ^CountDownLatch (:entered held) 3 TimeUnit/SECONDS))+ (testing "a detached holder blocks another detached acquirer"+ (is (not (acquirable-within? {:lock ::detached-mutex :detached? true+ :timeout-seconds 1 :retry-config {:max-retries 0}}+ 3000))))+ (testing "a detached holder blocks a transactional acquirer"+ (is (not (acquirable-within? {:lock ::detached-mutex+ :timeout-seconds 1 :retry-config {:max-retries 0}}+ 3000))))+ (.countDown ^CountDownLatch (:release held))+ (is (= :ok (deref (:done held) 3000 :timeout)))+ (testing "released once the body completes"+ (is (acquirable-within? {:lock ::detached-mutex :detached? true} 3000))))))++(defn- query-canceled-exception+ "Build a query-canceled SQLException for the current appdb type. Mirrors the codes in+ [[metabase.app-db.query-cancelation]] so the test exercises the real db-type path."+ []+ (case (mdb/db-type)+ :postgres (SQLException. "ERROR: canceling statement due to user request" "57014")+ :mysql (SQLException. "Statement cancelled due to timeout or client request" "70100" 1317)))++(deftest detached-lock-body-error-not-retried-test+ ;; h2 takes an in-process lock and bypasses the retry path entirely, so this only exercises real row locks.+ (when (not= (mdb/db-type) :h2)+ ;; warm up the row first so we're not racing on the initial INSERT+ (sut/with-detached-cluster-lock {:lock ::detached-body-error} :warm)+ (let [attempts (atom 0)+ e (is (thrown? Throwable+ (sut/with-detached-cluster-lock {:lock ::detached-body-error+ :retry-config {:max-retries 2 :delay-ms 1}}+ (swap! attempts inc)+ (throw (query-canceled-exception)))))]+ (testing "an acquisition-retryable error raised by the body must not re-run the body: its work commits+ incrementally, so a re-run double-applies it"+ (is (= 1 @attempts)))+ (testing "a body error must not be rewrapped as an acquisition failure — the lock WAS obtained"+ (is (not (re-find #"Failed to obtain cluster lock" (str (ex-message e)))))))))++(deftest detached-lock-nested-lock-timeout-not-retried-test+ ;; The audit-pipeline shape: a detached body that takes another cluster lock inside (serdes takes the+ ;; permissions lock when inserting a Database). When the inner acquisition times out, the canceled+ ;; SELECT stays in the cause chain of the inner "Failed to obtain cluster lock" wrapper, so the outer+ ;; machinery must not read it as a failure to acquire the OUTER lock: re-running the body would+ ;; double-apply its committed work, and attributing the error to the outer lock makes callers that+ ;; treat outer-lock contention as benign (ensure-audit-db-installed!) swallow a genuine body failure.+ (when (not= (mdb/db-type) :h2)+ (sut/with-cluster-lock ::nested-timeout-inner :warm)+ (sut/with-detached-cluster-lock {:lock ::nested-timeout-outer} :warm)+ (let [held (run-with-lock {:lock ::nested-timeout-inner})]+ (try+ (is (.await ^CountDownLatch (:entered held) 3 TimeUnit/SECONDS))+ (let [attempts (atom 0)+ e (is (thrown? Throwable+ (sut/with-detached-cluster-lock {:lock ::nested-timeout-outer+ :retry-config {:max-retries 2 :delay-ms 1}}+ (swap! attempts inc)+ (sut/with-cluster-lock {:lock ::nested-timeout-inner+ :timeout-seconds 1 :retry-config {:max-retries 0}}+ :never))))]+ (testing "the incrementally-committing outer body must run exactly once"+ (is (= 1 @attempts)))+ (testing "the failure must not be attributed to the outer lock the body already holds"+ (is (not= [(u/qualified-name ::nested-timeout-outer)]+ (:lock-names (ex-data e))))))+ (finally+ (.countDown ^CountDownLatch (:release held))+ (is (= :ok (deref (:done held) 3000 :timeout))))))))++(deftest detached-lock-released-after-body-throw-test+ ;; The throw path has no explicit rollback: it relies on with-open closing the dedicated+ ;; connection and the pool rolling back the unresolved transaction on check-in. If that ever+ ;; stopped happening (pool swap or misconfiguration), a boot-time audit failure would leave the+ ;; row lock held until the connection ages out, silently stalling other nodes' boots — the class+ ;; of bug the detached lock exists to fix. Release after *normal* completion is covered by+ ;; detached-lock-mutual-exclusion-test; this pins the throw path.+ (when (not= (mdb/db-type) :h2)+ ;; warm up the row first so we're not racing on the initial INSERT+ (sut/with-detached-cluster-lock {:lock ::detached-throw-release} :warm)+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"+ (sut/with-detached-cluster-lock {:lock ::detached-throw-release}+ (throw (ex-info "boom" {})))))+ (testing "the row lock is released once the throwing body's connection is checked back in"+ (is (acquirable-within? {:lock ::detached-throw-release :detached? true+ :timeout-seconds 1 :retry-config {:max-retries 0}}+ 3000))+ (is (acquirable-within? {:lock ::detached-throw-release+ :timeout-seconds 1 :retry-config {:max-retries 0}}+ 3000)))))diff --git a/test/metabase/app_db/connection_test.clj b/test/metabase/app_db/connection_test.cljindex e6d5eee3e1c..ad576573450 100644--- a/test/metabase/app_db/connection_test.clj+++ b/test/metabase/app_db/connection_test.clj@@ -5,7 +5,8 @@[metabase.test :as mt][metabase.test.fixtures :as fixtures][toucan2.connection :as t2.connection]- [toucan2.core :as t2])+ [toucan2.core :as t2]+ [toucan2.jdbc.options :as t2.jdbc.options])(:import(java.sql Connection)(java.util.concurrent Semaphore)))@@ -339,3 +340,90 @@(throw (ex-info "force savepoint rollback" {})))))(is (= {:outer-key "outer-val"} @mdb.connection/*transaction-state*)"data stashed by a rolled-back nested transaction is discarded from transaction-state")))++(deftest unshared-connection-not-bound-during-use-test+ (mdb.connection/with-unshared-connection [conn]+ (testing "explicitly passing the unshared connection works, but it is never bound as *current-connectable*"+ (let [rows (reduce (fn [acc row]+ ;; the contract is nil, not merely "not this connection": while an unshared+ ;; connection is in use there is no ambient connection at all+ (is (nil? t2.connection/*current-connectable*)+ "ambient resolution must be suppressed mid-reduction")+ (conj acc (into {} row)))+ []+ (t2/reducible-query conn ["select 1 as x"]))]+ (is (= 1 (count rows)))))+ (testing "a transaction opened while the unshared connection is in use gets its own connection"+ (reduce (fn [_ _row]+ (t2/with-transaction [tx-conn]+ (is (not (identical? tx-conn conn)))))+ nil+ (t2/reducible-query conn ["select 1 as x"])))))++(deftest unshared-connection-borrower-work-commits-independently-test+ (let [email (mt/random-email)]+ (try+ (testing "toucan work while the unshared connection is in use commits on its own connection"+ (mdb.connection/with-unshared-connection [conn]+ ;; open a transaction on the unshared connection that is never committed: if the insert below+ ;; wrongly rode this connection, it would be rolled back at pool check-in and the user would not exist+ (.setAutoCommit ^Connection conn false)+ (reduce (fn [_ _row]+ (t2/insert! :model/User (assoc (mt/with-temp-defaults :model/User) :email email)))+ nil+ (t2/reducible-query conn ["select 1 as x"])))+ (is (t2/exists? :model/User :email email)))+ (finally+ (t2/delete! :model/User :email email)))))++(deftest unshared-connection-postgres-portal-survival-test+ ;; postgres streams a result set through a portal that dies if anything commits on the connection+ ;; mid-iteration -- the failure mode that forced the revert of #76645. Other appdbs buffer, so there is+ ;; nothing to kill.+ (when (= (mdb.connection/db-type) :postgres)+ (let [portal-death? (fn [e]+ (loop [e e]+ (cond+ (nil? e) false+ (re-find #"portal" (str (ex-message e))) true+ :else (recur (ex-cause e)))))+ stream! (fn [^Connection conn]+ ;; fetch-size + autocommit off = pg streams through a portal+ (.setAutoCommit conn false)+ (binding [t2.jdbc.options/*options* {:fetch-size 50}]+ (reduce (fn [n _row]+ ;; unrelated toucan work mid-stream, resolved ambiently, that commits+ (when (= n 30)+ (t2/with-transaction [_]+ (t2/query ["select 1"])))+ (inc n))+ 0+ (t2/reducible-query conn ["select g from generate_series(1, 1000) g"]))))]+ (testing "control: an ordinary connection is ambiently visible mid-reduction; the borrower commit kills the portal"+ (with-open [^Connection raw (.getConnection (mdb.connection/data-source))]+ (let [e (try (stream! raw) nil (catch Throwable e e))]+ (is (portal-death? e)))+ (.rollback raw)+ (.setAutoCommit raw true)))+ (testing "an unshared connection is invisible to the borrower; the portal survives"+ (mdb.connection/with-unshared-connection [conn]+ (is (= 1000 (stream! conn))))))))++(deftest unshared-connection-uncommitted-work-rolls-back-on-close-test+ ;; the with-unshared-connection docstring promises the pool resets state on check-in, rolling+ ;; back any unresolved transaction — the same mechanism the detached cluster lock relies on to+ ;; release its row locks after a body throw. Pin the rollback itself.+ (let [lock-name "connection-test/unshared-rollback"]+ (try+ (mdb.connection/with-unshared-connection [conn]+ (.setAutoCommit ^Connection conn false)+ (t2/query-one conn {:insert-into :metabase_cluster_lock+ :columns [:lock_name]+ :values [[lock-name]]})+ (testing "the write is visible on the unshared connection before close"+ (is (= 1 (count (t2/query conn ["select * from metabase_cluster_lock where lock_name = ?"+ lock-name]))))))+ (testing "the uncommitted write vanished at pool check-in"+ (is (not (t2/exists? :metabase_cluster_lock :lock_name lock-name))))+ (finally+ (t2/delete! :metabase_cluster_lock :lock_name lock-name)))))