Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Make rollback-only app DB transactions actually roll back (#80065)

Chris Truter0be297287274
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 8dc9ec8c12e..aa1903ead73 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1451,6 +1451,7 @@
:api #{metabase.metrics.api
metabase.metrics.core}
:uses #{api
+ app-db
events
lib
lib-be
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/api/util_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/api/util_test.clj
index 944a9a867fc..361b5ee462b 100644
--- a/enterprise/backend/test/metabase_enterprise/sandbox/api/util_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sandbox/api/util_test.clj
@@ -16,12 +16,15 @@
(doall @data-perms/*sandboxes-for-user*)
;; make the cache wrong
(t2/delete! :model/Sandbox :group_id (:id &group))
- ;; subsequent calls should still use the cache, and not hit the DB at all
- (t2/with-call-count [call-count]
- (is (sandbox.api.util/sandboxed-user?))
- (is (zero? (call-count)))
- (is (= 1 (count (sandbox.api.util/enforced-sandboxes-for-tables [(mt/id :venues)]))))
- (is (zero? (call-count))))))))
+ ;; Resolve this outside the call-count window. Within a `with-temp` transaction, test-data ID lookups
+ ;; intentionally bypass their cache, so `(mt/id :venues)` executes another query.
+ (let [venues-id (mt/id :venues)]
+ ;; Subsequent calls should use the cache without querying the DB.
+ (t2/with-call-count [call-count]
+ (is (sandbox.api.util/sandboxed-user?))
+ (is (zero? (call-count)))
+ (is (= 1 (count (sandbox.api.util/enforced-sandboxes-for-tables [venues-id]))))
+ (is (zero? (call-count)))))))))
(deftest enforce-sandbox?-test
(testing "If a user is in a single group with a sandbox, the sandbox should be enforced"
diff --git a/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj b/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj
index 2fa48a864dc..2cee74007f7 100644
--- a/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj
@@ -9,10 +9,17 @@
[metabase-enterprise.search.scoring :as ee-scoring]
[metabase.search.appdb.scoring-test :as appdb.scoring-test]
[metabase.search.in-place.scoring :as scoring]
+ [metabase.search.test-util :as search.tu]
[metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
[metabase.util.json :as json])
(:import [java.time Instant]))
+;; Initialize before any tests run. The app DB index tests below write search-index bookkeeping before anything
+;; else accesses the app DB; a later lazy initialization, triggered by `mt/user->id` in `search-results`, would
+;; recreate the H2 test DB while a test is running.
+(use-fixtures :once (fixtures/initialize :db :test-users))
+
(set! *warn-on-reflection* true)
(deftest ^:parallel verified-score-test
@@ -188,29 +195,35 @@
(appdb.scoring-test/search-results* "card")))))))
(deftest transforms-user-recency-test
- (mt/with-premium-features #{:transforms-basic :hosting}
- (let [user-id (mt/user->id :crowberto)
- now (Instant/now)
- recent-view (fn [model-id timestamp]
- {:model "card"
- :model_id model-id
- :user_id user-id
- :timestamp timestamp})]
- (mt/with-temp [:model/Card {c1 :id} {}
- :model/Card {c2 :id} {}
- :model/Transform {t1 :id} {:name "test transform"
- :source {:type "query"
- :query (mt/native-query {:query "SELECT 1"})}
- :target {:type "table"
- :name (mt/random-name)}}
- :model/RecentViews _ (recent-view c1 now)]
- (appdb.scoring-test/with-index-contents
- [{:model "card" :id c1 :name "test card recent"}
- {:model "card" :id c2 :name "test card unseen"}
- {:model "transform" :id t1 :name "test transform" :source_type "mbql"}]
- (testing "Transforms get a hardcoded 1-day recency (between recently viewed card and never viewed card)"
- (is (= [["card" c1 "test card recent"]
- ["transform" t1 "test transform"]
- ["card" c2 "test card unseen"]]
- (appdb.scoring-test/search-results :user-recency "test" {:current-user-id user-id
- :context :metabot})))))))))
+ ;; Create the temporary index table before `with-temp` opens its transaction. On H2 and MySQL, DDL on the
+ ;; ambient connection would implicitly commit that transaction, preventing rollback and leaking rows into later
+ ;; tests. The nested index-table scope below reuses this table. Materialize the test-data Database first as well;
+ ;; otherwise its ingestion would enter the temporary index and appear as an extra search result.
+ (mt/id)
+ (search.tu/with-temp-index-table
+ (mt/with-premium-features #{:transforms-basic :hosting}
+ (let [user-id (mt/user->id :crowberto)
+ now (Instant/now)
+ recent-view (fn [model-id timestamp]
+ {:model "card"
+ :model_id model-id
+ :user_id user-id
+ :timestamp timestamp})]
+ (mt/with-temp [:model/Card {c1 :id} {}
+ :model/Card {c2 :id} {}
+ :model/Transform {t1 :id} {:name "test transform"
+ :source {:type "query"
+ :query (mt/native-query {:query "SELECT 1"})}
+ :target {:type "table"
+ :name (mt/random-name)}}
+ :model/RecentViews _ (recent-view c1 now)]
+ (appdb.scoring-test/with-index-contents
+ [{:model "card" :id c1 :name "test card recent"}
+ {:model "card" :id c2 :name "test card unseen"}
+ {:model "transform" :id t1 :name "test transform" :source_type "mbql"}]
+ (testing "Transforms get a hardcoded 1-day recency (between recently viewed card and never viewed card)"
+ (is (= [["card" c1 "test card recent"]
+ ["transform" t1 "test transform"]
+ ["card" c2 "test card unseen"]]
+ (appdb.scoring-test/search-results :user-recency "test" {:current-user-id user-id
+ :context :metabot}))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/test_util.clj b/enterprise/backend/test/metabase_enterprise/serialization/test_util.clj
index 9e385de3ea0..b6df445f635 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/test_util.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/test_util.clj
@@ -10,6 +10,7 @@
[metabase.models.visualization-settings :as mb.viz]
[metabase.test :as mt]
[metabase.test.data :as data]
+ [metabase.test.data.impl :as data.impl]
[metabase.util :as u]
[metabase.util.files :as u.files]
[next.jdbc]
@@ -78,7 +79,10 @@
(with-open [_conn (.getConnection data-source)]
(next.jdbc/execute! data-source ["RUNSCRIPT FROM ?" (str @data/h2-app-db-script)])
(with-db data-source (mdb/finish-db-setup!))
- (f data-source)))))
+ ;; These app DBs should contain only data loaded by the test. Prevent `with-temp` from prewarming the
+ ;; test-data Database, which would add an unexpected Database to serialization extracts.
+ (binding [data.impl/*skip-dataset-prewarm?* true]
+ (f data-source))))))
(defn do-with-dbs
"Given a function with the given arity, create an in-memory db for each argument and then call the fn with these dbs"
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
index c0783a79d09..cd4ef26d4bf 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
@@ -147,54 +147,65 @@
(deftest search-filters-transform-source-types-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/dataset transforms-dataset/transforms-test
- (let [search-term (str "transform-search-" (u/generate-nano-id))
- query-name (str search-term "-query")
- python-name (str search-term "-python")]
- (mt/with-temp [:model/Transform {query-id :id} (assoc (query-transform-payload (str "target_" (u/generate-nano-id)))
- :name query-name)
- :model/Transform {python-id :id} (assoc (python-transform-map (str "target_" (u/generate-nano-id)))
- :name python-name)]
- (search.tu/with-appdb-search-and-legacy-search
- (testing "no transforms feature"
- (mt/with-premium-features #{}
- (is (empty? (search-transform-ids search-term)))))
- (testing "transforms only"
- (mt/with-premium-features #{:transforms-basic :hosting}
- (is (= #{query-id} (search-transform-ids search-term)))))
- (testing "transforms and transforms-python"
- (mt/with-premium-features #{:transforms-basic :transforms-python :hosting}
- (is (= #{query-id python-id} (search-transform-ids search-term)))))))))))
+ ;; the temp index table is created here, before `with-temp` opens its transaction: creating (and
+ ;; especially dropping) it inside would run DDL on the ambient connection, which on H2/MySQL
+ ;; implicitly commits the transaction, so its rollback could not take the rows back and the
+ ;; Transforms would leak to every later test that counts them. The index scope nested inside
+ ;; `with-appdb-search-and-legacy-search` reuses this one rather than creating its own. The
+ ;; `-if-supported` variant keeps the legacy-search leg running on app dbs that cannot hold an index.
+ (search.tu/with-temp-index-table-if-supported
+ (let [search-term (str "transform-search-" (u/generate-nano-id))
+ query-name (str search-term "-query")
+ python-name (str search-term "-python")]
+ (mt/with-temp [:model/Transform {query-id :id} (assoc (query-transform-payload (str "target_" (u/generate-nano-id)))
+ :name query-name)
+ :model/Transform {python-id :id} (assoc (python-transform-map (str "target_" (u/generate-nano-id)))
+ :name python-name)]
+ (search.tu/with-appdb-search-and-legacy-search
+ (testing "no transforms feature"
+ (mt/with-premium-features #{}
+ (is (empty? (search-transform-ids search-term)))))
+ (testing "transforms only"
+ (mt/with-premium-features #{:transforms-basic :hosting}
+ (is (= #{query-id} (search-transform-ids search-term)))))
+ (testing "transforms and transforms-python"
+ (mt/with-premium-features #{:transforms-basic :transforms-python :hosting}
+ (is (= #{query-id python-id} (search-transform-ids search-term))))))))))))
(deftest search-filtering-updates-with-feature-flips-without-reindex-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/dataset transforms-dataset/transforms-test
- (let [search-term (str "transform-search-" (u/generate-nano-id))
- query-name (str search-term "-query")
- python-name (str search-term "-python")]
- (mt/with-temp [:model/Transform {query-id :id} (assoc (query-transform-payload (str "target_" (u/generate-nano-id)))
- :name query-name)
- :model/Transform {python-id :id} (assoc (python-transform-map (str "target_" (u/generate-nano-id)))
- :name python-name)]
- (search.tu/with-appdb-search-and-legacy-search
- (mt/with-premium-features #{:transforms-basic :transforms-python :hosting}
- (is (= #{query-id python-id} (search-transform-ids search-term))))
- (mt/with-premium-features #{:transforms-basic :hosting}
- (is (= #{query-id} (search-transform-ids search-term))))
- (mt/with-premium-features #{}
- (is (empty? (search-transform-ids search-term))))))))))
+ ;; see search-filters-transform-source-types-test for why the index scope sits outside `with-temp`
+ (search.tu/with-temp-index-table-if-supported
+ (let [search-term (str "transform-search-" (u/generate-nano-id))
+ query-name (str search-term "-query")
+ python-name (str search-term "-python")]
+ (mt/with-temp [:model/Transform {query-id :id} (assoc (query-transform-payload (str "target_" (u/generate-nano-id)))
+ :name query-name)
+ :model/Transform {python-id :id} (assoc (python-transform-map (str "target_" (u/generate-nano-id)))
+ :name python-name)]
+ (search.tu/with-appdb-search-and-legacy-search
+ (mt/with-premium-features #{:transforms-basic :transforms-python :hosting}
+ (is (= #{query-id python-id} (search-transform-ids search-term))))
+ (mt/with-premium-features #{:transforms-basic :hosting}
+ (is (= #{query-id} (search-transform-ids search-term))))
+ (mt/with-premium-features #{}
+ (is (empty? (search-transform-ids search-term)))))))))))
(deftest search-api-transform-models-empty-without-feature-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/with-premium-features #{:hosting}
(mt/dataset transforms-dataset/transforms-test
- (let [search-term (str "transform-search-" (u/generate-nano-id))
- query-name (str search-term "-query")]
- (mt/with-temp [:model/Transform {query-id :id} (assoc (query-transform-payload (str "target_" (u/generate-nano-id)))
- :name query-name)]
- (search.tu/with-appdb-search-and-legacy-search
- (let [ids (search-api-transform-ids :crowberto search-term)]
- (is (empty? ids))
- (is (not (contains? ids query-id)))))))))))
+ ;; see search-filters-transform-source-types-test for why the index scope sits outside `with-temp`
+ (search.tu/with-temp-index-table-if-supported
+ (let [search-term (str "transform-search-" (u/generate-nano-id))
+ query-name (str search-term "-query")]
+ (mt/with-temp [:model/Transform {query-id :id} (assoc (query-transform-payload (str "target_" (u/generate-nano-id)))
+ :name query-name)]
+ (search.tu/with-appdb-search-and-legacy-search
+ (let [ids (search-api-transform-ids :crowberto search-term)]
+ (is (empty? ids))
+ (is (not (contains? ids query-id))))))))))))
;;; ------------------------------------------------------------
;;; Run List Sorting - TODO [OSS] - move this to OSS
diff --git a/src/metabase/app_db/cluster_lock.clj b/src/metabase/app_db/cluster_lock.clj
index 3eded7e8780..0a02a2271cd 100644
--- a/src/metabase/app_db/cluster_lock.clj
+++ b/src/metabase/app_db/cluster_lock.clj
@@ -22,6 +22,7 @@
[metabase.app-db.query :as mdb.query]
[metabase.app-db.query-cancelation :as app-db.query-cancelation]
[metabase.app-db.transient-error :as transient-error]
+ [metabase.config.core :as config]
[metabase.util.connection :as u.connection]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
@@ -37,16 +38,21 @@
(def ^:private cluster-lock-timeout-seconds 1)
+(defn- duplicate-lock-row?
+ "Whether `e` indicates that another transaction inserted this lock row first."
+ [^Throwable e]
+ (or (instance? SQLIntegrityConstraintViolationException e)
+ (instance? SQLIntegrityConstraintViolationException (ex-cause e))
+ ;; PostgreSQL reports this as a generic PSQLException, so fall back to checking the message.
+ (some-> (ex-message e) (str/includes? "duplicate key value violates unique constraint \"metabase_cluster_lock_pkey\""))))
+
(defn- retryable?
"Errors that mean we failed to *acquire* the lock and should retry acquisition."
[^Throwable e]
;; We can retry getting the cluster lock if either we tried to concurrently insert the pk
;; for the lock resulting in a SQLIntegrityConstraintViolationException or if the query
;; was cancelled via timeout waiting to get the SELECT FOR UPDATE lock
- (or (instance? SQLIntegrityConstraintViolationException e)
- (instance? SQLIntegrityConstraintViolationException (ex-cause e))
- ;; Postgres does just uses PSQLException, so we need to fall back to checking the message.
- (some-> (ex-message e) (str/includes? "duplicate key value violates unique constraint \"metabase_cluster_lock_pkey\""))
+ (or (duplicate-lock-row? e)
(true? (::acquisition-timeout (ex-data e)))
(app-db.query-cancelation/query-canceled-exception? (mdb.connection/db-type) e)))
@@ -138,30 +144,146 @@
(when previous
(set-session-lock-wait-timeout! conn previous))))))
+(defn- lock-wait-timeout-code?
+ "Whether `e` or one of its causes has MySQL's ER_LOCK_WAIT_TIMEOUT error code. The server can raise this independently
+ of the JDBC timeout, so it does not always arrive as a `SQLTimeoutException` or through
+ [[do-with-lock-wait-timeout]]."
+ [^Throwable e]
+ (loop [^Throwable e e]
+ (cond
+ (nil? e) false
+ (and (instance? SQLException e)
+ (= (.getErrorCode ^SQLException e) lock-wait-timeout-error-code)) true
+ :else (recur (.getCause e)))))
+
+(defn- lock-wait-timed-out?
+ "Whether `e` indicates a row-lock timeout rather than another failure."
+ [^Throwable e]
+ (or (instance? java.sql.SQLTimeoutException e)
+ (instance? java.sql.SQLTimeoutException (ex-cause e))
+ (true? (::acquisition-timeout (ex-data e)))
+ (lock-wait-timeout-code? e)
+ (app-db.query-cancelation/query-canceled-exception? (mdb.connection/db-type) e)))
+
+(defn- checkout-connection!
+ "Check out an app-db connection for the out-of-band insert.
+
+ The caller already holds one connection, so this second checkout can time out when the pool is exhausted. Tag that
+ failure so [[fall-back-in-band?]] can retry on the caller's connection."
+ ^Connection []
+ (try
+ (.getConnection ^javax.sql.DataSource mdb.connection/*application-db*)
+ (catch Exception e
+ (throw (ex-info "Could not check out a connection to insert the cluster lock row"
+ {::checkout-failed true}
+ e)))))
+
+(defn- insert-lock-row-out-of-band!
+ "Insert and commit the lock row on a dedicated connection.
+
+ Lock acquisition assumes that each lock row is created once and remains available. A row created inside
+ `with-temp` would instead be rolled back, forcing later acquisitions to repeat the insert race. On MariaDB,
+ each repetition can incur the full lock-wait timeout.
+
+ This temporarily holds both the caller's connection and the dedicated connection. Once committed, later
+ acquisitions find the row and do not return to this path for the same lock name.
+
+ TODO (Chris 2026-08-21) -- This addresses a MariaDB CI failure with no local reproduction, so the explanation is
+ based on observed cross-connection behavior rather than direct measurement. If the failure persists, revisit this
+ path before adding more workarounds."
+ [lock-name-str timeout]
+ (let [[sql] (mdb.query/compile {:insert-into [:metabase_cluster_lock]
+ :columns [:lock_name]
+ :values [[[:raw "?"]]]})]
+ (with-open [conn (checkout-connection!)
+ stmt (.prepareStatement conn ^String sql)]
+ ;; The row must be durable before the SELECT below can find it, and the pool does not guarantee autocommit.
+ (when-not (.getAutoCommit conn)
+ (.setAutoCommit conn true))
+ (.setString stmt 1 lock-name-str)
+ (let [insert! (fn []
+ (try
+ (.executeUpdate stmt)
+ (catch Exception e
+ ;; Another transaction inserted the durable row first; that satisfies this operation.
+ (when-not (duplicate-lock-row? e)
+ (throw e)))))]
+ ;; Bound the wait on this connection. Because the caller uses another session, an uncommitted row from a
+ ;; competing transaction could otherwise block this insert for the server's default lock-wait timeout.
+ (u.connection/set-query-timeout! stmt timeout)
+ ;; Set a lock-wait timeout for MySQL-family servers. MariaDB's driver carries the query timeout as
+ ;; `max_statement_time`, which does not reliably interrupt a statement already waiting on a row lock.
+ (if (= (mdb.connection/db-type) :mysql)
+ (do-with-lock-wait-timeout conn timeout insert!)
+ (insert!))))))
+
+(defn- fall-back-in-band?
+ "Whether a failed out-of-band insert can be retried on the caller's own connection.
+
+ This covers a failed connection checkout and a lock-wait timeout on the dedicated connection. In either case, the
+ caller's connection needs no second checkout and cannot be blocked by its own locks."
+ [^Throwable e]
+ (or (true? (::checkout-failed (ex-data e)))
+ (lock-wait-timed-out? e)))
+
+(defn- insert-lock-row-in-band!
+ "Insert the lock row on the caller's connection, within its transaction.
+
+ The row is not visible until the transaction commits, so it does not need to be locked yet; concurrent
+ inserters receive a constraint violation and retry. This uses raw JDBC to guarantee that the insert runs on
+ `conn` (ambient resolution would select another connection under a detached lock) and uses the same timeout
+ as the SELECT, because concurrent first-time inserts block on the winner's uncommitted unique-index entry."
+ [^Connection conn lock-name-str timeout]
+ (let [[sql] (mdb.query/compile {:insert-into [:metabase_cluster_lock]
+ :columns [:lock_name]
+ :values [[[:raw "?"]]]})]
+ (with-open [insert-stmt (.prepareStatement conn ^String sql)]
+ (u.connection/set-query-timeout! insert-stmt timeout)
+ (.setString insert-stmt 1 lock-name-str)
+ (.executeUpdate insert-stmt))))
+
(defn- acquire-lock-row!*
- [^Connection conn lock-name-str timeout mode]
- (with-open [stmt (prepare-statement conn lock-name-str timeout mode)
- result-set (.executeQuery stmt)]
- (when-not (.next result-set)
- ;; this record will not be visible until the tx commits, so there's no need to lock it; concurrent
- ;; inserters get a constraint violation and retry. Raw JDBC because the insert must run on `conn`
- ;; (under a detached lock ambient resolution would hand it a different connection) and needs the
- ;; same query timeout as the SELECT — concurrent first-time inserters block on the winner's
- ;; uncommitted unique-index entry
- (let [[sql] (mdb.query/compile {:insert-into [:metabase_cluster_lock]
- :columns [:lock_name]
- :values [[[:raw "?"]]]})]
- (with-open [insert-stmt (.prepareStatement conn ^String sql)]
- (u.connection/set-query-timeout! insert-stmt timeout)
- (.setString insert-stmt 1 lock-name-str)
- (.executeUpdate insert-stmt)))))
+ [^Connection conn lock-name-str timeout mode ambient-transaction?]
+ (let [inserted-out-of-band?
+ (with-open [stmt (prepare-statement conn lock-name-str timeout mode)
+ result-set (.executeQuery stmt)]
+ (when-not (.next result-set)
+ ;; Only tests can have a surrounding transaction that never commits. `with-temp` rolls back its lock
+ ;; row, causing the next acquisition to repeat an insert race intended to happen once. Production
+ ;; transactions commit these rows normally; `load-from-h2!` is one such transactional caller.
+ (if (and ambient-transaction? config/is-test?)
+ (try
+ (insert-lock-row-out-of-band! lock-name-str timeout)
+ true
+ (catch Exception e
+ ;; The caller usually causes a timeout. On MySQL and MariaDB, its earlier SELECT ... FOR UPDATE
+ ;; can hold a gap lock over this key, and the dedicated connection cannot outwait the caller's
+ ;; transaction. `load-from-h2!`, for example, holds batch-permissions locks while inserting
+ ;; Databases in one long transaction. The pool can also refuse the second connection when enough
+ ;; threads are already holding one here. Retry on the caller's connection, which needs no checkout
+ ;; and cannot be blocked by the caller's own locks.
+ (if (fall-back-in-band? e)
+ (do (insert-lock-row-in-band! conn lock-name-str timeout)
+ false)
+ (throw e))))
+ (do (insert-lock-row-in-band! conn lock-name-str timeout)
+ false))))]
+ (when inserted-out-of-band?
+ ;; The dedicated connection committed the row; now acquire its lock on the caller's connection.
+ (with-open [stmt (prepare-statement conn lock-name-str timeout mode)
+ result-set (.executeQuery stmt)]
+ (when-not (.next result-set)
+ ;; The insert relies on the dedicated connection being in autocommit. Without the row there is no lock to
+ ;; hold, and returning here would let the body run as though we had one.
+ (throw (ex-info "Cluster lock row is missing after it was committed on a dedicated connection"
+ {:lock-name lock-name-str}))))))
(log/debugf "Obtained cluster lock: %s (%s)" lock-name-str mode))
(defn- acquire-lock-row!
- [^Connection conn lock-name-str timeout mode]
+ [^Connection conn lock-name-str timeout mode ambient-transaction?]
(if (u.connection/server-rejects-query-timeout? conn)
- (do-with-lock-wait-timeout conn timeout #(acquire-lock-row!* conn lock-name-str timeout mode))
- (acquire-lock-row!* conn lock-name-str timeout mode)))
+ (do-with-lock-wait-timeout conn timeout #(acquire-lock-row!* conn lock-name-str timeout mode ambient-transaction?))
+ (acquire-lock-row!* conn lock-name-str timeout mode ambient-transaction?)))
(def ^:private ^:dynamic *detached-locks-held*
"Lock-name strings currently held by [[do-with-detached-cluster-lock]] in this dynamic scope. A detached
@@ -177,10 +299,12 @@
(when (*detached-locks-held* lock-name-str)
(throw (ex-info "Cluster lock is already held detached in this scope"
{:lock-name lock-name-str}))))
- (t2/with-transaction [conn]
- (doseq [{:keys [lock-name-str mode]} locks]
- (acquire-lock-row! conn lock-name-str timeout-seconds mode))
- (thunk)))
+ ;; Read this before opening our transaction: only the caller's preexisting transaction could roll back the lock row.
+ (let [ambient-transaction? (mdb.connection/in-transaction?)]
+ (t2/with-transaction [conn]
+ (doseq [{:keys [lock-name-str mode]} locks]
+ (acquire-lock-row! conn lock-name-str timeout-seconds mode ambient-transaction?))
+ (thunk))))
;; ---------- h2 in-process rw locks ----------
;;
@@ -368,7 +492,8 @@
(retry/with-retry config
;; clear the aborted transaction a failed previous attempt leaves behind
(.rollback ^Connection conn)
- (acquire-lock-row! conn lock-name-str timeout-seconds :exclusive))
+ ;; A detached hold owns its connection, so the caller's transaction cannot roll back its row.
+ (acquire-lock-row! conn lock-name-str timeout-seconds :exclusive false))
(catch Throwable e
(if (retryable? e)
(throw (ex-info (str "Failed to obtain cluster lock: " lock-name-str)
diff --git a/src/metabase/app_db/connection.clj b/src/metabase/app_db/connection.clj
index 55d3dfc13e4..3cab7dd7edb 100644
--- a/src/metabase/app_db/connection.clj
+++ b/src/metabase/app_db/connection.clj
@@ -145,6 +145,14 @@
(def ^:private ^:dynamic *before-commit-callbacks* nil)
(def ^:private ^:dynamic *after-commit-callbacks* nil)
+;; Holds an atom set to true when a rollback fails and leaves behind writes that should have been discarded. The atom
+;; is shared across the transaction tree so that the outermost scope cannot commit those writes, even if an
+;; intermediate scope catches the rollback error.
+;;
+;; Once set it stays set. Clearing it correctly would mean tracking the depth that failed: a sibling scope rolling
+;; back to its own, later savepoint does not discard an earlier scope's writes.
+(def ^:private ^:dynamic *rollback-required* nil)
+
(def ^:dynamic *transaction-state*
"When non-nil, an atom holding a map of arbitrary per-transaction data, shared by the whole
nested-transaction tree and thrown away when the outermost transaction ends. Any subsystem can stash
@@ -205,57 +213,143 @@
;; captured closures) through the backing array until the outer transaction finishes
(into [] (subvec cbs 0 (min n (count cbs))))))))
-(defn- do-transaction [^java.sql.Connection connection f]
- (letfn [(thunk []
- (let [savepoint (.setSavepoint connection)
- before-count (some-> *before-commit-callbacks* deref count)
- after-count (some-> *after-commit-callbacks* deref count)
- state-snapshot (when (and *transaction-state* (> *transaction-depth* 1))
- @*transaction-state*)]
- (try
- (let [result (f connection)]
- (when (= *transaction-depth* 1)
- ;; top-level transaction. Run before-commit callbacks first, while the transaction is
- ;; still open, so their writes commit atomically with it. They are NOT wrapped in
- ;; try/catch — a throwing callback must propagate to the catch below and roll back.
- (loop []
- (when-let [callbacks (seq (first (reset-vals! *before-commit-callbacks* [])))]
- (doseq [cb callbacks] (cb))
- ;; a before-commit callback may register more (before- or after-commit); run those too
- (recur)))
- ;; commit; after-commit side effects run after the transaction bindings unwind
- (.commit connection))
- result)
- (catch Throwable txn-e
- ;; the nested body failed, so its before-/after-commit callbacks must never fire — discard
- ;; those it registered before rolling back, otherwise a throwing .rollback would leave them
- ;; in the shared accumulators to run at outer-commit time for data that was rolled back
- (when after-count (discard-callbacks-after! *after-commit-callbacks* after-count))
- (when before-count (discard-callbacks-after! *before-commit-callbacks* before-count))
- (try
- (.rollback connection savepoint)
- ;; restore transaction-state to its pre-savepoint value so data the rolled-back
- ;; sub-transaction stashed is discarded and not seen by the outer commit
- (when state-snapshot
- (reset! *transaction-state* state-snapshot))
- (catch Exception rollback-e
- (throw (ex-info
- (str "Error rolling back after previous error: " (ex-message txn-e))
- {:rollback-error rollback-e}
- txn-e))))
- (throw txn-e)))))]
- ;; optimization: don't set and unset autocommit if it's already false
- (if (.getAutoCommit connection)
- (try
- (.setAutoCommit connection false)
- (thunk)
- (finally
- ;; prevent a failing .setAutoCommit call from masking the original exception
+(defn- do-transaction [^java.sql.Connection connection rollback-only? f]
+ ;; Set when a connection rollback fails and leaves pending writes. Restoring autocommit would commit those writes,
+ ;; so this flag prevents the restore and leaves cleanup to the pool when the connection is checked in.
+ (let [discard-failed? (volatile! false)
+ rollback-connection! (fn []
+ (try
+ (.rollback connection)
+ (catch Throwable rollback-e
+ ;; This matters only when the transaction still contains writes that must be
+ ;; discarded. After a successful savepoint rollback, there is nothing left here
+ ;; to commit.
+ ;; TODO (Chris 2026-08-18) -- A caller-supplied connection is returned to its owner
+ ;; with these writes still pending. We prevent an accidental commit by leaving
+ ;; autocommit disabled, but cannot force the owner to roll them back.
+ (when (some-> *rollback-required* deref)
+ (vreset! discard-failed? true))
+ (log/warnf "Failed to roll back transaction: %s"
+ (ex-message rollback-e)))))]
+ (letfn [(thunk []
+ (let [savepoint (.setSavepoint connection)
+ before-count (some-> *before-commit-callbacks* deref count)
+ after-count (some-> *after-commit-callbacks* deref count)
+ state-snapshot (when (and *transaction-state* (> *transaction-depth* 1))
+ @*transaction-state*)]
+ (letfn [(rollback! []
+ ;; Rollback callbacks and transaction state together with the DB writes, whether the rollback
+ ;; is explicit or caused by an exception.
+ (when after-count (discard-callbacks-after! *after-commit-callbacks* after-count))
+ (when before-count (discard-callbacks-after! *before-commit-callbacks* before-count))
+ ;; Restore the state even if rollback throws. An outer scope must not see state from this one.
+ (try
+ (.rollback connection savepoint)
+ (catch Throwable rollback-e
+ ;; The writes remain pending. No enclosing scope may commit them.
+ ;; A vanished savepoint still counts as a rollback failure. DDL committing implicitly,
+ ;; or a server breaking a deadlock, ends the transaction and discards the writes so far,
+ ;; but anything written after that sits in a new transaction and is still pending.
+ (some-> *rollback-required* (reset! true))
+ (throw rollback-e))
+ (finally
+ (when state-snapshot
+ (reset! *transaction-state* state-snapshot)))))
+ (rollback-after-error! [txn-e]
+ (try
+ (rollback!)
+ (catch Exception rollback-e
+ (throw (ex-info
+ (str "Error rolling back after previous error: " (ex-message txn-e))
+ {:rollback-error rollback-e}
+ txn-e))))
+ (throw txn-e))]
+ (let [result (try
+ (f connection)
+ (catch Throwable txn-e
+ (rollback-after-error! txn-e)))]
+ (cond
+ rollback-only?
+ (do
+ (try
+ (rollback!)
+ (catch Exception rollback-e
+ (if (= *transaction-depth* 1)
+ ;; The body committed the transaction, so the savepoint is gone. H2 and MySQL do this
+ ;; implicitly for DDL. Those writes are already durable; discard anything still pending.
+ ;; TODO (Chris 2026-08-18) -- Consider rejecting DDL in rollback-only transactions and
+ ;; fixing each caller.
+ ;; The existing SQL guard does not cover raw SQL, and savepoints can also disappear in
+ ;; ordinary nested transactions. Callers such as search's `drop-table!` use the ambient
+ ;; connection; moving that DDL could block on locks held by the caller.
+ (do
+ (log/warnf (str "Could not roll back a rollback-only transaction (%s). Something in"
+ " it committed the transaction -- DDL commits implicitly on H2 and"
+ " MySQL -- so its writes up to that point are already durable.")
+ (ex-message rollback-e))
+ (rollback-connection!))
+ ;; In a nested scope, propagate the error. `rollback!` has marked the transaction tree
+ ;; as unsafe to commit.
+ (throw (ex-info "Error rolling back a rollback-only transaction" {} rollback-e)))))
+ [result false])
+
+ (and (= *transaction-depth* 1) (some-> *rollback-required* deref))
+ (do
+ ;; Discard the entire tree. A scope failed to undo its writes, so do not commit them or
+ ;; return as though the scope succeeded.
+ (rollback-connection!)
+ (throw (ex-info (str "Not committing: a transaction in this tree failed to "
+ "roll back, so its writes are still present")
+ {})))
+
+ (= *transaction-depth* 1)
+ (try
+ ;; Run before-commit callbacks while the top-level transaction is open. Their writes commit
+ ;; atomically with it, and a callback exception rolls back the transaction.
+ (loop []
+ (when-let [callbacks (seq (first (reset-vals! *before-commit-callbacks* [])))]
+ (doseq [cb callbacks] (cb))
+ ;; A before-commit callback may register more callbacks; run those as well.
+ (recur)))
+ ;; A callback can catch a nested rollback failure, so check the failure flag again here.
+ (when (some-> *rollback-required* deref)
+ (throw (ex-info (str "Not committing: a transaction in this tree failed to "
+ "roll back, so its writes are still present")
+ {})))
+ (.commit connection)
+ [result true]
+ (catch Throwable txn-e
+ (rollback-after-error! txn-e)))
+
+ :else
+ [result false])))))]
+ ;; Avoid toggling autocommit when the connection is already in a transaction.
+ (if (.getAutoCommit connection)
+ (try
+ (.setAutoCommit connection false)
(try
- (.setAutoCommit connection true)
+ (thunk)
(catch Throwable t
- (log/warnf "Failed to reset the connection's autocommit flag to true: %s" (ex-message t))))))
- (thunk))))
+ ;; Restoring autocommit commits any open transaction. On an exceptional exit, including a failed
+ ;; requested rollback, try to discard the transaction before reaching that restore.
+ (rollback-connection!)
+ (throw t)))
+ (finally
+ (if @discard-failed?
+ ;; The writes remain pending, so leave autocommit disabled and let the pool roll the connection back
+ ;; when it is checked in.
+ (log/warn (str "Leaving autocommit off: this connection holds writes that could not be"
+ " rolled back, and restoring autocommit would commit them."))
+ ;; Do not let a failed autocommit restore mask the original exception.
+ (try
+ (.setAutoCommit connection true)
+ (catch Throwable t
+ (log/warnf "Failed to reset the connection's autocommit flag to true: %s"
+ (ex-message t)))))))
+ (thunk)))))
+
+(def ^:private supported-transaction-options
+ #{:nested-transaction-rule :rollback-only})
(comment
;; in toucan2.jdbc.connection, there is a 'defmethod' for t2.conn/do-with-transaction java.sql.Connection
@@ -274,9 +368,30 @@
other transactions have made,
- in the presence of unsynchronized concurrent threads running nested transactions, the effects of rollback
are not well defined - a rollback will undo all work done by other transactions in the same tree that
- started later."
- [^java.sql.Connection connection {:keys [nested-transaction-rule] :or {nested-transaction-rule :allow} :as options} f]
- (assert (#{:allow :ignore :prohibit} nested-transaction-rule))
+ started later.
+
+ With `:rollback-only true`, a successful scope rolls back to its savepoint and discards the callbacks and transaction
+ state registered within that scope. This cannot be combined with `:nested-transaction-rule :ignore`, which skips the
+ savepoint entirely. Unsupported JDBC transaction options are rejected rather than silently treated as ordinary
+ writable transactions."
+ [^java.sql.Connection connection
+ {:keys [nested-transaction-rule rollback-only] :or {nested-transaction-rule :allow} :as options}
+ f]
+ ;; Sort so the message and the ex-data name the options in the same order every time.
+ (when-let [unsupported-options (not-empty (vec (sort (remove supported-transaction-options (keys options)))))]
+ (throw (ex-info (str "Unsupported transaction options: " (pr-str unsupported-options))
+ {:unsupported-options unsupported-options
+ :options options})))
+ ;; `assert` compiles away under `*assert*` false, and this validates a public option.
+ (when-not (#{:allow :ignore :prohibit} nested-transaction-rule)
+ (throw (ex-info (str "Invalid :nested-transaction-rule: " (pr-str nested-transaction-rule))
+ {:nested-transaction-rule nested-transaction-rule
+ :options options})))
+ ;; Reject this combination at every depth so a call site behaves consistently wherever it runs.
+ (when (and rollback-only (= nested-transaction-rule :ignore))
+ (throw (ex-info (str "Cannot combine :rollback-only with :nested-transaction-rule :ignore -- an ignored "
+ "nested transaction has no savepoint to roll back to")
+ {:options options})))
(cond
(and (pos? *transaction-depth*)
(= nested-transaction-rule :ignore))
@@ -290,13 +405,15 @@
:else
(let [outermost? (zero? *transaction-depth*)
callbacks (if outermost? (atom []) *after-commit-callbacks*)
- result (binding [*transaction-depth* (inc *transaction-depth*)
- ;; one set of accumulators + state for the whole tree, created at the outermost txn
- *before-commit-callbacks* (if outermost? (atom []) *before-commit-callbacks*)
- *transaction-state* (if outermost? (atom {}) *transaction-state*)
- *after-commit-callbacks* callbacks]
- (do-transaction connection f))]
- (when outermost?
+ [result committed?]
+ (binding [*transaction-depth* (inc *transaction-depth*)
+ ;; Create one set of callback accumulators and transaction state for the entire tree.
+ *before-commit-callbacks* (if outermost? (atom []) *before-commit-callbacks*)
+ *transaction-state* (if outermost? (atom {}) *transaction-state*)
+ *rollback-required* (if outermost? (atom false) *rollback-required*)
+ *after-commit-callbacks* callbacks]
+ (do-transaction connection rollback-only f))]
+ (when (and outermost? committed?)
(run-after-commit-callbacks! callbacks))
result)))
diff --git a/src/metabase/collections/models/collection.clj b/src/metabase/collections/models/collection.clj
index 13861ee046b..dd9e664d9d7 100644
--- a/src/metabase/collections/models/collection.clj
+++ b/src/metabase/collections/models/collection.clj
@@ -685,6 +685,11 @@
(some-> user-id user->personal-collection u/the-id))
;; cache the results for 60 minutes; TTL is here only to eventually clear out old entries/keep it from growing too
;; large
+ ;;
+ ;; TODO (Chris 2026-08-18) -- The claim that Personal Collections cannot be deleted does not hold when a
+ ;; transaction rolls back: the collection disappears while its ID remains cached for the rest of the TTL.
+ ;; Tests evict this cache at the `with-temp` boundary (see [[metabase.test.util]]). Production has no
+ ;; equivalent boundary and would need an after-rollback hook alongside the commit hooks.
:ttl/threshold (* 60 60 1000)))
(mu/defn user->personal-collection-and-descendant-ids :- [:sequential ms/PositiveInt]
diff --git a/src/metabase/metabot/tools/entity_details.clj b/src/metabase/metabot/tools/entity_details.clj
index f32451c95f0..8c26b9d6117 100644
--- a/src/metabase/metabot/tools/entity_details.clj
+++ b/src/metabase/metabot/tools/entity_details.clj
@@ -650,10 +650,20 @@
;; Returns `nil` (so `m/assoc-some` drops the key) for cards whose
;; `dataset_query` can't be exported, e.g. unusual / partially-broken legacy
;; queries; the rest of the payload is still useful.
- :query_json (when dataset-query
+ ;; Keep `lib/query` inside the guard. A stored `dataset_query` with no stages fails the query output
+ ;; schema; one such Card must not break an entire response containing multiple Cards.
+ :query_json (when-let [query (when dataset-query
+ (try
+ (lib/query metadata-provider (lib-be/normalize-query dataset-query))
+ (catch Exception e
+ ;; A Card whose query fails to build for another reason is
+ ;; otherwise indistinguishable from one that has no query.
+ (log/warnf "Could not build the stored query for Card %s; omitting it from the LLM payload: %s"
+ id (ex-message e))
+ nil)))]
(repr.resolve/try-export-query
metadata-provider
- (lib/query metadata-provider (lib-be/normalize-query dataset-query))
+ query
shared.content-store/default-store))
:metrics (when with-metrics?
(not-empty (mapv #(convert-metric % metadata-provider options)
diff --git a/src/metabase/metrics/api.clj b/src/metabase/metrics/api.clj
index f4a59487229..4d8e27bd81c 100644
--- a/src/metabase/metrics/api.clj
+++ b/src/metabase/metrics/api.clj
@@ -3,6 +3,7 @@
(:require
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
+ [metabase.app-db.core :as mdb]
[metabase.events.core :as events]
[metabase.lib-metric.core :as lib-metric]
[metabase.lib-metric.schema :as lib-metric.schema]
@@ -217,14 +218,20 @@
Must be called OUTSIDE streaming context to avoid JSON writer conflicts.
Returns {uuid -> qp-result}."
[leaves metric-card-ids]
- (let [uuid->future (into {}
- (map (fn [[uuid leaf-plan]]
- [uuid (future (process-leaf-query (:leaf/mbql leaf-plan)
- (get metric-card-ids uuid)))]))
- leaves)]
- (into {}
- (map (fn [[uuid f]] [uuid @f]))
- uuid->future)))
+ (letfn [(run-leaf [[uuid leaf-plan]]
+ (process-leaf-query (:leaf/mbql leaf-plan) (get metric-card-ids uuid)))]
+ (if (mdb/in-transaction?)
+ ;; A transaction owns one connection, and `future` conveys that binding to each thread. Parallel leaves
+ ;; would therefore interleave app DB writes and savepoints on the same session; one thread's rollback can
+ ;; invalidate its siblings' savepoints. Only tests run this code within a transaction. During a request,
+ ;; each future obtains its own pooled connection.
+ (into {} (map (fn [leaf] [(first leaf) (run-leaf leaf)])) leaves)
+ (let [uuid->future (into {}
+ (map (fn [leaf] [(first leaf) (future (run-leaf leaf))]))
+ leaves)]
+ (into {}
+ (map (fn [[uuid f]] [uuid @f]))
+ uuid->future)))))
(defn- stream-arithmetic-results
"Join leaf results and stream the computed output through the QP reduce pipeline.
diff --git a/src/metabase/xrays/related.clj b/src/metabase/xrays/related.clj
index 9d344530f01..ca1d410358c 100644
--- a/src/metabase/xrays/related.clj
+++ b/src/metabase/xrays/related.clj
@@ -9,6 +9,7 @@
[metabase.models.interface :as mi]
[metabase.query-processor.util :as qp.util]
[metabase.segments.schema :as segments.schema]
+ [metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.xrays.automagic-dashboards.schema :as ads]
@@ -48,9 +49,17 @@
(defmethod definition :model/Card
[card]
- (-> card
- :dataset_query
- ((juxt lib/breakouts lib/aggregations lib/expressions lib/fields))))
+ ;; The schema permits an empty or otherwise unparseable stored `dataset_query`. Treat such a Card as having
+ ;; no context-bearing forms so it does not break related-entity computation for every Card and x-ray.
+ (try
+ (-> card
+ :dataset_query
+ ((juxt lib/breakouts lib/aggregations lib/expressions lib/fields)))
+ (catch Exception e
+ ;; Runs for every candidate Card while ranking, so keep it quiet.
+ (log/debugf "Ignoring Card %s while finding related entities because its query could not be parsed: %s"
+ (:id card) (ex-message e))
+ nil)))
(mu/defmethod definition :model/Segment
[segment :- [:map [:definition ::segments.schema/definition]]]
diff --git a/test/metabase/app_db/cluster_lock_test.clj b/test/metabase/app_db/cluster_lock_test.clj
index 78eb13439aa..5b5b774d923 100644
--- a/test/metabase/app_db/cluster_lock_test.clj
+++ b/test/metabase/app_db/cluster_lock_test.clj
@@ -17,6 +17,18 @@
(use-fixtures :once (fixtures/initialize :db))
+(deftest lock-wait-timed-out?-test
+ (testing "MySQL's ER_LOCK_WAIT_TIMEOUT decides whether we fall back to an in-band insert"
+ (let [timed-out (SQLException. "Lock wait timeout exceeded; try restarting transaction" "HY000" 1205)
+ other (SQLException. "You have an error in your SQL syntax" "42000" 1064)]
+ (testing "recognised on its own"
+ (is (#'sut/lock-wait-timed-out? timed-out)))
+ (testing "and through a cause chain, which is how it usually arrives"
+ (is (#'sut/lock-wait-timed-out? (ex-info "wrapped" {} timed-out))))
+ (testing "any other error still propagates"
+ (is (not (#'sut/lock-wait-timed-out? other)))
+ (is (not (#'sut/lock-wait-timed-out? (ex-info "wrapped" {} other))))))))
+
(defn- deadlock-exception
"Build a deadlock SQLException for the current appdb type. Mirrors the codes in
[[metabase.app-db.transient-error]] so the test exercises the real db-type path."
@@ -235,6 +247,55 @@
[:b :done]]
@results))))))
+(deftest fall-back-in-band?-test
+ (testing "the out-of-band insert retries on the caller's connection when"
+ (testing "the pool cannot provide a second connection"
+ (is (#'sut/fall-back-in-band? (ex-info "checkout timed out" {::sut/checkout-failed true}))))
+ (testing "the dedicated connection timed out on a row lock"
+ (is (#'sut/fall-back-in-band?
+ (SQLException. "Lock wait timeout exceeded; try restarting transaction" "HY000" 1205)))))
+ (testing "other errors still propagate"
+ (is (not (#'sut/fall-back-in-band?
+ (SQLException. "You have an error in your SQL syntax" "42000" 1064))))))
+
+(deftest checkout-connection!-tags-pool-failures-test
+ (testing "a failed checkout produces an error that triggers the fallback"
+ (let [exhausted-pool (reify javax.sql.DataSource
+ (getConnection [_]
+ (throw (SQLException. "An attempt by a client to checkout a Connection has timed out."))))]
+ (mdb/with-application-db exhausted-pool
+ (is (#'sut/fall-back-in-band? (try
+ (#'sut/checkout-connection!)
+ (catch Exception e e))))))))
+
+(deftest checkout-failure-still-acquires-a-first-time-lock-test
+ (testing "a first-time lock is acquired even when the pool cannot provide a second connection"
+ ;; H2 takes an in-process lock and never writes a row.
+ (when (not= (mdb/db-type) :h2)
+ (let [lock-name (u/qualified-name ::checkout-failure-lock)]
+ (t2/delete! :metabase_cluster_lock :lock_name lock-name)
+ (mt/with-dynamic-fn-redefs [sut/checkout-connection!
+ (fn [] (throw (ex-info "checkout timed out" {::sut/checkout-failed true})))]
+ ;; An ambient transaction is what sends the insert out of band in the first place.
+ (t2/with-transaction [_conn]
+ (is (= :ok (sut/with-cluster-lock ::checkout-failure-lock :ok)))
+ (is (t2/exists? :metabase_cluster_lock :lock_name lock-name))))))))
+
+(deftest out-of-band-insert-commits-without-help-from-the-pool-test
+ (testing "the lock row outlives the caller's rollback even when the dedicated connection arrives in a transaction"
+ ;; H2 takes an in-process lock and never writes a row.
+ (when (not= (mdb/db-type) :h2)
+ (let [lock-name (u/qualified-name ::autocommit-off-lock)]
+ (t2/delete! :metabase_cluster_lock :lock_name lock-name)
+ (mt/with-dynamic-fn-redefs [sut/checkout-connection!
+ (fn []
+ (doto (.getConnection (mdb/data-source))
+ (.setAutoCommit false)))]
+ (t2/with-transaction [_conn nil {:rollback-only true}]
+ (is (= :ok (sut/with-cluster-lock ::autocommit-off-lock :ok)))))
+ (is (t2/exists? :metabase_cluster_lock :lock_name lock-name)
+ "the dedicated connection committed it, so the caller's rollback cannot take it away")))))
+
(deftest detached-lock-basic-test
(testing "returns the body's value and works non-concurrently"
(is (= :ok (sut/with-detached-cluster-lock {:lock ::detached-basic} :ok)))))
diff --git a/test/metabase/app_db/connection_test.clj b/test/metabase/app_db/connection_test.clj
index ad576573450..585ad9b7653 100644
--- a/test/metabase/app_db/connection_test.clj
+++ b/test/metabase/app_db/connection_test.clj
@@ -8,7 +8,7 @@
[toucan2.core :as t2]
[toucan2.jdbc.options :as t2.jdbc.options])
(:import
- (java.sql Connection)
+ (java.sql Connection SQLException)
(java.util.concurrent Semaphore)))
(set! *warn-on-reflection* true)
@@ -185,16 +185,19 @@
(setSavepoint [_])
(commit [_]))]
(binding [t2.connection/*current-connectable* mock-conn]
- (t2/with-transaction [_conn]
- (mdb.connection/do-after-commit (fn [] (swap! calls conj :outer)))
- ;; the nested body registers a callback then throws; its savepoint rollback also throws
- (is (thrown?
- Exception
- (t2/with-transaction [_]
- (mdb.connection/do-after-commit (fn [] (swap! calls conj :nested)))
- (throw (ex-info "boom" {})))))))
- ;; the nested callback is discarded even though the rollback threw — only :outer survives to run
- (is (= [:outer] @calls))))
+ ;; The nested callback is discarded even though rollback throws. The outer scope then refuses to commit
+ ;; because the nested writes remain pending, so its callbacks do not run either.
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo #"Not committing"
+ (t2/with-transaction [_conn]
+ (mdb.connection/do-after-commit (fn [] (swap! calls conj :outer)))
+ ;; The nested body registers a callback and throws; its savepoint rollback also throws.
+ (is (thrown?
+ Exception
+ (t2/with-transaction [_]
+ (mdb.connection/do-after-commit (fn [] (swap! calls conj :nested)))
+ (throw (ex-info "boom" {})))))))))
+ (is (= [] @calls))))
(deftest after-commit-callback-registering-another-runs-it-immediately-test
(let [calls (atom [])]
@@ -208,11 +211,178 @@
(mdb.connection/do-after-commit (fn [] (swap! calls conj :b))))
(is (= [:a :nested-from-a :b] @calls))))
+(deftest rollback-only-transaction-rolls-back-and-discards-callbacks-test
+ (let [email (mt/random-email)
+ calls (atom [])]
+ (try
+ (is (= :result
+ (t2/with-transaction [_conn nil {:rollback-only true}]
+ (t2/insert! :model/User (assoc (mt/with-temp-defaults :model/User) :email email))
+ (mdb.connection/do-before-commit (fn [] (swap! calls conj :before)))
+ (mdb.connection/do-after-commit (fn [] (swap! calls conj :after)))
+ :result))
+ "a requested rollback returns the transaction body's result")
+ (is (not (t2/exists? :model/User :email email))
+ "a successful rollback-only transaction does not commit its writes")
+ (is (= [] @calls)
+ "neither before- nor after-commit callbacks run when no commit occurs")
+ (finally
+ (t2/delete! :model/User :email email)))))
+
+(deftest with-temp-rollback-boundary-discards-after-commit-callback-test
+ (let [calls (atom [])]
+ (mt/with-temp [:model/User _user]
+ (mdb.connection/do-after-commit (fn [] (swap! calls conj :after))))
+ (is (= [] @calls)
+ "with-temp exits through rollback-only, so its deferred callback must not run")))
+
+(deftest nested-rollback-only-transaction-restores-savepoint-state-test
+ (let [email (mt/random-email)
+ calls (atom [])]
+ (try
+ (t2/with-transaction [conn]
+ (swap! mdb.connection/*transaction-state* assoc :outer "kept")
+ (mdb.connection/do-before-commit (fn [] (swap! calls conj :outer-before)))
+ (mdb.connection/do-after-commit (fn [] (swap! calls conj :outer-after)))
+ (is (= :nested-result
+ (t2/with-transaction [_ conn {:rollback-only true}]
+ (t2/insert! :model/User (assoc (mt/with-temp-defaults :model/User) :email email))
+ (swap! mdb.connection/*transaction-state* assoc :nested "discarded")
+ (mdb.connection/do-before-commit (fn [] (swap! calls conj :nested-before)))
+ (mdb.connection/do-after-commit (fn [] (swap! calls conj :nested-after)))
+ :nested-result)))
+ (is (not (t2/exists? :model/User :email email))
+ "the nested write is rolled back to its savepoint")
+ (is (= {:outer "kept"} @mdb.connection/*transaction-state*)
+ "transaction state is restored to its pre-savepoint value"))
+ (is (= [:outer-before :outer-after] @calls)
+ "only callbacks registered outside the rolled-back nested transaction run")
+ (is (not (t2/exists? :model/User :email email)))
+ (finally
+ (t2/delete! :model/User :email email)))))
+
+(deftest unsupported-transaction-options-are-rejected-test
+ (t2/with-connection [conn]
+ (let [e (is (thrown?
+ clojure.lang.ExceptionInfo
+ (t2/with-transaction [_ conn {:read-only true}])))]
+ (is (= "Unsupported transaction options: [:read-only]" (ex-message e)))
+ (is (= [:read-only] (:unsupported-options (ex-data e)))))))
+
+(deftest rollback-only-with-ignored-nesting-is-rejected-test
+ (testing ":ignore skips the savepoint :rollback-only needs, so the pair is rejected at every depth"
+ (let [msg #"Cannot combine :rollback-only with :nested-transaction-rule :ignore"]
+ (testing "outside a transaction"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo msg
+ (t2/with-transaction [_ nil {:nested-transaction-rule :ignore :rollback-only true}]))))
+ (testing "inside a transaction"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo msg
+ (t2/with-transaction [conn nil {:rollback-only true}]
+ (t2/with-transaction [_ conn {:nested-transaction-rule :ignore :rollback-only true}]))))))))
+
+(deftest failed-nested-rollback-only-blocks-the-outer-commit-test
+ (testing "swallowing the error from a failed rollback-only rollback must not let the outer scope commit it"
+ ;; Track autocommit instead of pinning it to true so only the outermost scope believes it owns the connection.
+ ;; Pinning it would make the nested scope install another rollback and hide the behavior under test.
+ (let [calls (atom [])
+ auto? (atom true)
+ mock-conn (reify Connection
+ (rollback [_ _savepoint]
+ (throw (ex-info "Savepoint rollback error" {})))
+ (rollback [_] (swap! calls conj :rollback))
+ (commit [_] (swap! calls conj :commit))
+ (setAutoCommit [_ v] (reset! auto? v))
+ (getAutoCommit [_] @auto?)
+ (setSavepoint [_]))]
+ (binding [t2.connection/*current-connectable* mock-conn]
+ (let [e (is (thrown? Exception
+ (t2/with-transaction [conn]
+ (try
+ (t2/with-transaction [_ conn {:rollback-only true}] :nested)
+ (catch Exception _ :swallowed))
+ :outer)))]
+ (is (re-find #"Not committing" (ex-message e)))))
+ (is (= #{:rollback} (set @calls))
+ "the tree is rolled back, and never committed"))))
+
+(deftest a-vanished-savepoint-still-blocks-the-outer-commit-test
+ (testing "a savepoint the server already discarded says nothing about what was written after it"
+ ;; MySQL and MariaDB raise 1305 and PostgreSQL 3B001 after the transaction ends -- when a deadlock is broken or
+ ;; DDL commits implicitly. Writes made after that point sit in a fresh transaction and are
+ ;; still pending, so the tree must not commit.
+ (doseq [[label sqlstate error-code] [["MySQL 1305" "42000" 1305]
+ ["PostgreSQL 3B001" "3B001" 0]]]
+ (testing label
+ (let [calls (atom [])
+ auto? (atom true)
+ mock-conn (reify Connection
+ (rollback [_ _savepoint]
+ (throw (SQLException. "SAVEPOINT does not exist" ^String sqlstate ^int error-code)))
+ (rollback [_] (swap! calls conj :rollback))
+ (commit [_] (swap! calls conj :commit))
+ (setAutoCommit [_ v] (reset! auto? v))
+ (getAutoCommit [_] @auto?)
+ (setSavepoint [_]))]
+ (binding [t2.connection/*current-connectable* mock-conn]
+ (let [e (is (thrown? Exception
+ (t2/with-transaction [conn]
+ (try
+ (t2/with-transaction [_ conn {:rollback-only true}] :nested)
+ (catch Exception _ :swallowed))
+ :outer)))]
+ (is (re-find #"Not committing" (ex-message e)))))
+ (is (= #{:rollback} (set @calls))
+ "the tree is rolled back, and never committed"))))))
+
+(deftest failed-savepoint-rollback-still-restores-transaction-state-test
+ (testing "a nested scope whose savepoint rollback throws must not leave its state visible to the outer scope"
+ (let [mock-conn (reify Connection
+ (rollback [_ _savepoint]
+ (throw (ex-info "Rollback error" {})))
+ (rollback [_])
+ (setAutoCommit [_ _])
+ (getAutoCommit [_] true)
+ (setSavepoint [_])
+ (commit [_]))]
+ (binding [t2.connection/*current-connectable* mock-conn]
+ ;; The outer scope refuses to commit because the nested writes could not be rolled back and remain pending.
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo #"Not committing"
+ (t2/with-transaction [conn]
+ (swap! mdb.connection/*transaction-state* assoc :outer "kept")
+ (is (thrown? Exception
+ (t2/with-transaction [_ conn]
+ (swap! mdb.connection/*transaction-state* assoc :nested "discarded")
+ (throw (ex-info "Original error" {})))))
+ (is (= {:outer "kept"} @mdb.connection/*transaction-state*)))))))))
+
+(deftest failed-rollback-only-rollback-discards-the-transaction-test
+ (testing "an outermost rollback-only whose savepoint is gone discards what is pending instead of failing"
+ ;; The savepoint normally disappears because the body committed; DDL does this implicitly on H2 and MySQL.
+ ;; Those writes are already durable, so only any remaining pending work can be rolled back.
+ (let [calls (atom [])
+ mock-conn (reify Connection
+ (rollback [_ _savepoint]
+ (throw (ex-info "Savepoint rollback error" {})))
+ (rollback [_] (swap! calls conj :rollback))
+ (commit [_] (swap! calls conj :commit))
+ (setAutoCommit [_ _])
+ (getAutoCommit [_] true)
+ (setSavepoint [_]))]
+ (binding [t2.connection/*current-connectable* mock-conn]
+ (is (= :result (t2/with-transaction [_ nil {:rollback-only true}] :result))
+ "the body's result is still returned"))
+ (is (= [:rollback] @calls)
+ "the connection is rolled back outright, never committed"))))
+
(deftest rollback-error-handling
(testing "rollback error handling"
(let [mock-conn (reify Connection
(rollback [_ _savepoint]
(throw (ex-info "Rollback error" {})))
+ (rollback [_])
(setAutoCommit [_ _])
(getAutoCommit [_] true)
(setSavepoint [_])
diff --git a/test/metabase/app_db/schema_migrations_test/impl.clj b/test/metabase/app_db/schema_migrations_test/impl.clj
index 7eea04abd15..b1a59cb9ce1 100644
--- a/test/metabase/app_db/schema_migrations_test/impl.clj
+++ b/test/metabase/app_db/schema_migrations_test/impl.clj
@@ -21,6 +21,7 @@
[metabase.driver :as driver]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.test.data.datasets :as datasets]
+ [metabase.test.data.impl :as data.impl]
[metabase.test.data.interface :as tx]
[metabase.test.initialize :as initialize]
[metabase.util :as u]
@@ -77,7 +78,10 @@
;; open
(with-open [conn (.getConnection data-source)]
(binding [mdb.connection/*application-db* (mdb.connection/application-db driver data-source)
- custom-migrations.util/*allow-temp-scheduling* false]
+ custom-migrations.util/*allow-temp-scheduling* false
+ ;; This app DB must remain empty, or contain only what the test loads. Prevent `with-temp` from
+ ;; prewarming the test-data Database within it.
+ data.impl/*skip-dataset-prewarm?* true]
(f conn))))))
(defmacro with-temp-empty-app-db
diff --git a/test/metabase/collections/models/collection_test.clj b/test/metabase/collections/models/collection_test.clj
index 35d8f369606..c7b76301a0f 100644
--- a/test/metabase/collections/models/collection_test.clj
+++ b/test/metabase/collections/models/collection_test.clj
@@ -2,6 +2,7 @@
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.collections.models.collection-test]}}}}}}
(:refer-clojure :exclude [descendants])
(:require
+ [clojure.core.cache :as cache]
[clojure.math.combinatorics :as math.combo]
[clojure.set :as set]
[clojure.string :as str]
@@ -9,6 +10,7 @@
[clojure.walk :as walk]
[java-time.api :as t]
[metabase.api.common :as api]
+ [metabase.app-db.core :as mdb]
[metabase.audit-app.impl :as audit]
[metabase.collections.models.collection :as collection]
[metabase.config.core :as config]
@@ -55,6 +57,21 @@
(is (= "Lucky Pigeon's Personal Collection"
(collection/user->personal-collection-name (mt/user->id :lucky) :user)))))
+(deftest with-temp-user-evicts-personal-collection-id-cache-test
+ (testing "a temporary User cannot leave its deleted Personal Collection id in the production cache"
+ (let [user-id (atom nil)
+ collection-id (atom nil)]
+ (mt/with-temp [:model/User {id :id}]
+ (reset! user-id id)
+ (reset! collection-id (@#'collection/user->personal-collection-id id))
+ (is (t2/exists? :model/Collection :id @collection-id)))
+ (is (not (t2/exists? :model/Collection :id @collection-id))
+ "with-temp removed the Personal Collection")
+ (is (not (cache/has? @(-> @#'collection/user->personal-collection-id
+ meta :clojure.core.memoize/cache)
+ [(mdb/unique-identifier) @user-id]))
+ "with-temp evicted the stale cache entry"))))
+
(deftest user->personal-collection-names-test
(is (= {(mt/user->id :rasta) "Rasta Toucan's Personal Collection"
(mt/user->id :lucky) "Lucky Pigeon's Personal Collection"}
diff --git a/test/metabase/dashboards_rest/api_test.clj b/test/metabase/dashboards_rest/api_test.clj
index 2013c85e8d2..202376e5df1 100644
--- a/test/metabase/dashboards_rest/api_test.clj
+++ b/test/metabase/dashboards_rest/api_test.clj
@@ -3486,7 +3486,11 @@
(deftest chain-filter-should-use-cached-field-values-test
(testing "Chain filter endpoints should use cached FieldValues if applicable (#13832)"
;; ignore the cache entries added by #23699
- (mt/with-temp-vals-in-db :model/FieldValues (t2/select-one-pk :model/FieldValues :field_id (mt/id :categories :name) :hash_key nil) {:values ["Good" "Bad"]}
+ ;; Request the complete FieldValues instead of relying on another test to create it. The row is lazy, and
+ ;; creating it within `with-temp` rolls it back afterward.
+ (mt/with-temp-vals-in-db :model/FieldValues (field-values/get-or-create-full-field-values!
+ (t2/select-one :model/Field :id (mt/id :categories :name)))
+ {:values ["Good" "Bad"]}
(with-chain-filter-fixtures [{:keys [dashboard]}]
(testing "GET /api/dashboard/:id/params/:param-key/values"
(mt/let-url [url (chain-filter-values-url dashboard "_CATEGORY_NAME_")]
diff --git a/test/metabase/documents/search_test.clj b/test/metabase/documents/search_test.clj
index 04f4c9018cb..7a7308907ed 100644
--- a/test/metabase/documents/search_test.clj
+++ b/test/metabase/documents/search_test.clj
@@ -87,53 +87,59 @@
"User with write permissions should see archived documents in accessible collections"))))))))
(deftest document-view-tracking-integration-test
- (testing "Document view tracking integrates with search"
- (mt/with-temp [:model/Document {doc-id :id} {:name "Viewed Document"
- :document (documents.test-util/text->prose-mirror-ast "content")
- :view_count 0}]
- (testing "Document has initial state"
- (let [doc (t2/select-one :model/Document :id doc-id)]
- (is (= 0 (:view_count doc)))
- ;; last_viewed_at has a default timestamp from migration, not nil
- (is (some? (:last_viewed_at doc)))))
- (testing "Publishing document-read event works without errors"
- ;; The actual view count increment is batched and asynchronous
- ;; So we test that the event publishes successfully
- (is (some? (events/publish-event! :event/document-read
- {:object-id doc-id
- :user-id (mt/user->id :crowberto)}))
- "Document read event should publish successfully"))
- (testing "Document with higher view count appears in search"
- ;; Manually set view count to test search integration
- (t2/update! :model/Document doc-id {:view_count 5
- :last_viewed_at (t/offset-date-time)})
- (search.tu/with-temp-index-table
- (let [results (mt/user-http-request :crowberto :get 200 "search" {:q "Viewed" :models "document"})
- doc-results (filter #(= "document" (:model %)) (:data results))]
- (when (seq doc-results)
- (let [doc-result (first doc-results)]
- (is (= doc-id (:id doc-result)))
- ;; View count affects scoring but isn't directly in result
- (is (some #(= "view-count" (:name %)) (:scores doc-result))))))))
- (testing "Document with recent view appears in search results"
- (let [recent-time (t/minus (t/offset-date-time) (t/minutes 5))]
- (t2/update! :model/Document doc-id {:last_viewed_at recent-time})
+ ;; Create the index table before `with-temp` opens its transaction. On H2 and MySQL, DDL on the ambient
+ ;; connection would commit that transaction, preventing rollback and leaking rows into later tests. The nested
+ ;; scope below reuses this table. Use `-if-supported` because the rest of the test does not require an index and
+ ;; should still run where the app DB cannot support one.
+ (search.tu/with-temp-index-table-if-supported
+ (testing "Document view tracking integrates with search"
+ (mt/with-temp [:model/Document {doc-id :id} {:name "Viewed Document"
+ :document (documents.test-util/text->prose-mirror-ast "content")
+ :view_count 0}]
+ (testing "Document has initial state"
+ (let [doc (t2/select-one :model/Document :id doc-id)]
+ (is (= 0 (:view_count doc)))
+ ;; The migration gives `last_viewed_at` a non-nil default timestamp.
+ (is (some? (:last_viewed_at doc)))))
+ (testing "Publishing document-read event works without errors"
+ ;; View-count updates are asynchronous and batched, so assert only that publishing succeeds.
+ (is (some? (events/publish-event! :event/document-read
+ {:object-id doc-id
+ :user-id (mt/user->id :crowberto)}))
+ "Document read event should publish successfully"))
+ (testing "Document with higher view count appears in search"
+ ;; Set the view count directly to test its search integration.
+ (t2/update! :model/Document doc-id {:view_count 5
+ :last_viewed_at (t/offset-date-time)})
(search.tu/with-temp-index-table
(let [results (mt/user-http-request :crowberto :get 200 "search" {:q "Viewed" :models "document"})
doc-results (filter #(= "document" (:model %)) (:data results))]
(when (seq doc-results)
(let [doc-result (first doc-results)]
(is (= doc-id (:id doc-result)))
- ;; User recency affects scoring
- (is (some #(= "user-recency" (:name %)) (:scores doc-result))))))))))))
+ ;; View count affects scoring but is not included directly in the result.
+ (is (some #(= "view-count" (:name %)) (:scores doc-result))))))))
+ (testing "Document with recent view appears in search results"
+ (let [recent-time (t/minus (t/offset-date-time) (t/minutes 5))]
+ (t2/update! :model/Document doc-id {:last_viewed_at recent-time})
+ (search.tu/with-temp-index-table
+ (let [results (mt/user-http-request :crowberto :get 200 "search" {:q "Viewed" :models "document"})
+ doc-results (filter #(= "document" (:model %)) (:data results))]
+ (when (seq doc-results)
+ (let [doc-result (first doc-results)]
+ (is (= doc-id (:id doc-result)))
+ ;; View recency contributes the `user-recency` score.
+ (is (some #(= "user-recency" (:name %)) (:scores doc-result)))))))))))))
(deftest document-content-search-test
(testing "Documents are searchable by their body content, not just their name (UXW-4199)"
;; Both engines search document bodies: the appdb engine indexes clean text (via ast->text),
;; the legacy in-place engine LIKE-matches the raw prose-mirror JSON.
- (mt/with-temp [:model/Document {doc-id :id} {:name "Annual Summary"
- :document (documents.test-util/text->prose-mirror-ast "quarterly revenue projections and growth")}]
- (search.tu/with-appdb-search-and-legacy-search
+ ;; The index scope must sit outside `with-temp`: it creates and drops the index table on the ambient
+ ;; connection, and on H2 and MySQL that DDL commits the transaction `with-temp` meant to roll back.
+ (search.tu/with-appdb-search-and-legacy-search
+ (mt/with-temp [:model/Document {doc-id :id} {:name "Annual Summary"
+ :document (documents.test-util/text->prose-mirror-ast "quarterly revenue projections and growth")}]
(testing "found by a term that appears only in the body"
(let [results (mt/user-http-request :crowberto :get 200 "search" :q "projections" :models "document")]
(is (contains? (set (map :id (:data results))) doc-id))))
@@ -146,14 +152,15 @@
(deftest document-smart-link-label-search-test
(testing "Documents are searchable by the visible label of an embedded smart link (UXW-4199)"
- (mt/with-temp [:model/Document {doc-id :id}
- {:name "Reference Doc"
- :document {:type "doc"
- :content [{:type "paragraph"
- :content [{:type "text" :text "see"}
- {:type "smartLink"
- :attrs {:model "card" :entityId "abc" :label "Customer Retention"}}]}]}}]
- (search.tu/with-appdb-search-and-legacy-search
+ ;; The index scope must sit outside `with-temp`, as in `document-content-search-test` above.
+ (search.tu/with-appdb-search-and-legacy-search
+ (mt/with-temp [:model/Document {doc-id :id}
+ {:name "Reference Doc"
+ :document {:type "doc"
+ :content [{:type "paragraph"
+ :content [{:type "text" :text "see"}
+ {:type "smartLink"
+ :attrs {:model "card" :entityId "abc" :label "Customer Retention"}}]}]}}]
(testing "found by a term that appears only in the smart-link label"
(let [results (mt/user-http-request :crowberto :get 200 "search" :q "Retention" :models "document")]
(is (contains? (set (map :id (:data results))) doc-id))))))))
diff --git a/test/metabase/metabot/tools/entity_details_test.clj b/test/metabase/metabot/tools/entity_details_test.clj
index e95a8f364f4..cf584a0ed92 100644
--- a/test/metabase/metabot/tools/entity_details_test.clj
+++ b/test/metabase/metabot/tools/entity_details_test.clj
@@ -578,6 +578,25 @@
(is (vector? (get-in exported ["stages" 0 "source-table"])))
(is (not (contains? exported "lib/metadata")))))))))
+(deftest card-details-tolerates-a-query-that-will-not-build-test
+ (testing "one Card whose stored query has no stages does not take down the response the others are in"
+ (mt/test-driver :h2
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-temp [:model/Card broken {:name "Broken", :type :question, :dataset_query {}}
+ :model/Card good {:database_id (mt/id)
+ :type :question
+ :name "Venues by Price"
+ :dataset_query (mt/mbql-query venues
+ {:aggregation [[:count]]
+ :breakout [$price]})}]
+ ;; Exercise `cards-details`, the batch path used by the typed-schemas endpoint. Calling
+ ;; `get-table-details` for each Card would miss a failure that terminates the complete sequence.
+ (let [details (->> (entity-details/cards-details :question (mt/id) [broken good] {})
+ (into [] (map #(select-keys % [:name :query_json]))))]
+ (is (= ["Broken" "Venues by Price"] (mapv :name details)))
+ (is (nil? (:query_json (first details))))
+ (is (map? (:query_json (second details))))))))))
+
(deftest card-details-exposes-query-json-native-test
(testing "card-details surfaces native saved queries as a portable repr map, preserving the SQL inside"
(mt/test-driver :h2
diff --git a/test/metabase/queries_rest/api/card_test.clj b/test/metabase/queries_rest/api/card_test.clj
index f1738c26e49..6955cd45395 100644
--- a/test/metabase/queries_rest/api/card_test.clj
+++ b/test/metabase/queries_rest/api/card_test.clj
@@ -631,7 +631,9 @@
[:model/Card {card-id :id} {:name "Card"
:display "line"
:dataset_query (mt/mbql-query venues)
- :collection_id (t2/select-one-pk :model/Collection :personal_owner_id (mt/user->id :crowberto))}]
+ ;; Use get-or-create because the Personal Collection may not exist yet. A nil
+ ;; `collection_id` would place the Card in the root Collection, which Rasta can read.
+ :collection_id (u/the-id (collection/user->personal-collection (mt/user->id :crowberto)))}]
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :get 403 (format "card/%d/series" card-id))))
(is (seq? (mt/user-http-request :crowberto :get 200 (format "card/%d/series" card-id))))))
@@ -1344,52 +1346,55 @@
(deftest updating-model-query-does-not-shift-metadata-overrides-test
(testing "Metadata should not shift to another column with the same name when the query changes (#60930)"
- (let [mp (mt/metadata-provider)
- orders-table (lib.metadata/table mp (mt/id :orders))
- products-table (lib.metadata/table mp (mt/id :products))
- reviews-table (lib.metadata/table mp (mt/id :reviews))
- orders-id (lib.metadata/field mp (mt/id :orders :id))
- orders-user-id (lib.metadata/field mp (mt/id :orders :user_id))
- orders-product-id (lib.metadata/field mp (mt/id :orders :product_id))
- orders-created-at (lib.metadata/field mp (mt/id :orders :created_at))
- products-id (lib.metadata/field mp (mt/id :products :id))
- products-created (lib.metadata/field mp (mt/id :products :created_at))
- reviews-id (lib.metadata/field mp (mt/id :reviews :id))
- reviews-product-id (lib.metadata/field mp (mt/id :reviews :product_id))
- reviews-created-at (lib.metadata/field mp (mt/id :reviews :created_at))
- join-query (fn [products-fields]
- (-> (lib/query mp orders-table)
- (lib/with-fields [orders-id orders-user-id orders-product-id orders-created-at])
- (lib/join (-> (lib/join-clause products-table
- [(lib/= orders-product-id products-id)])
- (lib/with-join-alias "Products")
- (lib/with-join-fields products-fields)))
- (lib/join (-> (lib/join-clause reviews-table
- [(lib/= orders-product-id reviews-product-id)])
- (lib/with-join-alias "Reviews")
- (lib/with-join-fields [reviews-id reviews-product-id reviews-created-at])))))
- with-products (mt/user-http-request :crowberto :post 200 "card"
- {:name "model 60930"
- :type :model
- :display :table
- :dataset_query (join-query [products-id products-created])
- :visualization_settings {}})
- card-id (:id with-products)
- without-products (mt/user-http-request :crowberto :put 200 (str "card/" card-id)
- {:dataset_query (join-query :none)})]
- (testing "columns get the correct display name after columns with the same name are removed"
- (is (= ["ID" "User ID" "Product ID" "Created At"
- "Reviews → ID" "Reviews → Product ID" "Reviews → Created At"]
- (map :display_name (:result_metadata without-products))
- (map :display_name (t2/select-one-fn :result_metadata :model/Card :id card-id)))))
- (let [with-products-again (mt/user-http-request :crowberto :put 200 (str "card/" card-id)
- {:dataset_query (join-query [products-id products-created])})]
- (testing "columns get the correct display name after columns with the same name are added"
+ ;; Clean up Revisions explicitly. Direct Card cleanup bypasses the hook that would normally delete the
+ ;; Revisions created by this test's POST and PUT requests.
+ (mt/with-model-cleanup [:model/Card :model/Revision]
+ (let [mp (mt/metadata-provider)
+ orders-table (lib.metadata/table mp (mt/id :orders))
+ products-table (lib.metadata/table mp (mt/id :products))
+ reviews-table (lib.metadata/table mp (mt/id :reviews))
+ orders-id (lib.metadata/field mp (mt/id :orders :id))
+ orders-user-id (lib.metadata/field mp (mt/id :orders :user_id))
+ orders-product-id (lib.metadata/field mp (mt/id :orders :product_id))
+ orders-created-at (lib.metadata/field mp (mt/id :orders :created_at))
+ products-id (lib.metadata/field mp (mt/id :products :id))
+ products-created (lib.metadata/field mp (mt/id :products :created_at))
+ reviews-id (lib.metadata/field mp (mt/id :reviews :id))
+ reviews-product-id (lib.metadata/field mp (mt/id :reviews :product_id))
+ reviews-created-at (lib.metadata/field mp (mt/id :reviews :created_at))
+ join-query (fn [products-fields]
+ (-> (lib/query mp orders-table)
+ (lib/with-fields [orders-id orders-user-id orders-product-id orders-created-at])
+ (lib/join (-> (lib/join-clause products-table
+ [(lib/= orders-product-id products-id)])
+ (lib/with-join-alias "Products")
+ (lib/with-join-fields products-fields)))
+ (lib/join (-> (lib/join-clause reviews-table
+ [(lib/= orders-product-id reviews-product-id)])
+ (lib/with-join-alias "Reviews")
+ (lib/with-join-fields [reviews-id reviews-product-id reviews-created-at])))))
+ with-products (mt/user-http-request :crowberto :post 200 "card"
+ {:name "model 60930"
+ :type :model
+ :display :table
+ :dataset_query (join-query [products-id products-created])
+ :visualization_settings {}})
+ card-id (:id with-products)
+ without-products (mt/user-http-request :crowberto :put 200 (str "card/" card-id)
+ {:dataset_query (join-query :none)})]
+ (testing "columns get the correct display name after columns with the same name are removed"
(is (= ["ID" "User ID" "Product ID" "Created At"
- "Products → ID" "Products → Created At"
"Reviews → ID" "Reviews → Product ID" "Reviews → Created At"]
- (map :display_name (:result_metadata with-products-again))
- (map :display_name (t2/select-one-fn :result_metadata :model/Card :id card-id)))))))))
+ (map :display_name (:result_metadata without-products))
+ (map :display_name (t2/select-one-fn :result_metadata :model/Card :id card-id)))))
+ (let [with-products-again (mt/user-http-request :crowberto :put 200 (str "card/" card-id)
+ {:dataset_query (join-query [products-id products-created])})]
+ (testing "columns get the correct display name after columns with the same name are added"
+ (is (= ["ID" "User ID" "Product ID" "Created At"
+ "Products → ID" "Products → Created At"
+ "Reviews → ID" "Reviews → Product ID" "Reviews → Created At"]
+ (map :display_name (:result_metadata with-products-again))
+ (map :display_name (t2/select-one-fn :result_metadata :model/Card :id card-id))))))))))
(deftest ^:parallel updating-native-card-preserves-metadata
(testing "A trivial change in a native question should not remove result_metadata (#37009)"
@@ -1566,7 +1571,9 @@
(create-card! :rasta 403))))))))))
(deftest ^:parallel create-card-with-type-and-dataset-test
- (t2/with-transaction [_]
+ ;; Use `:rollback-only` like the sibling tests below. Otherwise, the two Cards created through the API
+ ;; commit and leak into later tests that scan the Card table.
+ (t2/with-transaction [_ nil {:rollback-only true}]
(testing "can create a model using type"
(is (=? {:type "model"}
(mt/user-http-request :crowberto :post 200 "card" (assoc (card-with-name-and-query (mt/random-name))
@@ -5198,7 +5205,10 @@
resp (mt/user-http-request :rasta :post 202 (format "card/%d/query" (:id card)))]
(is (= 3 (count (get-in resp [:data :rows]))))))))))
-(deftest ^:parallel reduced-fields-propagate-to-downstream-card-test
+;; Not `^:parallel`: running alongside other tests, something commits the transaction holding this test's
+;; rollback-only savepoint, which discards both `with-temp` Cards. The downstream query then reports its source Card
+;; as missing. Only MySQL shows it, and only under load -- the test passes on its own.
+(deftest ^:synchronized reduced-fields-propagate-to-downstream-card-test
(testing "A card with reduced :fields only exposes those columns to a card sourced from it (#30610)"
(let [mp (mt/metadata-provider)
venues-id (lib.metadata/field mp (mt/id :venues :id))
diff --git a/test/metabase/query_processor/card_test.clj b/test/metabase/query_processor/card_test.clj
index 6ae2128612b..1594288b574 100644
--- a/test/metabase/query_processor/card_test.clj
+++ b/test/metabase/query_processor/card_test.clj
@@ -21,8 +21,6 @@
[metabase.query-processor.test :as qp]
[metabase.query-processor.test-util :as qp.test-util]
[metabase.test :as mt]
- [metabase.test.data.users :as test.users]
- [metabase.test.http-client :as client]
[metabase.util :as u]
[metabase.util.json :as json]))
@@ -381,8 +379,11 @@
:dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}]
(doseq [export-format [:csv :json :xlsx]]
(testing (str "format: " export-format)
- (let [response (client/client-full-response
- (test.users/username->token :crowberto)
+ ;; through `user-http-request-full-response`, not a raw cached token: that skips the retry in
+ ;; `client-fn`, so a session that was rolled back with the scope that made it comes back here
+ ;; as a bare 401
+ (let [response (mt/user-http-request-full-response
+ :crowberto
:post 200
(format "card/%d/query/%s" (:id card) (name export-format))
{})]
diff --git a/test/metabase/search/api_test.clj b/test/metabase/search/api_test.clj
index 46645c30efd..5831c6f1248 100644
--- a/test/metabase/search/api_test.clj
+++ b/test/metabase/search/api_test.clj
@@ -2274,22 +2274,27 @@
:models "measure")))))))))
(deftest ^:synchronized search-results-do-not-expose-is-published-test
- (testing "the internal is_published permission signal never carries a value in search API responses"
- (let [table-name (mt/random-name)]
- (mt/with-temp [:model/Table _ {:name table-name :is_published true}]
- (testing "in-place engine"
- (let [rows (:data (mt/user-http-request :crowberto :get 200 "search"
- :q table-name :models "table" :search_engine "in-place"))]
- (is (seq rows))
- (is (every? (comp nil? :is_published) rows))))
- (when (search/supports-index?)
- (testing "appdb engine"
- (search.tu/with-temp-index-table
- (search/reindex! {:async? false :in-place? true})
- (let [rows (:data (mt/user-http-request :crowberto :get 200 "search"
- :q table-name :models "table" :search_engine "appdb"))]
- (is (seq rows))
- (is (every? (comp nil? :is_published) rows))))))))))
+ ;; Create the index table before `with-temp` opens its transaction. On H2 and MySQL, DDL on the ambient
+ ;; connection would commit that transaction, preventing rollback and leaking rows into later tests. The nested
+ ;; scope below reuses this table. Use `-if-supported` because the rest of the test does not require an index and
+ ;; should still run where the app DB cannot support one.
+ (search.tu/with-temp-index-table-if-supported
+ (testing "the internal is_published permission signal never carries a value in search API responses"
+ (let [table-name (mt/random-name)]
+ (mt/with-temp [:model/Table _ {:name table-name :is_published true}]
+ (testing "in-place engine"
+ (let [rows (:data (mt/user-http-request :crowberto :get 200 "search"
+ :q table-name :models "table" :search_engine "in-place"))]
+ (is (seq rows))
+ (is (every? (comp nil? :is_published) rows))))
+ (when (search/supports-index?)
+ (testing "appdb engine"
+ (search.tu/with-temp-index-table
+ (search/reindex! {:async? false :in-place? true})
+ (let [rows (:data (mt/user-http-request :crowberto :get 200 "search"
+ :q table-name :models "table" :search_engine "appdb"))]
+ (is (seq rows))
+ (is (every? (comp nil? :is_published) rows)))))))))))
(deftest exploration-description-searchable-in-place-test
(testing "explorations match on :description in the in-place engine (parity with the appdb spec)"
diff --git a/test/metabase/search/appdb/scoring_test.clj b/test/metabase/search/appdb/scoring_test.clj
index 81045867e9c..159a1e3b4db 100644
--- a/test/metabase/search/appdb/scoring_test.clj
+++ b/test/metabase/search/appdb/scoring_test.clj
@@ -14,6 +14,7 @@
[metabase.search.spec :as search.spec]
[metabase.search.test-util :as search.tu]
[metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
[metabase.util :as u]
[toucan2.core :as t2])
(:import
@@ -22,6 +23,10 @@
(set! *warn-on-reflection* true)
+;; Initialize before any tests run. The index scopes below write search-index bookkeeping; a later lazy
+;; initialization would recreate the H2 test DB while a test is running.
+(use-fixtures :once (fixtures/initialize :db :test-users))
+
;; We act on a random localized table, making this thread-safe.
(defmacro with-index-contents
"Populate the index with the given appdb agnostic entity shapes."
@@ -256,72 +261,80 @@
;; These require some related appdb content
(deftest bookmark-test
- (let [crowberto (mt/user->id :crowberto)
- rasta (mt/user->id :rasta)]
- (mt/with-temp [:model/Card {c1 :id} {}
- :model/Card {c2 :id} {}]
- (testing "bookmarked items are ranker higher"
- (with-index-contents
- [{:model "card" :id c1 :name "card normal"}
- {:model "card" :id c2 :name "card crowberto loved"}]
- (mt/with-temp [:model/CardBookmark _ {:card_id c2 :user_id crowberto}
- :model/CardBookmark _ {:card_id c1 :user_id rasta}]
- (is (= [["card" c2 "card crowberto loved"]
- ["card" c1 "card normal"]]
- (search-results :bookmarked "card" {:current-user-id crowberto})))))))
- (mt/with-temp [:model/Dashboard {d1 :id} {}
- :model/Dashboard {d2 :id} {}]
- (testing "bookmarked dashboard"
- (with-index-contents
- [{:model "dashboard" :id d1 :name "dashboard normal"}
- {:model "dashboard" :id d2 :name "dashboard crowberto loved"}]
- (mt/with-temp [:model/DashboardBookmark _ {:dashboard_id d2 :user_id crowberto}
- :model/DashboardBookmark _ {:dashboard_id d1 :user_id rasta}]
- (is (= [["dashboard" d2 "dashboard crowberto loved"]
- ["dashboard" d1 "dashboard normal"]]
- (search-results :bookmarked "dashboard" {:current-user-id crowberto})))))))
- (mt/with-temp [:model/Collection {c1 :id} {}
- :model/Collection {c2 :id} {}]
- (testing "bookmarked collection"
- (with-index-contents
- [{:model "collection" :id c1 :name "collection normal"}
- {:model "collection" :id c2 :name "collection crowberto loved"}]
- (mt/with-temp [:model/CollectionBookmark _ {:collection_id c2 :user_id crowberto}
- :model/CollectionBookmark _ {:collection_id c1 :user_id rasta}]
- (is (= [["collection" c2 "collection crowberto loved"]
- ["collection" c1 "collection normal"]]
- (search-results :bookmarked "collection" {:current-user-id crowberto})))))))))
+ ;; Create the temporary index table before `with-temp` opens its transaction. On H2 and MySQL, DDL on the
+ ;; ambient connection would implicitly commit that transaction, preventing rollback and leaking rows into later
+ ;; tests. The nested index-table scope below reuses this table.
+ (search.tu/with-temp-index-table
+ (let [crowberto (mt/user->id :crowberto)
+ rasta (mt/user->id :rasta)]
+ (mt/with-temp [:model/Card {c1 :id} {}
+ :model/Card {c2 :id} {}]
+ (testing "bookmarked items are ranker higher"
+ (with-index-contents
+ [{:model "card" :id c1 :name "card normal"}
+ {:model "card" :id c2 :name "card crowberto loved"}]
+ (mt/with-temp [:model/CardBookmark _ {:card_id c2 :user_id crowberto}
+ :model/CardBookmark _ {:card_id c1 :user_id rasta}]
+ (is (= [["card" c2 "card crowberto loved"]
+ ["card" c1 "card normal"]]
+ (search-results :bookmarked "card" {:current-user-id crowberto})))))))
+ (mt/with-temp [:model/Dashboard {d1 :id} {}
+ :model/Dashboard {d2 :id} {}]
+ (testing "bookmarked dashboard"
+ (with-index-contents
+ [{:model "dashboard" :id d1 :name "dashboard normal"}
+ {:model "dashboard" :id d2 :name "dashboard crowberto loved"}]
+ (mt/with-temp [:model/DashboardBookmark _ {:dashboard_id d2 :user_id crowberto}
+ :model/DashboardBookmark _ {:dashboard_id d1 :user_id rasta}]
+ (is (= [["dashboard" d2 "dashboard crowberto loved"]
+ ["dashboard" d1 "dashboard normal"]]
+ (search-results :bookmarked "dashboard" {:current-user-id crowberto})))))))
+ (mt/with-temp [:model/Collection {c1 :id} {}
+ :model/Collection {c2 :id} {}]
+ (testing "bookmarked collection"
+ (with-index-contents
+ [{:model "collection" :id c1 :name "collection normal"}
+ {:model "collection" :id c2 :name "collection crowberto loved"}]
+ (mt/with-temp [:model/CollectionBookmark _ {:collection_id c2 :user_id crowberto}
+ :model/CollectionBookmark _ {:collection_id c1 :user_id rasta}]
+ (is (= [["collection" c2 "collection crowberto loved"]
+ ["collection" c1 "collection normal"]]
+ (search-results :bookmarked "collection" {:current-user-id crowberto}))))))))))
(deftest user-recency-test
- (let [user-id (mt/user->id :crowberto)
- right-now (Instant/now)
- long-ago (.minus right-now 10 ChronoUnit/DAYS)
- forever-ago (.minus right-now 30 ChronoUnit/DAYS)
- recent-view (fn [model-id timestamp]
- {:model "card"
- :model_id model-id
- :user_id user-id
- :timestamp timestamp})]
- (mt/with-temp [:model/Card {c1 :id} {}
- :model/Card {c2 :id} {}
- :model/Card {c3 :id} {}
- :model/Card {c4 :id} {}
- :model/RecentViews _ (recent-view c1 forever-ago)
- :model/RecentViews _ (recent-view c2 right-now)
- :model/RecentViews _ (recent-view c2 forever-ago)
- :model/RecentViews _ (recent-view c4 forever-ago)
- :model/RecentViews _ (recent-view c4 long-ago)]
- (with-index-contents
- [{:model "card" :id c1 :name "card ancient"}
- {:model "metric" :id c2 :name "card recent"}
- {:model "dataset" :id c3 :name "card unseen"}
- {:model "dataset" :id c4 :name "card old"}]
- (testing "We prefer results more recently viewed by the current user"
- (is (= [["metric" c2 "card recent"]
- ["dataset" c4 "card old"]
- ["card" c1 "card ancient"]
- ["dataset" c3 "card unseen"]]
- (search-results :user-recency "card" {:current-user-id user-id}))))))))
+ ;; Create the temporary index table before `with-temp` opens its transaction. On H2 and MySQL, DDL on the
+ ;; ambient connection would implicitly commit that transaction, preventing rollback and leaking rows into later
+ ;; tests. The nested index-table scope below reuses this table.
+ (search.tu/with-temp-index-table
+ (let [user-id (mt/user->id :crowberto)
+ right-now (Instant/now)
+ long-ago (.minus right-now 10 ChronoUnit/DAYS)
+ forever-ago (.minus right-now 30 ChronoUnit/DAYS)
+ recent-view (fn [model-id timestamp]
+ {:model "card"
+ :model_id model-id
+ :user_id user-id
+ :timestamp timestamp})]
+ (mt/with-temp [:model/Card {c1 :id} {}
+ :model/Card {c2 :id} {}
+ :model/Card {c3 :id} {}
+ :model/Card {c4 :id} {}
+ :model/RecentViews _ (recent-view c1 forever-ago)
+ :model/RecentViews _ (recent-view c2 right-now)
+ :model/RecentViews _ (recent-view c2 forever-ago)
+ :model/RecentViews _ (recent-view c4 forever-ago)
+ :model/RecentViews _ (recent-view c4 long-ago)]
+ (with-index-contents
+ [{:model "card" :id c1 :name "card ancient"}
+ {:model "metric" :id c2 :name "card recent"}
+ {:model "dataset" :id c3 :name "card unseen"}
+ {:model "dataset" :id c4 :name "card old"}]
+ (testing "We prefer results more recently viewed by the current user"
+ (is (= [["metric" c2 "card recent"]
+ ["dataset" c4 "card old"]
+ ["card" c1 "card ancient"]
+ ["dataset" c3 "card unseen"]]
+ (search-results :user-recency "card" {:current-user-id user-id})))))))))
(deftest mine-test
(let [crowberto (mt/user->id :crowberto)
@@ -336,30 +349,34 @@
(search-results :mine "card" {:current-user-id rasta}))))))
(deftest library-test
- (testing "Library-collection cards rank above non-library cards"
- ;; Real Collections are needed in appdb so the `:root-collection-type` fn attr can resolve the
- ;; top-level ancestor's `:type` from each row's `:collection_location` materialized path.
- (mt/with-temp [:model/Collection lib {:name "lib top" :type "library" :location "/"}
- :model/Collection lib-data {:name "lib data" :type "library-data" :location "/"}
- :model/Collection lib-met {:name "lib metrics" :type "library-metrics" :location "/"}
- :model/Collection sub {:name "sub of lib" :location (format "/%d/" (:id lib))}
- :model/Collection sub-sub {:name "sub of sub" :location (format "/%d/%d/" (:id lib) (:id sub))}
- :model/Collection other {:name "non-library" :location "/"}]
- (with-index-contents
- [{:model "card" :id 1 :name "plain card" :collection_id (:id other) :collection_location (:location other)}
- {:model "card" :id 2 :name "lib-tree card library" :collection_id (:id lib) :collection_location (:location lib) :collection_type "library"}
- {:model "card" :id 3 :name "lib-tree card library-data" :collection_id (:id lib-data) :collection_location (:location lib-data) :collection_type "library-data"}
- {:model "card" :id 4 :name "lib-tree card library-metrics" :collection_id (:id lib-met) :collection_location (:location lib-met) :collection_type "library-metrics"}
- {:model "card" :id 5 :name "lib-tree card sub" :collection_id (:id sub) :collection_location (:location sub)}
- {:model "card" :id 6 :name "lib-tree card sub-sub" :collection_id (:id sub-sub) :collection_location (:location sub-sub)}
- {:model "card" :id 7 :name "trashed card" :collection_type "trash"}]
- (let [in-library? (fn [[_ _ nm]] (str/includes? nm "lib-tree"))]
- (testing "with positive :library weight, items inside library trees come first"
- (is (= [true true true true true false false]
- (map in-library? (with-weights {:library 1} (search-results* "card"))))))
- (testing "with negative :library weight, library items come last"
- (is (= [false false true true true true true]
- (map in-library? (with-weights {:library -1} (search-results* "card")))))))))))
+ ;; Create the temporary index table before `with-temp` opens its transaction. On H2 and MySQL, DDL on the
+ ;; ambient connection would implicitly commit that transaction, preventing rollback and leaking rows into later
+ ;; tests. The nested index-table scope below reuses this table.
+ (search.tu/with-temp-index-table
+ (testing "Library-collection cards rank above non-library cards"
+ ;; Use real Collections so the app DB index's `:root-collection-type` function attribute can resolve the
+ ;; top-level ancestor's `:type` from each row's materialized `:collection_location` path.
+ (mt/with-temp [:model/Collection lib {:name "lib top" :type "library" :location "/"}
+ :model/Collection lib-data {:name "lib data" :type "library-data" :location "/"}
+ :model/Collection lib-met {:name "lib metrics" :type "library-metrics" :location "/"}
+ :model/Collection sub {:name "sub of lib" :location (format "/%d/" (:id lib))}
+ :model/Collection sub-sub {:name "sub of sub" :location (format "/%d/%d/" (:id lib) (:id sub))}
+ :model/Collection other {:name "non-library" :location "/"}]
+ (with-index-contents
+ [{:model "card" :id 1 :name "plain card" :collection_id (:id other) :collection_location (:location other)}
+ {:model "card" :id 2 :name "lib-tree card library" :collection_id (:id lib) :collection_location (:location lib) :collection_type "library"}
+ {:model "card" :id 3 :name "lib-tree card library-data" :collection_id (:id lib-data) :collection_location (:location lib-data) :collection_type "library-data"}
+ {:model "card" :id 4 :name "lib-tree card library-metrics" :collection_id (:id lib-met) :collection_location (:location lib-met) :collection_type "library-metrics"}
+ {:model "card" :id 5 :name "lib-tree card sub" :collection_id (:id sub) :collection_location (:location sub)}
+ {:model "card" :id 6 :name "lib-tree card sub-sub" :collection_id (:id sub-sub) :collection_location (:location sub-sub)}
+ {:model "card" :id 7 :name "trashed card" :collection_type "trash"}]
+ (let [in-library? (fn [[_ _ nm]] (str/includes? nm "lib-tree"))]
+ (testing "with positive :library weight, items inside library trees come first"
+ (is (= [true true true true true false false]
+ (map in-library? (with-weights {:library 1} (search-results* "card"))))))
+ (testing "with negative :library weight, library items come last"
+ (is (= [false false true true true true true]
+ (map in-library? (with-weights {:library -1} (search-results* "card"))))))))))))
(deftest ^:parallel data-layer-test
(testing ":data-layer scorer reads the active per-tier weight via :data-layer/* params"
diff --git a/test/metabase/search/test_util.clj b/test/metabase/search/test_util.clj
index 67898ee42c7..463596f94c9 100644
--- a/test/metabase/search/test_util.clj
+++ b/test/metabase/search/test_util.clj
@@ -20,14 +20,29 @@
~@body))
(defmacro with-temp-index-table
- "Create a temporary index table for the duration of the body."
+ "Create a temporary index table for the duration of the body.
+
+ The table belongs to the app DB, so app-DB support is the condition. [[metabase.search.core/supports-index?]] is
+ true for any active engine, so with semantic search enabled it would make MySQL and MariaDB try to build a table
+ their app DB cannot hold."
[& body]
- `(when (search/supports-index?)
+ `(when (search.engine/supported-engine? :search.engine/appdb)
(search.index/with-temp-index-table
;; We need ingestion to happen on the same thread so that it uses the right search index.
(with-sync-search-indexing
~@body))))
+(defmacro with-temp-index-table-if-supported
+ "Like [[with-temp-index-table]], but always runs `body`, even when the app DB cannot support an index.
+
+ Use this for tests that cover both indexed and non-indexed behavior. Wrapping such a test in
+ [[with-temp-index-table]] would skip it entirely on MySQL and MariaDB."
+ [& body]
+ `(let [thunk# (fn [] ~@body)]
+ (if (search.engine/supported-engine? :search.engine/appdb)
+ (with-temp-index-table (thunk#))
+ (thunk#))))
+
(defmacro with-appdb-search-if-available*
"Create a temporary index table for the duration of the body."
[& body]
@@ -42,20 +57,6 @@
(search/reindex! {:async? false :in-place? true})
~@body)))
-(defmacro with-appdb-search-if-available-otherwise-legacy
- "Create a temporary index table for the duration of the body."
- [& body]
- `(if (search/supports-index?)
- (with-appdb-search-if-available* ~@body)
- ~@body))
-
-(defmacro with-appdb-search-if-available-without-fallback
- "Create a temporary index table for the duration of the body.
- Only runs if the appdb search engine is supported."
- [& body]
- `(when (search.engine/supported-engine? :search.engine/appdb)
- (with-appdb-search-if-available* ~@body)))
-
(defmacro with-legacy-search
"Ensure legacy search, which doesn't require an index, is used.
Semantic queries go to :search.engine/semantic and keyword queries fall back to :search.engine/in-place."
@@ -69,6 +70,22 @@
[:search.engine/semantic]))]
~@body))
+(defmacro with-appdb-search-if-available-otherwise-legacy
+ "Create a temporary index table for the duration of the body, or run it under legacy search."
+ [& body]
+ `(if (search.engine/supported-engine? :search.engine/appdb)
+ (with-appdb-search-if-available* ~@body)
+ ;; Pin the engine rather than taking whatever the default is. Semantic can be active here, and the body is
+ ;; expects an index that this app DB cannot hold.
+ (with-legacy-search ~@body)))
+
+(defmacro with-appdb-search-if-available-without-fallback
+ "Create a temporary index table for the duration of the body.
+ Only runs if the app DB search engine is supported."
+ [& body]
+ `(when (search.engine/supported-engine? :search.engine/appdb)
+ (with-appdb-search-if-available* ~@body)))
+
(defmacro with-appdb-search-and-legacy-search
"Run the body twice, once with the legacy search engine, and once with the appdb search engine."
[& body]
diff --git a/test/metabase/task_history/task/task_run_heartbeat_test.clj b/test/metabase/task_history/task/task_run_heartbeat_test.clj
index 5c64684e069..614f002a9bf 100644
--- a/test/metabase/task_history/task/task_run_heartbeat_test.clj
+++ b/test/metabase/task_history/task/task_run_heartbeat_test.clj
@@ -17,7 +17,7 @@
(deftest send-heartbeat-updates-own-runs-test
(mt/with-model-cleanup [:model/TaskRun]
(testing "send-heartbeat! updates updated_at for runs belonging to current process"
- (let [old-time (t/minus (t/offset-date-time) (t/hours 3))]
+ (let [old-time (t/truncate-to (t/minus (t/offset-date-time) (t/hours 3)) :millis)]
(mt/with-temp [:model/TaskRun {run-id :id} {:run_type :sync
:entity_type :database
:entity_id 1
@@ -52,7 +52,7 @@
(deftest send-heartbeat-ignores-completed-runs-test
(mt/with-model-cleanup [:model/TaskRun]
(testing "send-heartbeat! does NOT update runs that are already completed"
- (let [old-time (t/minus (t/offset-date-time) (t/hours 3))]
+ (let [old-time (t/truncate-to (t/minus (t/offset-date-time) (t/hours 3)) :millis)]
(mt/with-temp [:model/TaskRun {run-id :id} {:run_type :sync
:entity_type :database
:entity_id 1
@@ -75,7 +75,7 @@
(deftest mark-orphaned-runs-marks-old-runs-test
(mt/with-model-cleanup [:model/TaskRun]
(testing "mark-orphaned-runs! marks runs with old updated_at as :abandoned"
- (let [old-time (t/minus (t/offset-date-time) (t/hours 3))]
+ (let [old-time (t/truncate-to (t/minus (t/offset-date-time) (t/hours 3)) :millis)]
(mt/with-temp [:model/TaskRun {run-id :id} {:run_type :sync
:entity_type :database
:entity_id 1
@@ -91,7 +91,7 @@
(deftest mark-orphaned-runs-ignores-recent-runs-test
(mt/with-model-cleanup [:model/TaskRun]
(testing "mark-orphaned-runs! does NOT mark runs with recent updated_at"
- (let [recent-time (t/minus (t/offset-date-time) (t/minutes 30))]
+ (let [recent-time (t/truncate-to (t/minus (t/offset-date-time) (t/minutes 30)) :millis)]
(mt/with-temp [:model/TaskRun {run-id :id} {:run_type :sync
:entity_type :database
:entity_id 1
@@ -107,7 +107,7 @@
(deftest mark-orphaned-runs-ignores-completed-runs-test
(mt/with-model-cleanup [:model/TaskRun]
(testing "mark-orphaned-runs! does NOT mark already-completed runs"
- (let [old-time (t/minus (t/offset-date-time) (t/hours 3))]
+ (let [old-time (t/truncate-to (t/minus (t/offset-date-time) (t/hours 3)) :millis)]
(mt/with-temp [:model/TaskRun {run-id :id} {:run_type :sync
:entity_type :database
:entity_id 1
@@ -123,7 +123,7 @@
(deftest mark-orphaned-runs-marks-long-running-runs-test
(mt/with-model-cleanup [:model/TaskRun]
(testing "mark-orphaned-runs! marks runs older than 24 hours even with recent heartbeat"
- (let [very-old-time (t/minus (t/offset-date-time) (t/hours 30))
+ (let [very-old-time (t/truncate-to (t/minus (t/offset-date-time) (t/hours 30)) :millis)
recent-time (t/offset-date-time)]
(mt/with-temp [:model/TaskRun {run-id :id} {:run_type :sync
:entity_type :database
@@ -144,7 +144,7 @@
(deftest mark-orphaned-tasks-test
(mt/with-model-cleanup [:model/TaskRun :model/TaskHistory]
(testing "mark-orphaned-tasks! marks started tasks belonging to orphaned runs"
- (let [old-time (t/minus (t/offset-date-time) (t/hours 3))]
+ (let [old-time (t/truncate-to (t/minus (t/offset-date-time) (t/hours 3)) :millis)]
(mt/with-temp [:model/TaskRun {run-id :id} {:run_type :sync
:entity_type :database
:entity_id 1
@@ -165,7 +165,7 @@
(deftest mark-orphaned-tasks-ignores-completed-tasks-test
(mt/with-model-cleanup [:model/TaskRun :model/TaskHistory]
(testing "mark-orphaned-tasks! does NOT mark already-completed tasks"
- (let [old-time (t/minus (t/offset-date-time) (t/hours 3))]
+ (let [old-time (t/truncate-to (t/minus (t/offset-date-time) (t/hours 3)) :millis)]
(mt/with-temp [:model/TaskRun {run-id :id} {:run_type :sync
:entity_type :database
:entity_id 1
@@ -211,7 +211,7 @@
(deftest full-heartbeat-cycle-test
(mt/with-model-cleanup [:model/TaskRun :model/TaskHistory]
(testing "full heartbeat cycle: heartbeat, orphan detection, task cleanup"
- (let [old-time (t/minus (t/offset-date-time) (t/hours 3))]
+ (let [old-time (t/truncate-to (t/minus (t/offset-date-time) (t/hours 3)) :millis)]
;; Create runs: one from current process (should get heartbeat), one from dead process (should be orphaned)
(mt/with-temp [:model/TaskRun {live-run-id :id} {:run_type :sync
:entity_type :database
diff --git a/test/metabase/test/data.clj b/test/metabase/test/data.clj
index bc70661e2d3..73bfcee46e5 100644
--- a/test/metabase/test/data.clj
+++ b/test/metabase/test/data.clj
@@ -320,7 +320,9 @@
`(schema-migrations-test.impl/with-temp-empty-app-db [conn# :h2]
(next.jdbc/execute! conn# ["RUNSCRIPT FROM ?" (str @h2-app-db-script)])
(mdb/finish-db-setup!)
- ~@body))
+ ;; This app DB must remain empty, so `with-temp` must not materialize the test-data Database in it.
+ (binding [data.impl/*skip-dataset-prewarm?* true]
+ ~@body)))
;; Non-"normal" timeseries drivers are tested in [[metabase.query-processor.timeseries-test]] and elsewhere
(def timeseries-drivers
diff --git a/test/metabase/test/data/impl.clj b/test/metabase/test/data/impl.clj
index 1a82d8b7ba2..b0dc1c813df 100644
--- a/test/metabase/test/data/impl.clj
+++ b/test/metabase/test/data/impl.clj
@@ -24,6 +24,12 @@
(p/import-vars
[verify verify-data-loaded-correctly])
+(def ^:dynamic *skip-dataset-prewarm?*
+ "Whether `with-temp` should skip materializing the test-data Database before opening its transaction.
+
+ Bind this in helpers whose app DB must remain empty. See [[metabase.test.data/with-empty-h2-app-db!]]."
+ false)
+
(defmulti get-or-create-database!
"Create data warehouse database associated with `database-definition`, create corresponding Metabase Databases/Tables/Fields,
and sync the Database. `driver` is a keyword name of a driver that implements test extension methods (as defined in
@@ -61,9 +67,24 @@
That is memoized for the current application database."
[]
- (mdb/memoize-for-application-db
- (fn [driver]
- (u/the-id (get-or-create-default-dataset! driver)))))
+ (let [cached (mdb/memoize-for-application-db
+ (fn [driver]
+ (u/the-id (get-or-create-default-dataset! driver))))]
+ (fn [driver]
+ ;; A cached ID can outlive the transaction that created its Database. Bypass the cache within a transaction so
+ ;; rollback cannot leave a stale ID. Do not create the Database on a dedicated connection: the caller may hold
+ ;; cluster-lock rows that would block that connection until timeout.
+ ;; TODO (Chris 2026-08-18) -- On a cache miss, concurrent transactions cannot see one another's uncommitted
+ ;; Database and may each create and sync a duplicate because `(name, engine)` is not unique. Quartz triggers
+ ;; from the after-insert hook can also outlive a rollback. A dedicated connection can deadlock on cluster
+ ;; locks held by the caller, while coordination that lasts until commit would be complex for a test-only path.
+ ;; Materializing the dataset before opening the transaction avoids both problems.
+ ;;
+ ;; Inside a transaction, `mt/id` therefore costs a query, so a `t2/with-call-count` window expecting zero
+ ;; calls must resolve its ids before opening.
+ (if (mdb/in-transaction?)
+ (u/the-id (get-or-create-default-dataset! driver))
+ (cached driver)))))
(def ^:private memoized-test-data-database-id-fn
"Atom with a function with the signature
@@ -140,12 +161,24 @@
:table_id table-id
:active true))
-(def ^:private ^{:arglists '([database-id])} table-lookup-map
+;; Like the Database ID above, these maps are memoized for the application DB. Bypass the caches within a transaction
+;; so a rollback cannot leave IDs for Tables and Fields that no longer exist.
+(def ^:private ^{:arglists '([database-id])} cached-table-lookup-map
(mdb/memoize-for-application-db build-table-lookup-map))
-(def ^:private ^{:arglists '([field-lookup-map])} field-lookup-map
+(defn- table-lookup-map [database-id]
+ (if (mdb/in-transaction?)
+ (build-table-lookup-map database-id)
+ (cached-table-lookup-map database-id)))
+
+(def ^:private ^{:arglists '([table-id])} cached-field-lookup-map
(mdb/memoize-for-application-db build-field-lookup-map))
+(defn- field-lookup-map [table-id]
+ (if (mdb/in-transaction?)
+ (build-field-lookup-map table-id)
+ (cached-field-lookup-map table-id)))
+
(defn- cached-table-id [db-id table-name]
(get (table-lookup-map db-id) [db-id table-name]))
@@ -393,12 +426,14 @@
"Impl for [[metabase.test/dataset]] macro."
[dataset-definition f]
(let [dbdef (tx/get-dataset-definition dataset-definition)
- get-db-for-driver (mdb/memoize-for-application-db
- (fn [driver]
- (let [db (get-or-create-database! driver dbdef)]
- (assert db)
- (assert (pos-int? (:id db)))
- db)))
+ get-db! (fn [driver]
+ (let [db (get-or-create-database! driver dbdef)]
+ (assert db)
+ (assert (pos-int? (:id db)))
+ db))
+ cached (mdb/memoize-for-application-db get-db!)
+ ;; Bypass the cache within a transaction; see [[make-memoized-test-database-id-fn]].
+ get-db-for-driver #(if (mdb/in-transaction?) (get-db! %) (cached %))
db-fn #(get-db-for-driver (tx/driver))]
(binding [*db-fn* db-fn
*db-id-fn* #(u/the-id (db-fn))
diff --git a/test/metabase/test/data/users.clj b/test/metabase/test/data/users.clj
index f89906c5aa3..4894b1751b0 100644
--- a/test/metabase/test/data/users.clj
+++ b/test/metabase/test/data/users.clj
@@ -5,6 +5,7 @@
[medley.core :as m]
[metabase.app-db.core :as mdb]
[metabase.auth-identity.core :as auth-identity]
+ [metabase.models.interface :as mi]
[metabase.request.core :as request]
[metabase.session.core :as session]
[metabase.test.http-client :as client]
@@ -92,9 +93,9 @@
;;; If we run INSIDE of a transaction before the test users have been created globally then do not memoize the IDs,
;;; since the users will get discarded and we will need to recreate them next time around.
;;;
-;;; "Globally" is per application database: [[metabase.test.data/with-empty-h2-app-db!]] swaps in an empty
-;;; one, where the users another database created do not exist. Tracking it as a boolean meant the first
-;;; branch handed back ids for absent users, and the session insert failed on FK_SESSION_REF_USER_ID.
+;;; "Globally" means within one application DB. [[metabase.test.data/with-empty-h2-app-db!]] swaps in an empty
+;;; application DB where users created in another DB do not exist. Tracking this state with one boolean returned
+;;; stale IDs and caused session inserts to violate FK_SESSION_REF_USER_ID.
(let [f (fn []
(zipmap usernames
(map (comp u/the-id fetch-user) usernames)))
@@ -145,11 +146,11 @@
{:username email
:password password}))
-;; {[app-db username] session-key}
+;; {[app-db-id username] session-key}
;;
-;; The app db is in the key because `with-empty-h2-app-db!` swaps in a different one. A token from the old
-;; database names a session the new one has never heard of, so the request 401s. Tokens minted while the empty
-;; db is in place would stick around after the original comes back, too.
+;; Include the application DB in the key because [[metabase.test.data/with-empty-h2-app-db!]] swaps databases. A
+;; token from one database names a session the other has never heard of, so the request returns 401. Tokens created
+;; in the replacement database must not survive its restoration.
(defonce ^:private tokens (atom {}))
;;; This is done by hitting the app DB directly instead of hitting [[metabase.test.http-client/authenticate]] to avoid
@@ -185,9 +186,13 @@
(throw (Exception. (format "Authentication failed for %s with credentials %s"
username (user->credentials username)))))))
+;; Standard-user sessions are normally created before a transaction opens. [[metabase.test.redefs]] now materializes
+;; `:test-users` before each top-level `with-temp`, so initialization no longer first happens inside one -- except for
+;; the helpers that skip the prewarm to keep an empty app DB. There, sessions created inside `with-temp` are rolled
+;; back and subsequent requests authenticate again.
(defn clear-cached-session-tokens!
- "Clear any cached session tokens, which may have expired or been removed. You should do this in the even you get a
- `401` unauthenticated response, and then retry the request."
+ "Clear cached session tokens that may have expired or been removed. Call this after receiving a `401` response, then
+ retry the request."
[]
(locking tokens
(reset! tokens {})))
@@ -208,8 +213,9 @@
(thunk)
(catch ExceptionInfo e
(rethrow-when-not-401 e)
- ;; second retry: clear cached session tokens, then try again one last time
- (clear-cached-session-tokens!)
+ ;; For the final retry, evict only this user's token. A session created in a transaction can be
+ ;; rolled back with it; one resulting 401 should not evict every other user's durable token.
+ (swap! tokens dissoc [(mdb/unique-identifier) username])
(try
(thunk)
(catch ExceptionInfo e
@@ -225,13 +231,25 @@
(fetch-user user)
(apply client-fn the-client user args))
(let [user-id (u/the-id user)
- session-key (session/generate-session-key)]
+ session-key (session/generate-session-key)
+ session-id (session/generate-session-id)]
(when-not (t2/exists? :model/User :id user-id)
(throw (ex-info "User does not exist" {:user user})))
- (t2.with-temp/with-temp [:model/Session _ {:id (session/generate-session-id)
- :key_hashed (session/hash-session-key session-key)
- :user_id user-id}]
- (apply the-client session-key args)))))
+ ;; Do not use `with-temp` here. The request handler runs in-process on this thread, so a rollback-only
+ ;; transaction around the Session would also discard every app DB write made by the request. Delete only
+ ;; the Session explicitly instead.
+ ;;
+ ;; Write the row rather than the model. `:model/Session`'s after-insert hook publishes `:event/user-login`,
+ ;; which stamps `User.last_login`, and `:event/user-joined` for a User that has never logged in. Deleting the
+ ;; Session does not undo either, so they leak into whatever runs next.
+ (t2/insert! :core_session {:id session-id
+ :key_hashed (session/hash-session-key session-key)
+ :user_id user-id
+ :created_at (mi/now)})
+ (try
+ (apply the-client session-key args)
+ (finally
+ (t2/delete! :core_session :id session-id))))))
(def ^{:arglists '([test-user-name-or-user-or-id method expected-status-code? endpoint
request-options? http-body-map? & {:as query-params}])} user-http-request
diff --git a/test/metabase/test/data/users_test.clj b/test/metabase/test/data/users_test.clj
new file mode 100644
index 00000000000..389090343f0
--- /dev/null
+++ b/test/metabase/test/data/users_test.clj
@@ -0,0 +1,14 @@
+(ns metabase.test.data.users-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
+
+(deftest user-http-request-does-not-log-the-user-in-test
+ (testing "the session a request borrows is not a login"
+ (mt/with-temp [:model/User {user-id :id} {:last_login nil}]
+ (mt/user-http-request {:id user-id} :get 200 "user/current")
+ (is (nil? (t2/select-one-fn :last_login :model/User :id user-id))
+ "last_login stays empty, so :event/user-login never fired")
+ (is (false? (t2/exists? :core_session :user_id user-id))
+ "and the borrowed session is deleted"))))
diff --git a/test/metabase/test/initialize/test_users.clj b/test/metabase/test/initialize/test_users.clj
index 25a8108ad7c..36bafca3c84 100644
--- a/test/metabase/test/initialize/test_users.clj
+++ b/test/metabase/test/initialize/test_users.clj
@@ -7,4 +7,8 @@
[]
(doseq [username test.users/usernames]
;; fetch-user will force creation of users
- (test.users/fetch-user username)))
+ (test.users/fetch-user username)
+ ;; Create the session here, before a transaction opens, so it remains durable. A session created lazily
+ ;; inside `with-temp` would be rolled back, causing each later request to receive a 401 and create another
+ ;; session, with each attempt also updating the User's `last_login`.
+ (test.users/username->token username)))
diff --git a/test/metabase/test/redefs.clj b/test/metabase/test/redefs.clj
index b961434f9c5..3362bc378c6 100644
--- a/test/metabase/test/redefs.clj
+++ b/test/metabase/test/redefs.clj
@@ -5,6 +5,7 @@
[mb.hawk.parallel]
[metabase.classloader.core :as classloader]
[metabase.test.util.thread-local :as tu.thread-local]
+ [metabase.util.log :as log]
[methodical.core :as methodical]
[toucan2.connection :as t2.connection]
[toucan2.tools.with-temp]))
@@ -13,6 +14,10 @@
"Used to detect whether we're in a nested [[with-temp]]. Default is false."
false)
+(defonce ^:private prewarm-failures-logged
+ ;; Failure messages already reported by the dataset prewarm below, so repeated failures are logged once.
+ (atom #{}))
+
(methodical/defmethod toucan2.tools.with-temp/do-with-temp* :around :default
"Initialize the DB before doing the other with-temp stuff.
Make sure metabase.test.util is loaded.
@@ -24,9 +29,37 @@
(classloader/require 'metabase.test.util)
;; run `f` in a transaction if it's the top-level with-temp
(if (and tu.thread-local/*thread-local* (not *in-with-temp*))
- (binding [*in-with-temp* true]
- (t2.connection/with-transaction [_ t2.connection/*current-connectable* {:rollback-only true}]
- (next-method model attributes f)))
+ (do
+ ;; Materialize the test-data Database before opening the transaction. If created inside it, the Database
+ ;; and its Tables are rolled back while memoized IDs in [[metabase.test.data.impl]] survive and refer to
+ ;; missing rows. The dataset would also be rebuilt for every test because its creation could never commit.
+ ;; Skip this for helpers whose app DB must remain empty. Otherwise this is best-effort: failure to build a
+ ;; dataset must not fail an unrelated `with-temp`.
+ (classloader/require 'metabase.test.data.impl)
+ (when-not @(resolve 'metabase.test.data.impl/*skip-dataset-prewarm?*)
+ (try
+ ((resolve 'metabase.test.data.impl/db-id))
+ (catch Throwable e
+ ;; A failed prewarm resurfaces much later as a Database that does not exist. Log each message once, since a
+ ;; warehouse that is down would otherwise produce a warning for every top-level `with-temp`.
+ (let [message (ex-message e)]
+ (when-not (contains? (first (swap-vals! prewarm-failures-logged conj message)) message)
+ (log/warnf "Could not materialize the test dataset before with-temp: %s" message))))))
+ ;; Materialize the test users and their Personal Collections for the same reason as the dataset above.
+ ;; `user->personal-collection` is a get-or-create, and the permissions path calls it on every API request. One
+ ;; first created inside the transaction is rolled back, so every later request tries to create it again, and
+ ;; the parallel ones time out waiting on each other's COLLECTION table lock.
+ (when-not @(resolve 'metabase.test.data.impl/*skip-dataset-prewarm?*)
+ (try
+ ((resolve 'metabase.test.initialize/initialize-if-needed!) :test-users-personal-collections)
+ (catch Throwable e
+ (let [message (ex-message e)]
+ (when-not (contains? (first (swap-vals! prewarm-failures-logged conj message)) message)
+ (log/warnf "Could not materialize the test users' Personal Collections before with-temp: %s"
+ message))))))
+ (binding [*in-with-temp* true]
+ (t2.connection/with-transaction [_ t2.connection/*current-connectable* {:rollback-only true}]
+ (next-method model attributes f))))
(next-method model attributes f)))
;;; wrap `with-redefs-fn` (used by `with-redefs`) so it calls `assert-test-is-not-parallel`
diff --git a/test/metabase/test/util.clj b/test/metabase/test/util.clj
index ede415935e8..812be0cf703 100644
--- a/test/metabase/test/util.clj
+++ b/test/metabase/test/util.clj
@@ -1,6 +1,7 @@
(ns metabase.test.util
"Helper functions and macros for writing unit tests."
(:require
+ [clojure.core.memoize :as memoize]
[clojure.java.io :as io]
[clojure.set :as set]
[clojure.string :as str]
@@ -435,6 +436,39 @@
(binding [pgm/*allow-direct-deletion* true]
(next-method model explicit-attributes f)))
+;; A Personal Collection created within `with-temp` is rolled back, but the production cache assumes it cannot be
+;; deleted. Evicting only the temporary User's entry is insufficient: a committed User's collection may be created
+;; inside another User's `with-temp` scope. Clear the cache whenever a `with-temp` scope exits.
+;; Use a unique method key because [[metabase.test.redefs]] already defines `:around :default`. Methods with the same
+;; key overwrite one another, which would disable either this eviction or the rollback-only transaction wrapper.
+(methodical/add-aux-method-with-unique-key!
+ #'t2.with-temp/do-with-temp* :around :default
+ (fn [next-method model explicit-attributes f]
+ (next-method model explicit-attributes
+ (fn [temp-object]
+ (try
+ (f temp-object)
+ (finally
+ (memoize/memo-clear! @#'collection/user->personal-collection-id))))))
+ ::evict-personal-collection-cache)
+
+(def ^:private dimension-lock
+ "Serialises temporary Dimensions against each other.
+
+ A Dimension names a Field the whole suite shares, and it lives until its scope ends. Two threads creating one at
+ once deadlock on the unique index over `dimension.field_id`: each holds the record lock the other's uniqueness
+ check wants, while asking for the insert-intention gap in front of it. MySQL and MariaDB pick a victim and roll
+ it back, taking its savepoints with it.
+
+ Tests that create no Dimension keep running in parallel alongside these."
+ (Object.))
+
+(methodical/defmethod t2.with-temp/do-with-temp* :around :model/Dimension
+ [model explicit-attributes f]
+ ;; `locking` is reentrant, so nesting Dimensions in one `with-temp` is fine.
+ (locking dimension-lock
+ (next-method model explicit-attributes f)))
+
(defn- set-with-temp-defaults! []
(doseq [[model defaults-fn] with-temp-defaults-fns]
(methodical/defmethod t2.with-temp/with-temp-defaults model
@@ -1306,8 +1340,6 @@
[locale-tag & body]
`(call-with-locale! ~locale-tag (fn [] ~@body)))
-;;; TODO -- this could be made thread-safe if we made [[with-temp-vals-in-db]] thread-safe which I think is pretty
-;;; doable (just do it in a transaction?)
(defn do-with-column-remappings! [orig->remapped thunk]
(transduce
identity
diff --git a/test/metabase/transforms_rest/api/transform_test.clj b/test/metabase/transforms_rest/api/transform_test.clj
index dd5901f8d0e..bb49240bfec 100644
--- a/test/metabase/transforms_rest/api/transform_test.clj
+++ b/test/metabase/transforms_rest/api/transform_test.clj
@@ -1974,74 +1974,85 @@
(mt/with-premium-features #{:transforms-basic :hosting}
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/dataset transforms-dataset/transforms-test
- (let [search-term (str "transform-search-" (u/generate-nano-id))
- query-name (str search-term "-query")
- python-name (str search-term "-python")
- query-source {:type "query"
- :query (lib/native-query (mt/metadata-provider) "SELECT 1")}
- python-source {:type "python"
- :body "print('hello world')"
- :source-tables []
- :source-database (mt/id)}]
- (mt/with-temp [:model/Transform {query-id :id} {:name query-name
- :source query-source
- :target {:type "table"
- :schema (get-test-schema)
- :name (str "target_" (u/generate-nano-id))}}
- :model/Transform {python-id :id} {:name python-name
- :source python-source
- :target {:type "table"
- :schema (get-test-schema)
- :name (str "target_" (u/generate-nano-id))
- :database (mt/id)}}]
- (search.tu/with-appdb-search-and-legacy-search
- (let [transform-ids (search-transform-ids search-term)]
- (is (contains? transform-ids query-id))
- (is (not (contains? transform-ids python-id)))))))))))
+ ;; the temp index table is created here, before `with-temp` opens its transaction: creating (and
+ ;; especially dropping) it inside would run DDL on the ambient connection, which on H2/MySQL
+ ;; implicitly commits the transaction, so its rollback could not take the rows back and the
+ ;; Transforms would leak to every later test that counts them. The index scope nested inside
+ ;; `with-appdb-search-and-legacy-search` reuses this one rather than creating its own. The
+ ;; `-if-supported` variant keeps the legacy-search leg running on app dbs that cannot hold an index.
+ (search.tu/with-temp-index-table-if-supported
+ (let [search-term (str "transform-search-" (u/generate-nano-id))
+ query-name (str search-term "-query")
+ python-name (str search-term "-python")
+ query-source {:type "query"
+ :query (lib/native-query (mt/metadata-provider) "SELECT 1")}
+ python-source {:type "python"
+ :body "print('hello world')"
+ :source-tables []
+ :source-database (mt/id)}]
+ (mt/with-temp [:model/Transform {query-id :id} {:name query-name
+ :source query-source
+ :target {:type "table"
+ :schema (get-test-schema)
+ :name (str "target_" (u/generate-nano-id))}}
+ :model/Transform {python-id :id} {:name python-name
+ :source python-source
+ :target {:type "table"
+ :schema (get-test-schema)
+ :name (str "target_" (u/generate-nano-id))
+ :database (mt/id)}}]
+ (search.tu/with-appdb-search-and-legacy-search
+ (let [transform-ids (search-transform-ids search-term)]
+ (is (contains? transform-ids query-id))
+ (is (not (contains? transform-ids python-id))))))))))))
(deftest search-hides-transforms-for-non-superusers-test
(mt/with-premium-features #{:transforms-basic :hosting}
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/dataset transforms-dataset/transforms-test
- (let [search-term (str "transform-search-" (u/generate-nano-id))
- query-name (str search-term "-query")
- query-source {:type "query"
- :query (lib/native-query (mt/metadata-provider) "SELECT 1")}]
- (mt/with-temp [:model/Transform {transform-id :id} {:name query-name
- :source query-source
- :target {:type "table"
- :schema (get-test-schema)
- :name (str "target_" (u/generate-nano-id))}}]
- (search.tu/with-appdb-search-and-legacy-search
- (let [results (mt/user-http-request :rasta :get 200 "search" :q search-term :models "transform")
- ids (set (map :id (:data results)))]
- (is (empty? ids))
- (is (not (contains? ids transform-id)))))))))))
+ ;; see search-filters-transform-source-types-test for why the index scope sits outside `with-temp`
+ (search.tu/with-temp-index-table-if-supported
+ (let [search-term (str "transform-search-" (u/generate-nano-id))
+ query-name (str search-term "-query")
+ query-source {:type "query"
+ :query (lib/native-query (mt/metadata-provider) "SELECT 1")}]
+ (mt/with-temp [:model/Transform {transform-id :id} {:name query-name
+ :source query-source
+ :target {:type "table"
+ :schema (get-test-schema)
+ :name (str "target_" (u/generate-nano-id))}}]
+ (search.tu/with-appdb-search-and-legacy-search
+ (let [results (mt/user-http-request :rasta :get 200 "search" :q search-term :models "transform")
+ ids (set (map :id (:data results)))]
+ (is (empty? ids))
+ (is (not (contains? ids transform-id))))))))))))
(deftest search-includes-native-and-mbql-query-transforms-test
(mt/with-premium-features #{:transforms-basic :hosting}
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/dataset transforms-dataset/transforms-test
- (let [search-term (str "transform-search-" (u/generate-nano-id))
- native-name (str search-term "-native")
- mbql-name (str search-term "-mbql")
- native-source {:type "query"
- :query (lib/native-query (mt/metadata-provider) "SELECT 1")}
- mbql-source {:type "query"
- :query (mt/mbql-query transforms_products)}]
- (mt/with-temp [:model/Transform {native-id :id} {:name native-name
- :source native-source
+ ;; see search-filters-transform-source-types-test for why the index scope sits outside `with-temp`
+ (search.tu/with-temp-index-table-if-supported
+ (let [search-term (str "transform-search-" (u/generate-nano-id))
+ native-name (str search-term "-native")
+ mbql-name (str search-term "-mbql")
+ native-source {:type "query"
+ :query (lib/native-query (mt/metadata-provider) "SELECT 1")}
+ mbql-source {:type "query"
+ :query (mt/mbql-query transforms_products)}]
+ (mt/with-temp [:model/Transform {native-id :id} {:name native-name
+ :source native-source
+ :target {:type "table"
+ :schema (get-test-schema)
+ :name (str "target_" (u/generate-nano-id))}}
+ :model/Transform {mbql-id :id} {:name mbql-name
+ :source mbql-source
:target {:type "table"
:schema (get-test-schema)
- :name (str "target_" (u/generate-nano-id))}}
- :model/Transform {mbql-id :id} {:name mbql-name
- :source mbql-source
- :target {:type "table"
- :schema (get-test-schema)
- :name (str "target_" (u/generate-nano-id))}}]
- (search.tu/with-appdb-search-and-legacy-search
- (is (= #{native-id mbql-id}
- (search-transform-ids search-term))))))))))
+ :name (str "target_" (u/generate-nano-id))}}]
+ (search.tu/with-appdb-search-and-legacy-search
+ (is (= #{native-id mbql-id}
+ (search-transform-ids search-term)))))))))))
(deftest get-runs-sort-by-duration-test
(testing "GET /api/transform/run supports sort-column=duration"
diff --git a/test/metabase/typed_schemas/core_test.clj b/test/metabase/typed_schemas/core_test.clj
index 69861e4825f..b7484312a78 100644
--- a/test/metabase/typed_schemas/core_test.clj
+++ b/test/metabase/typed_schemas/core_test.clj
@@ -110,45 +110,48 @@
;; metric cards need no result_metadata — columns are computed from metadata.
(deftest full-pipeline-end-to-end-test
(mt/dataset test-data
- (mt/with-actions-enabled
- (let [mp (mt/metadata-provider)
- orders-query (lib/query mp (lib.metadata/table mp (mt/id :orders)))
- revenue-query (lib/aggregate orders-query
- (lib/sum (lib.metadata/field mp (mt/id :orders :total))))]
- (mt/with-temp [:model/Card _question {:name "Order totals", :database_id (mt/id), :table_id (mt/id :orders)
- :type :question, :display :table
- :dataset_query orders-query}
- :model/Card _metric {:name "Order revenue", :database_id (mt/id), :table_id (mt/id :orders)
- :type :metric, :display :scalar
- :dataset_query revenue-query}
- :model/Card model {:name "Order model", :database_id (mt/id), :table_id (mt/id :orders)
- :type :model
- :dataset_query orders-query
- :result_metadata [{:name "total", :display_name "Total"
- :base_type :type/Float
- :field_ref [:field (mt/id :orders :total) nil]
- :id (mt/id :orders :total)}]}
- :model/Action action {:name "Update order", :model_id (:id model), :type :implicit}
- :model/ImplicitAction _ {:action_id (:id action), :kind "row/update"}]
- (mt/with-current-user (mt/user->id :crowberto)
- (let [schema (typed-schemas/build-semantic-schema {:database {:id (mt/id)}} test-info)
- body (typed-schemas/render-typescript schema)]
- (testing "every entity kind lands in the schema with its real relationships"
- (is (=? {:generatedAt "2026-01-01T00:00:00Z"
- :metabase {:instanceUrl "https://metabase.example.com"}
- :questions {"orderTotals" {:type "card", :id int?}}
- :tables {"orders" {:fields {"total" {:jsType "number"}}}}
- :metrics {"orderRevenue" {:mappedTableIds [(mt/id :orders)]
- :columns [{:displayName "Sum of Total"
- :jsType "number"}]}}
- :models {"orderModel" {:actions {"updateOrder" {:kind "action"}}}}}
- schema)))
- (testing "only the temp cards are in scope for the dataset database"
- (is (= {:questions ["orderTotals"], :metrics ["orderRevenue"], :models ["orderModel"]}
- (update-vals (select-keys schema [:questions :metrics :models])
- (comp vec keys)))))
- (testing "the rendered module carries the real entities"
- (is (str/includes? body "orders: {"))
- (is (str/includes? body "name: \"Order revenue\""))
- (is (str/includes? body "updateOrder: {"))
- (is (str/ends-with? body "export default schema;\n"))))))))))
+ ;; Use a copy of the Database so Cards committed concurrently by other namespaces cannot affect the
+ ;; database-scoped assertion below.
+ (mt/with-temp-copy-of-db
+ (mt/with-actions-enabled
+ (let [mp (mt/metadata-provider)
+ orders-query (lib/query mp (lib.metadata/table mp (mt/id :orders)))
+ revenue-query (lib/aggregate orders-query
+ (lib/sum (lib.metadata/field mp (mt/id :orders :total))))]
+ (mt/with-temp [:model/Card _question {:name "Order totals", :database_id (mt/id), :table_id (mt/id :orders)
+ :type :question, :display :table
+ :dataset_query orders-query}
+ :model/Card _metric {:name "Order revenue", :database_id (mt/id), :table_id (mt/id :orders)
+ :type :metric, :display :scalar
+ :dataset_query revenue-query}
+ :model/Card model {:name "Order model", :database_id (mt/id), :table_id (mt/id :orders)
+ :type :model
+ :dataset_query orders-query
+ :result_metadata [{:name "total", :display_name "Total"
+ :base_type :type/Float
+ :field_ref [:field (mt/id :orders :total) nil]
+ :id (mt/id :orders :total)}]}
+ :model/Action action {:name "Update order", :model_id (:id model), :type :implicit}
+ :model/ImplicitAction _ {:action_id (:id action), :kind "row/update"}]
+ (mt/with-current-user (mt/user->id :crowberto)
+ (let [schema (typed-schemas/build-semantic-schema {:database {:id (mt/id)}} test-info)
+ body (typed-schemas/render-typescript schema)]
+ (testing "every entity kind lands in the schema with its real relationships"
+ (is (=? {:generatedAt "2026-01-01T00:00:00Z"
+ :metabase {:instanceUrl "https://metabase.example.com"}
+ :questions {"orderTotals" {:type "card", :id int?}}
+ :tables {"orders" {:fields {"total" {:jsType "number"}}}}
+ :metrics {"orderRevenue" {:mappedTableIds [(mt/id :orders)]
+ :columns [{:displayName "Sum of Total"
+ :jsType "number"}]}}
+ :models {"orderModel" {:actions {"updateOrder" {:kind "action"}}}}}
+ schema)))
+ (testing "only the temp cards are in scope for the dataset database"
+ (is (= {:questions ["orderTotals"], :metrics ["orderRevenue"], :models ["orderModel"]}
+ (update-vals (select-keys schema [:questions :metrics :models])
+ (comp vec keys)))))
+ (testing "the rendered module carries the real entities"
+ (is (str/includes? body "orders: {"))
+ (is (str/includes? body "name: \"Order revenue\""))
+ (is (str/includes? body "updateOrder: {"))
+ (is (str/ends-with? body "export default schema;\n")))))))))))
diff --git a/test/metabase/xrays/related_test.clj b/test/metabase/xrays/related_test.clj
index 8da1127aa5d..868da09349b 100644
--- a/test/metabase/xrays/related_test.clj
+++ b/test/metabase/xrays/related_test.clj
@@ -11,9 +11,14 @@
[metabase.sync.core :as sync]
[metabase.test :as mt]
[metabase.test.data.one-off-dbs :as one-off-dbs]
+ [metabase.test.fixtures :as fixtures]
[metabase.xrays.related :as related]
[toucan2.core :as t2]))
+;; Initialize test users before these tests run in parallel. Each creates a `with-temp` Card whose creator defaults
+;; to Rasta; creating that User within the transaction would roll it back and make every test race to reinsert it.
+(use-fixtures :once (fixtures/initialize :test-users))
+
(deftest ^:parallel collect-context-bearing-forms-test
(is (= #{[:field 1 nil] [:metric 1] [:field 2 nil] [:segment 1]}
(#'related/collect-context-bearing-forms [[:> [:field 1 nil] 3]
@@ -50,6 +55,23 @@
(is (= expected-similarity
(double (#'related/similarity (t2/select-one :model/Card :id (get cards card-x)) (t2/select-one :model/Card :id (get cards card-y)))))))))))
+(deftest ^:parallel similarity-tolerates-an-unparseable-query-test
+ (testing "a Card whose stored query will not parse has no context-bearing forms, and does not stop the ranking"
+ (mt/with-temp [:model/Card broken {:name "broken", :dataset_query {}}
+ :model/Card {:as reference} {:name "reference"
+ :dataset_query (mt/mbql-query venues
+ {:aggregation [[:sum $price]]
+ :breakout [$category_id]})}
+ :model/Card {:as similar} {:name "similar"
+ :dataset_query (mt/mbql-query venues
+ {:aggregation [[:sum $longitude]]
+ :breakout [$category_id]})}]
+ (is (nil? (#'related/definition broken)))
+ (is (zero? (double (#'related/similarity reference broken))))
+ (testing "the similar Card still ranks above the broken one"
+ (is (= ["similar" "broken"]
+ (map :name (#'related/rank-by-similarity reference [broken similar]))))))))
+
(def ^:private ^:dynamic *world* {})
(defn- do-with-world [f]
diff --git a/test_config/log4j2-test.xml b/test_config/log4j2-test.xml
index 731f9bb7367..8698f4a62cf 100644
--- a/test_config/log4j2-test.xml
+++ b/test_config/log4j2-test.xml
@@ -7,6 +7,14 @@
<ThresholdFilter level="FATAL" onMatch="ACCEPT" onMismatch="DENY"/>
</Filters>
</Console>
+ <!-- The shared Console only accepts FATAL, which is right for a suite this chatty. Warnings that mean a
+ test silently broke its own isolation need to be seen, so they get an appender that accepts them. -->
+ <Console name="WarnConsole" target="SYSTEM_OUT" follow="true">
+ <PatternLayout pattern="[%t] %-5level %logger{36} - %msg %notEmpty{%X}%n"/>
+ <Filters>
+ <ThresholdFilter level="WARN" onMatch="ACCEPT" onMismatch="DENY"/>
+ </Filters>
+ </Console>
<File name="File" fileName="logs/test-log.json" append="false">
<JsonLayout locationInfo="false" properties="true" compact="true" eventEol="true"/>
<Filters>
@@ -25,6 +33,13 @@
resources/log4j2.xml, minus the per-session RoutingAppender (tests don't read the files). -->
<Logger name="metabase.ai-tracing.log" level="info" additivity="false"/>
+ <!-- A rollback-only scope that could not roll back leaves its rows behind for every later test, and the
+ tests that trip over them fail somewhere else entirely. Say so where the run can be read. -->
+ <Logger name="metabase.app-db.connection" level="INFO" additivity="false">
+ <AppenderRef ref="WarnConsole"/>
+ <AppenderRef ref="File"/>
+ </Logger>
+
<Root level="INFO">
<AppenderRef ref="Console"/>
<AppenderRef ref="File"/>