Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Keep H2 index cleanup out of caller transactions (#82810)

Chris Trutere1136464bd23
diff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.clj
index 5fe9ec7aa95..05bae517314 100644
--- a/src/metabase/search/appdb/index.clj
+++ b/src/metabase/search/appdb/index.clj
@@ -119,24 +119,50 @@
(map (comp keyword u/lower-case-en :table_name)
(search.db/orphan-index-table-names)))
+(defn- drop-orphan-index!
+ "Drop the orphan index `table`, returning it, or nil if the drop failed.
+ A failed drop is logged and never propagates."
+ [table]
+ (try
+ (search.db/drop-search-index-table! table)
+ table
+ ;; Deletion could fail if it races with other instances
+ (catch Exception e
+ (log/warnf "Failed to drop orphan index %s: %s" table (ex-message e))
+ nil)))
+
+(defn- drop-orphan-indexes!
+ "Drop every index table that has no metadata row, returning those dropped.
+ Best effort: a failed drop is logged and never propagates."
+ []
+ (let [dropped (into [] (keep drop-orphan-index!) (orphan-indexes))]
+ (log/infof "Dropped %d orphan indexes: %s" (count dropped) dropped)
+ dropped))
+
+(defn- ddl-commits-caller-transaction?
+ "Whether dropping a table here would commit the transaction the caller is holding."
+ []
+ ;; Dropping a table is DDL, which h2 and mysql commit implicitly.
+ ;; Postgres keeps DDL inside the transaction, where it does no harm.
+ ;; Mysql cannot reach this while the appdb engine supports postgres and h2 only, but listing it means
+ ;; adding support does not quietly bring the rollback failure back.
+ (and (mdb/in-transaction?) (contains? #{:h2 :mysql} (mdb/db-type))))
+
(defn delete-obsolete-tables!
- "Drop index tables that are no longer needed. Best effort: failures are logged and never propagate. Does nothing
- while mocking tables, where the pending table is tracked in an atom and has no metadata row to find it by."
+ "Prune obsolete index metadata, then drop the index tables that leaves unreferenced.
+ Best effort: failures are logged and never propagate.
+ Does nothing while mocking tables, where the pending table lives in an atom with no metadata row.
+ Skips the drops when they would commit the caller's transaction, leaving those tables to a later sweep."
[]
(when-not *mocking-tables*
(try
;; Delete metadata around indexes that are no longer needed.
(search-index-metadata/delete-obsolete! (search.spec/index-version-hash))
;; Drop any indexes that are no longer referenced.
- (let [dropped (volatile! [])]
- (doseq [table (orphan-indexes)]
- (try
- (search.db/drop-search-index-table! table)
- (vswap! dropped conj table)
- ;; Deletion could fail if it races with other instances
- (catch Exception e
- (log/warnf "Failed to drop stale index %s: %s" table (ex-message e)))))
- (log/infof "Dropped %d stale indexes: %s" (count @dropped) @dropped))
+ ;; A sweep outside the caller's transaction gets the rest.
+ (if (ddl-commits-caller-transaction?)
+ (log/debug "Deferring orphan index drops: dropping here would commit the caller's transaction")
+ (drop-orphan-indexes!))
(catch Exception e
(log/warnf "Failed to clean up obsolete indexes: %s" (ex-message e))))))
diff --git a/test/metabase/search/appdb/index_test.clj b/test/metabase/search/appdb/index_test.clj
index 0d4e6ae1fae..65798504d2e 100644
--- a/test/metabase/search/appdb/index_test.clj
+++ b/test/metabase/search/appdb/index_test.clj
@@ -3,6 +3,7 @@
(:require
[clojure.test :refer :all]
[java-time.api :as t]
+ [metabase.app-db.cluster-lock :as cluster-lock]
[metabase.app-db.core :as mdb]
[metabase.indexed-entities.models.model-index :as model-index]
[metabase.lib.core :as lib]
@@ -27,6 +28,17 @@
(use-fixtures :once (fixtures/initialize :db :test-users))
+(defmacro ^:private with-temp-index-version
+ "Run `body` under index version `version`, then drop its metadata rows and any tables they tracked."
+ [version & body]
+ `(let [version# ~version]
+ (binding [search.spec/*testing-only-index-version-hash* version#]
+ (try
+ ~@body
+ (finally
+ (t2/delete! :model/SearchIndexMetadata :version version#)
+ (search.index/delete-obsolete-tables!))))))
+
(defn- index-hits [term]
(count (search.index/search term)))
@@ -566,81 +578,99 @@
(deftest auto-refresh-test
(when (search/supports-index?)
- (binding [search.spec/*testing-only-index-version-hash* "auto-refresh-test"]
- (try
- (reset! @#'search.index/next-sync-at nil)
- (search.index/reset-index!)
- (let [active-before (search.index/active-table)
- active-after (search.index/gen-table-name)
- pending-after (search.index/gen-table-name)
- period @#'search.index/sync-tracking-period
- version (search.spec/index-version-hash)]
- (search-index-metadata/create-pending! :appdb version active-after)
- (search.index/create-table! active-after)
- (search-index-metadata/active-pending! :appdb version)
- (search-index-metadata/create-pending! :appdb version pending-after)
- (search.index/create-table! pending-after)
- (testing "We continue using our cached references for some time"
- (is (= active-before (active-table-after 100)))
- (is (= active-before (active-table-after (/ period 2)))))
- (testing "But eventually we refresh"
- (is (= active-after (active-table-after period)))))
- (finally
- (t2/delete! :model/SearchIndexMetadata :version "auto-refresh-test")
- (search.index/delete-obsolete-tables!))))))
+ (with-temp-index-version "auto-refresh-test"
+ (reset! @#'search.index/next-sync-at nil)
+ (search.index/reset-index!)
+ (let [active-before (search.index/active-table)
+ active-after (search.index/gen-table-name)
+ pending-after (search.index/gen-table-name)
+ period @#'search.index/sync-tracking-period
+ version (search.spec/index-version-hash)]
+ (search-index-metadata/create-pending! :appdb version active-after)
+ (search.index/create-table! active-after)
+ (search-index-metadata/active-pending! :appdb version)
+ (search-index-metadata/create-pending! :appdb version pending-after)
+ (search.index/create-table! pending-after)
+ (testing "We continue using our cached references for some time"
+ (is (= active-before (active-table-after 100)))
+ (is (= active-before (active-table-after (/ period 2)))))
+ (testing "But eventually we refresh"
+ (is (= active-after (active-table-after period))))))))
(deftest pending-table-expiry-test
(when (search/supports-index?)
- (binding [search.spec/*testing-only-index-version-hash* "pending-timeout-test"]
- (try
- (reset! @#'search.index/next-sync-at nil)
- (search.index/reset-index!)
- (let [active-table (search.index/active-table)
- pending-old (search.index/gen-table-name)
- pending-new (search.index/gen-table-name)
- version (search.spec/index-version-hash)]
- ;; Set up old pending table (more than a day old)
- (search.index/create-table! pending-old)
- (search-index-metadata/create-pending! :appdb version pending-old)
- (t2/update! :model/SearchIndexMetadata
- {:index_name (name pending-old)}
- {:created_at (t/minus (t/offset-date-time) (t/days 2))})
- (#'search.index/sync-tracking-atoms!)
- (testing "Active table is returned"
- (is (= active-table (search.index/active-table))))
- (testing "Old pending table is ignored (more than a day old)"
- (is (nil? (#'search.index/pending-table))))
- ;; Create new pending table (less than a day old)
- (search.index/create-table! pending-new)
- (search-index-metadata/create-pending! :appdb version pending-new)
- (#'search.index/sync-tracking-atoms!)
- (testing "New pending table is included (less than a day old)"
- (is (= active-table (search.index/active-table)))
- (is (= pending-new (#'search.index/pending-table)))))
- (finally
- (t2/delete! :model/SearchIndexMetadata :version "pending-timeout-test")
- (search.index/delete-obsolete-tables!))))))
+ (with-temp-index-version "pending-timeout-test"
+ (reset! @#'search.index/next-sync-at nil)
+ (search.index/reset-index!)
+ (let [active-table (search.index/active-table)
+ pending-old (search.index/gen-table-name)
+ pending-new (search.index/gen-table-name)
+ version (search.spec/index-version-hash)]
+ ;; Set up old pending table (more than a day old)
+ (search.index/create-table! pending-old)
+ (search-index-metadata/create-pending! :appdb version pending-old)
+ (t2/update! :model/SearchIndexMetadata
+ {:index_name (name pending-old)}
+ {:created_at (t/minus (t/offset-date-time) (t/days 2))})
+ (#'search.index/sync-tracking-atoms!)
+ (testing "Active table is returned"
+ (is (= active-table (search.index/active-table))))
+ (testing "Old pending table is ignored (more than a day old)"
+ (is (nil? (#'search.index/pending-table))))
+ ;; Create new pending table (less than a day old)
+ (search.index/create-table! pending-new)
+ (search-index-metadata/create-pending! :appdb version pending-new)
+ (#'search.index/sync-tracking-atoms!)
+ (testing "New pending table is included (less than a day old)"
+ (is (= active-table (search.index/active-table)))
+ (is (= pending-new (#'search.index/pending-table))))))))
(deftest failed-reindex-drops-orphaned-tables-test
(when (search/supports-index?)
- (binding [search.spec/*testing-only-index-version-hash* "orphan-cleanup-test"]
- (try
- (reset! @#'search.index/next-sync-at nil)
- (search.index/reset-index!)
- (let [orphan (search.index/gen-table-name)]
- (search.index/create-table! orphan)
- (mt/with-dynamic-fn-redefs [search.ingestion/searchable-documents #(throw (ex-info "Simulated connection loss" {}))]
- (mt/with-log-level [metabase.search.appdb.core :fatal]
- (is (thrown-with-msg? Exception #"Simulated connection loss"
- (search.engine/reindex! :search.engine/appdb {})))))
- (testing "the orphan is dropped even though the reindex never reached activation"
- (is (not (search.index/exists? orphan))))
- (testing "the active table and the pending table left behind by the failed run are kept"
- (is (search.index/exists? (search.index/active-table)))
- (is (search.index/exists? (#'search.index/pending-table)))))
- (finally
- (t2/delete! :model/SearchIndexMetadata :version "orphan-cleanup-test")
- (search.index/delete-obsolete-tables!))))))
+ (with-temp-index-version "orphan-cleanup-test"
+ (reset! @#'search.index/next-sync-at nil)
+ (search.index/reset-index!)
+ (let [orphan (search.index/gen-table-name)]
+ (search.index/create-table! orphan)
+ (mt/with-dynamic-fn-redefs [search.ingestion/searchable-documents #(throw (ex-info "Simulated connection loss" {}))]
+ (mt/with-log-level [metabase.search.appdb.core :fatal]
+ (is (thrown-with-msg? Exception #"Simulated connection loss"
+ (search.engine/reindex! :search.engine/appdb {})))))
+ (testing "the orphan is dropped even though the reindex never reached activation"
+ (is (not (search.index/exists? orphan))))
+ (testing "the active table and the pending table left behind by the failed run are kept"
+ (is (search.index/exists? (search.index/active-table)))
+ (is (search.index/exists? (#'search.index/pending-table))))))))
+
+(deftest sweep-leaves-an-enclosing-transaction-intact-test
+ (when (search/supports-index?)
+ (with-temp-index-version "sweep-in-transaction-test"
+ (search.index/reset-index!)
+ (let [orphan (search.index/gen-table-name)
+ before (t2/count :model/Collection)]
+ (search.index/create-table! orphan)
+ (testing "a rollback-only transaction survives a sweep inside it, which leaves the orphan behind"
+ (mt/with-temp [:model/Collection _ {}]
+ (mt/with-temp [:model/Collection _ {}]
+ (search.index/delete-obsolete-tables!)))
+ (is (= before (t2/count :model/Collection)))
+ (is (search.index/exists? orphan)))
+ (testing "a later sweep, outside any transaction, drops it"
+ (search.index/delete-obsolete-tables!)
+ (is (not (search.index/exists? orphan))))))))
+
+(deftest sweep-under-the-cluster-lock-still-drops-orphans-test
+ (when (search/supports-index?)
+ (with-temp-index-version "sweep-under-lock-test"
+ (search.index/reset-index!)
+ (let [orphan (search.index/gen-table-name)]
+ (search.index/create-table! orphan)
+ ;; Deferring the drops inside a transaction is only safe because this lock opens none on h2.
+ ;; Pin that here, so making it transactional fails a test instead of silently stopping the sweep.
+ (testing "the lock leaves the sweep able to drop an orphan"
+ (cluster-lock/with-cluster-lock ::sweep-test-lock
+ (search.index/delete-obsolete-tables!))
+ (is (not (search.index/exists? orphan))))))))
(deftest strip-junk-chars-test
(let [strip @#'search.index/strip-junk-chars]
@@ -792,24 +822,20 @@
(deftest when-index-created
(when (search/supports-index?)
- (binding [search.spec/*testing-only-index-version-hash* "index-age-test"]
- (try
- (let [table-name (search.index/gen-table-name)
- version (search.spec/index-version-hash)]
- (testing "Nil age if no active table"
- (is (nil? (#'search.index/when-index-created))))
- (testing "Returns age of active table"
- (let [update-time (t/truncate-to (t/minus (t/offset-date-time) (t/days 2)) :millis)]
- (search.index/create-table! table-name)
- (search-index-metadata/create-pending! :appdb version table-name)
- (search-index-metadata/active-pending! :appdb version)
- (t2/update! :model/SearchIndexMetadata
- :index_name (name table-name)
- {:created_at update-time})
- (is (= update-time (t/truncate-to (#'search.index/when-index-created) :millis))))))
- (finally
- (t2/delete! :model/SearchIndexMetadata :version "index-age-test")
- (search.index/delete-obsolete-tables!))))))
+ (with-temp-index-version "index-age-test"
+ (let [table-name (search.index/gen-table-name)
+ version (search.spec/index-version-hash)]
+ (testing "Nil age if no active table"
+ (is (nil? (#'search.index/when-index-created))))
+ (testing "Returns age of active table"
+ (let [update-time (t/truncate-to (t/minus (t/offset-date-time) (t/days 2)) :millis)]
+ (search.index/create-table! table-name)
+ (search-index-metadata/create-pending! :appdb version table-name)
+ (search-index-metadata/active-pending! :appdb version)
+ (t2/update! :model/SearchIndexMetadata
+ :index_name (name table-name)
+ {:created_at update-time})
+ (is (= update-time (t/truncate-to (#'search.index/when-index-created) :millis)))))))))
(deftest missing-index-table-does-not-abort-enclosing-transaction-test
(when (search/supports-index?)
diff --git a/test/metabase/server/middleware/data_app_scope_test.clj b/test/metabase/server/middleware/data_app_scope_test.clj
index ea99366b06d..3c1792c68df 100644
--- a/test/metabase/server/middleware/data_app_scope_test.clj
+++ b/test/metabase/server/middleware/data_app_scope_test.clj
@@ -112,16 +112,11 @@
;; The SDK's "Detail" visualization renders `ObjectDetailPanel`, which lists a writable model's
;; actions, and its `ActionExecuteModal` fetches the one clicked. Both are reads, and both must
;; carry the tag or the menu silently disappears for an admin inside a data app.
- ;; Built with plain `with-temp` rather than `mt/with-actions`: that helper's cleanup reindexes search, which
- ;; drops orphan index tables, and that DDL commits the rollback-only `with-temp` transaction on H2 whenever the
- ;; `metabase.search.*` namespaces ran earlier in the same JVM.
(mt/with-actions-test-data-and-actions-enabled
- (mt/with-temp [:model/Card {model-id :id} {:type :model
- :dataset_query (lib/query (mt/metadata-provider)
- (lib.metadata/table (mt/metadata-provider)
- (mt/id :categories)))}
- :model/Action {action-id :id} {:name "Update category" :model_id model-id :type :implicit}
- :model/ImplicitAction _ {:action_id action-id :kind "row/update"}]
+ (mt/with-actions [_ {:type :model
+ :dataset_query (lib/query (mt/metadata-provider)
+ (lib.metadata/table (mt/metadata-provider) (mt/id :categories)))}
+ {:keys [action-id model-id]} {:type :implicit :kind "row/update"}]
(let [as-data-app {:request-options {:headers {"x-metabase-client" "data-app"}}}]
(testing "GET /api/action?model-id= -- the object-detail action menu"
(is (=? [{:id action-id}]