Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix search index metadata deletion guard (#80309)

Chris Truter1f1643d5b8c6
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index ec0b984e419..964d6753907 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -349,11 +349,12 @@
app-db
{:team "UX West"
- :api #{metabase.app-db.cluster-lock ; TODO FIXME
+ :api #{metabase.app-db.cluster-lock ; TODO (Chris 2026-09-16) -- callers should go through app-db.core
metabase.app-db.core
metabase.app-db.init
metabase.app-db.setting
metabase.app-db.setup
+ metabase.app-db.sql-errors
metabase.app-db.transient-error}
:uses #{auth-provider
connection-pool
diff --git a/src/metabase/app_db/core.clj b/src/metabase/app_db/core.clj
index 80b17e93f3a..a6fb131a962 100644
--- a/src/metabase/app_db/core.clj
+++ b/src/metabase/app_db/core.clj
@@ -16,7 +16,6 @@
[metabase.app-db.encryption :as mdb.encryption]
[metabase.app-db.env :as mdb.env]
[metabase.app-db.format]
- [metabase.app-db.h2 :as mdb.h2]
[metabase.app-db.honeysql-guard]
[metabase.app-db.jdbc-protocols :as mdb.jdbc-protocols]
[metabase.app-db.liquibase :as liquibase]
@@ -55,8 +54,6 @@
broken-out-details->DataSource]
[mdb.env
db-file]
- [mdb.h2
- jdbc-sql-syntax-error-exception-classname]
[mdb.jdbc-protocols
clob->str]
[mdb.encryption
diff --git a/src/metabase/app_db/h2.clj b/src/metabase/app_db/h2.clj
index 339e9edb6ba..67c468c3b70 100644
--- a/src/metabase/app_db/h2.clj
+++ b/src/metabase/app_db/h2.clj
@@ -5,7 +5,3 @@
"Value of `org.h2.api.ErrorCode/STATEMENT_WAS_CANCELED`, inlined so callers can recognize an H2
query cancelation without the H2 library on the classpath."
57014)
-
-(def jdbc-sql-syntax-error-exception-classname
- "Class name of H2's syntax-error exception, matched by name so callers need not import the class."
- "org.h2.jdbc.JdbcSQLSyntaxErrorException")
diff --git a/src/metabase/app_db/sql_errors.clj b/src/metabase/app_db/sql_errors.clj
new file mode 100644
index 00000000000..649694213ca
--- /dev/null
+++ b/src/metabase/app_db/sql_errors.clj
@@ -0,0 +1,82 @@
+(ns metabase.app-db.sql-errors
+ "Classification of SQL errors raised by the supported application databases.
+ [[error-kind]] names the class of error an exception represents; [[table-not-found?]] is its common special case."
+ (:import
+ (java.sql SQLException)))
+
+(set! *warn-on-reflection* true)
+
+;; The registries name every code we recognize so the classifier reads by intent rather than by bare string.
+;; Keep them cumulative: append newly recognized codes rather than removing entries that temporarily have no caller.
+;; They are the shared app-db SQL vocabulary, not an inventory of what the current change happens to use.
+;; Keep the missing-table states aligned with `impl-table-known-to-not-exist?` in the H2, Postgres, and MySQL
+;; drivers: this module cannot depend on the driver module, so the lists are maintained in both places.
+(def sql-states
+ "SQLSTATE codes returned by supported application databases."
+ ;; `undefined_table` (42P01) is PostgreSQL-specific. 42S02 is the ODBC/X-Open state for a missing table; H2 adds
+ ;; two of its own, mapping `TABLE_OR_VIEW_NOT_FOUND_1`, `..._WITH_CANDIDATES_2`, and `..._DATABASE_EMPTY_1` in
+ ;; `org.h2.api.ErrorCode` to 42S02, 42S03, and 42S04. 23000 is the standard integrity-constraint state; 23505 is
+ ;; the PostgreSQL and H2 unique-violation state.
+ {:undefined-table "42P01"
+ :table-or-view-not-found "42S02"
+ :table-or-view-not-found-with-candidates "42S03"
+ :table-or-view-not-found-database-empty "42S04"
+ :unique-violation "23505"
+ :integrity-constraint-violation "23000"})
+
+(def error-codes
+ "Vendor-specific error codes returned by supported application databases."
+ ;; MySQL and MariaDB use one SQLSTATE for every integrity-constraint failure, so `ER_DUP_ENTRY` identifies
+ ;; duplicate keys. Consult a vendor code only after its SQLSTATE matched: H2 and MySQL both use small integers,
+ ;; so a bare code lookup would classify the other vendor's error.
+ {:mysql/duplicate-entry 1062})
+
+(def ^:private table-not-found-states
+ (into #{} (map sql-states) [:undefined-table
+ :table-or-view-not-found
+ :table-or-view-not-found-with-candidates
+ :table-or-view-not-found-database-empty]))
+
+(defn- sql-error-kind [^SQLException e]
+ (let [state (.getSQLState e)]
+ (cond
+ (contains? table-not-found-states state)
+ :table-not-found
+
+ (= (sql-states :unique-violation) state)
+ :duplicate-key
+
+ (and (= (sql-states :integrity-constraint-violation) state)
+ (= (error-codes :mysql/duplicate-entry) (.getErrorCode e)))
+ :duplicate-key)))
+
+(defn- exception-chain
+ "Every exception reachable from `e`, following causes and JDBC's next-exception links without revisiting cycles."
+ [e]
+ (loop [pending [e]
+ seen #{}
+ found []]
+ (if-let [^Throwable x (first pending)]
+ (if (seen x)
+ (recur (rest pending) seen found)
+ (recur (into (vec (rest pending))
+ (remove nil?)
+ [(.getCause x)
+ (when (instance? SQLException x)
+ (.getNextException ^SQLException x))])
+ (conj seen x)
+ (conj found x)))
+ found)))
+
+(defn error-kind
+ "The class of database error `e` represents: `:table-not-found`, `:duplicate-key`, or nil when unrecognized.
+ Walks the full cause and JDBC next-exception chains; the first recognized class wins."
+ [e]
+ (some #(when (instance? SQLException %)
+ (sql-error-kind %))
+ (exception-chain e)))
+
+(defn table-not-found?
+ "Whether `e` was caused by querying a table that does not exist."
+ [e]
+ (= :table-not-found (error-kind e)))
diff --git a/src/metabase/app_db/transient_error.clj b/src/metabase/app_db/transient_error.clj
index 7671fe383a1..440cb8f7541 100644
--- a/src/metabase/app_db/transient_error.clj
+++ b/src/metabase/app_db/transient_error.clj
@@ -1,6 +1,8 @@
(ns metabase.app-db.transient-error
"Detection of transient application database errors (deadlocks, lock timeouts, serialization failures)
that may succeed on retry."
+ (:require
+ [metabase.util :as u])
(:import
(java.sql SQLException)))
@@ -30,4 +32,4 @@
(boolean
(some #(and (instance? SQLException %)
(transient-sql-exception? codes %))
- (take-while some? (iterate ex-cause e))))))
+ (u/full-exception-chain e)))))
diff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.clj
index 911fb4527c4..7c0732821dc 100644
--- a/src/metabase/search/appdb/index.clj
+++ b/src/metabase/search/appdb/index.clj
@@ -4,6 +4,7 @@
[clojure.string :as str]
[metabase.analytics-interface.core :as analytics]
[metabase.app-db.core :as mdb]
+ [metabase.app-db.sql-errors :as sql-errors]
[metabase.config.core :as config]
[metabase.search.appdb.specialization.api :as specialization]
[metabase.search.appdb.specialization.h2 :as h2]
@@ -19,9 +20,7 @@
[metabase.util.json :as json]
[metabase.util.log :as log]
[metabase.util.string :as string]
- [toucan2.core :as t2])
- (:import
- (org.postgresql.util PSQLException)))
+ [toucan2.core :as t2]))
(comment
h2/keep-me
@@ -170,7 +169,7 @@
(log/errorf "Error creating pending index table, cleaning up metadata: %s" (ex-message e))
(try
(t2/with-connection [safe-conn (mdb/app-db)]
- (search.db/delete-index-metadata-by-name-on-conn! safe-conn (name table-name)))
+ (search.db/delete-index-metadata-by-name! safe-conn (name table-name)))
(catch Exception del-e
(log/warnf "Error clearing out search metadata after failure: %s" (ex-message del-e))))
(sync-tracking-atoms!))))
@@ -232,13 +231,6 @@
(dissoc :native_query)
(merge (specialization/extra-entry-fields entity)))))
-(defn- table-not-found-exception? [e]
- ;; Use with care, obviously this can give false positives if used with a query that's *actually* malformed.
- ;; TODO we should handle the MySQL and MariaDB flavors here too
- (or (instance? PSQLException (ex-cause e))
- (= mdb/jdbc-sql-syntax-error-exception-classname
- (some-> e ex-cause class .getName))))
-
(defn- retry-upsert-ex [table-type table-name-before table-name-after e-before e-after]
(ex-info "Failed retrying search index batch upsert"
{:table-type table-type
@@ -258,15 +250,12 @@
(f)))
(defn- safe-batch-upsert!
- "A version of batch-upsert! that no-ops for missing indexes, and handles stale index tracking metadata.
-
- Returns the name of the table that was written to, or nil if there is none being tracked, or nil
- if the upsert failed for any other reason — in which case the failure is logged at ERROR and we
- continue so the rest of the reindex can finish and activate whatever was successfully written.
+ "Upsert a batch into the tracked table, refreshing stale tracking once when the table turns out to be missing.
- We recover gracefully the first time if the tracking atom was stale, but do not check again on retry."
+ Returns the table name written, or nil if no table is tracked or the batch is skipped.
+ Throws when the tracked table is missing and the refresh names the same table, when the retry hits a missing
+ table again, and on interruption. Any other failure is logged and the batch is skipped."
[table-type table-name-fn entries]
- ;; For convenience, no-op if we are not tracking any table.
(when-let [table-name (table-name-fn)]
(let [upsert! (fn [t]
(isolate-write! #(specialization/batch-upsert! t entries))
@@ -278,7 +267,7 @@
(throw ie))
(catch Exception e
;; If the failure is a legitimately non-existent table, refresh tracking and retry once.
- (if (and (table-not-found-exception? e) (not (exists? table-name)))
+ (if (and (sql-errors/table-not-found? e) (not (exists? table-name)))
(when-let [refreshed-table-name (do (sync-tracking-atoms!) (table-name-fn))]
(if (= table-name refreshed-table-name)
(throw (ex-info "Currently tracked index does not exist" {:table-name table-name} e))
@@ -288,7 +277,7 @@
(.interrupt (Thread/currentThread))
(throw ie))
(catch Exception e2
- (if (table-not-found-exception? e2)
+ (if (sql-errors/table-not-found? e2)
(throw (retry-upsert-ex table-type table-name refreshed-table-name e e2))
(do (analytics/inc! :metabase-search/appdb-index-batches-skipped {:table-type table-type})
(log/errorf "Error upserting search index batch into %s table %s after refresh; skipping batch and continuing: %s"
@@ -373,7 +362,7 @@
;; The table can disappear after we read its name, especially during tests.
{search-model (try (isolate-write!
#(search.db/delete-index-rows! table-name search-model (set ids)))
- (catch Exception e (if (table-not-found-exception? e) 0 (throw e))))})))
+ (catch Exception e (if (sql-errors/table-not-found? e) 0 (throw e))))})))
(apply merge-with +)
(into {}))))
@@ -398,9 +387,11 @@
;; stop tracking any pending table
(when-let [table-name (pending-table)]
(when-not *mocking-tables*
- (let [deleted (search-index-metadata/delete-index! :appdb (search.spec/index-version-hash) table-name)]
+ (let [deleted (search-index-metadata/delete-non-active-index! :appdb
+ (search.spec/index-version-hash)
+ table-name)]
(when (pos? deleted)
- (log/infof "Deleted %d pending indices" deleted))))
+ (log/infof "Deleted %d non-active metadata rows for index %s" deleted table-name))))
(swap! *indexes* assoc :pending nil))
(maybe-create-pending!)
(activate-table!))]
diff --git a/src/metabase/search/core.clj b/src/metabase/search/core.clj
index 5ed0db4be15..bff0a5019f6 100644
--- a/src/metabase/search/core.clj
+++ b/src/metabase/search/core.clj
@@ -203,10 +203,13 @@
(throw e)))))))
(defn reindex!
- "Populate a new index, and make it active. Simultaneously updates the current index.
- Returns a future that will complete when the reindexing is done.
- Respects `search.ingestion/*force-sync*` and waits for the future if it's true.
- Alternately, if `:async?` is false, it will also run synchronously."
+ "Rebuild the search index.
+ By default, stages a new index and activates it once populated.
+ The active index keeps receiving updates meanwhile.
+ With `:in-place? true`, empties and repopulates the active index instead.
+ Runs asynchronously and returns a future.
+ Runs synchronously and returns a delivered promise when `:async?` is false or
+ [[search.ingestion/*force-sync*]] is true."
[& {:keys [async?] :or {async? true} :as opts}]
(let [f (fn []
(try
diff --git a/src/metabase/search/db.clj b/src/metabase/search/db.clj
index a702047c954..627ccf697de 100644
--- a/src/metabase/search/db.clj
+++ b/src/metabase/search/db.clj
@@ -29,7 +29,6 @@
:native_search_terms}
(remove #{:id :created_at :updated_at} search.spec/attr-columns))]
[column {:optional true} ::h2x/expr])))
-
(mu/defn spec-index-reducible-rows
"A reducible of the indexable rows of `search-model` (see `metabase.search.ingestion.query/spec-index-query`)
matching `where-clause`, or every row when it is nil.
@@ -120,8 +119,10 @@
[table-name :- [:or :keyword :string]]
(t2/query (sql.helpers/drop-table :if-exists table-name)))
+;; `IF EXISTS` cannot reliably report whether it dropped a table: PostgreSQL emits only a JDBC warning, which Toucan
+;; does not expose, and H2 emits nothing. Let an absent table throw so callers can detect races.
(mu/defn drop-search-index-table!
- "Drop the search index table named `table-name`."
+ "Drop the search index table named `table-name`, throwing if it is already gone."
[table-name :- [:or :keyword :string]]
(t2/query (sql.helpers/drop-table table-name)))
@@ -266,19 +267,25 @@
[version :- :string]
(t2/delete! :model/SearchIndexMetadata :version version))
-(mu/defn delete-index-metadata-by-name-on-conn!
- "Delete the SearchIndexMetadata rows named `index-name`, on `conn`."
+(mu/defn delete-index-metadata-by-name!
+ "Delete the SearchIndexMetadata rows named `index-name` using `conn`."
[conn :- (ms/InstanceOfClass java.sql.Connection)
index-name :- :string]
(t2/delete! :conn conn :model/SearchIndexMetadata :index_name index-name))
-(mu/defn delete-index-metadata!
- "Delete the SearchIndexMetadata row of `engine`, `version`, `lang-code`, and `index-name`."
+(mu/defn delete-non-active-index-metadata!
+ "Delete the SearchIndexMetadata rows of `engine`, `version`, `lang-code`, and `index-name` in any
+ non-active state (pending or retired)."
[engine :- :keyword
version :- :string
lang-code :- :string
index-name :- :string]
- (t2/delete! :model/SearchIndexMetadata :engine engine :version version :lang_code lang-code :index_name index-name))
+ (t2/delete! :model/SearchIndexMetadata
+ :engine engine
+ :version version
+ :lang_code lang-code
+ :index_name index-name
+ :status [:not= :active]))
(mu/defn index-metadata
"The name, status, and creation time of the active and pending SearchIndexMetadata rows of `engine`, `version`, and
@@ -309,6 +316,19 @@
lang-code :- :string]
(t2/exists? :model/SearchIndexMetadata :engine engine :version version :lang_code lang-code :status :pending))
+(mu/defn lock-pending-index-metadata!
+ "Lock and return the pending SearchIndexMetadata row of `engine`, `version`, and `lang-code`, if one exists.
+ Must be called inside the transaction that will promote the row."
+ [engine :- :keyword
+ version :- :string
+ lang-code :- :string]
+ (t2/select-one [:model/SearchIndexMetadata :id]
+ :engine engine
+ :version version
+ :lang_code lang-code
+ :status :pending
+ {:for :update}))
+
(mu/defn delete-retired-index-metadata!
"Delete the retired SearchIndexMetadata rows of `engine`, `version`, and `lang-code`."
[engine :- :keyword
diff --git a/src/metabase/search/models/search_index_metadata.clj b/src/metabase/search/models/search_index_metadata.clj
index 9d9993291ed..68233aa5ac1 100644
--- a/src/metabase/search/models/search_index_metadata.clj
+++ b/src/metabase/search/models/search_index_metadata.clj
@@ -52,16 +52,19 @@
;; We assume that failure corresponds to a unique index conflict (a pending entry already exists)
false)))))
-(defn delete-index!
- "Delete the given pending index, as long as its still pending."
+(defn delete-non-active-index!
+ "Delete the metadata rows for `index-name` unless it is the active index.
+ Returns the number of rows deleted."
[engine version index-name]
- (search.db/delete-index-metadata! engine version (i18n/site-locale-string) (name index-name)))
+ (search.db/delete-non-active-index-metadata! engine version (i18n/site-locale-string) (name index-name)))
(defn active-pending!
"If there is 'pending' index, make it 'active'. Return the name of the active index, regardless."
[engine version]
(t2/with-transaction [_conn]
- (when (search.db/pending-index-metadata-exists? engine version (i18n/site-locale-string))
+ ;; Lock the exact row before retiring the active index. A concurrent reset can otherwise delete the pending row
+ ;; after the existence check, leaving the coordinate with no active index.
+ (when (search.db/lock-pending-index-metadata! engine version (i18n/site-locale-string))
(search.db/delete-retired-index-metadata! engine version (i18n/site-locale-string))
(search.db/retire-active-index-metadata! engine version (i18n/site-locale-string))
(search.db/activate-pending-index-metadata! engine version (i18n/site-locale-string)))
diff --git a/test/metabase/app_db/sql_errors_test.clj b/test/metabase/app_db/sql_errors_test.clj
new file mode 100644
index 00000000000..3399c0c6a9c
--- /dev/null
+++ b/test/metabase/app_db/sql_errors_test.clj
@@ -0,0 +1,46 @@
+(ns metabase.app-db.sql-errors-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.app-db.sql-errors :as sql-errors])
+ (:import
+ (java.sql SQLException)))
+
+(set! *warn-on-reflection* true)
+
+(deftest ^:parallel error-kind-test
+ (testing "missing tables"
+ (are [state] (= :table-not-found (sql-errors/error-kind (SQLException. "missing" state)))
+ "42P01" "42S02" "42S03" "42S04"))
+ (testing "duplicate keys"
+ (is (= :duplicate-key (sql-errors/error-kind (SQLException. "duplicate" "23505"))))
+ (testing "MySQL and MariaDB report a catch-all state, disambiguated by the vendor code"
+ (is (= :duplicate-key (sql-errors/error-kind (SQLException. "Duplicate entry" "23000" 1062))))
+ (is (nil? (sql-errors/error-kind (SQLException. "Cannot add foreign key" "23000" 1452))))))
+ (testing "unrecognized errors"
+ (is (nil? (sql-errors/error-kind (SQLException. "syntax" "42000"))))
+ (is (nil? (sql-errors/error-kind (SQLException. "no state"))))
+ (is (nil? (sql-errors/error-kind (ex-info "plain" {}))))))
+
+(deftest ^:parallel error-kind-cause-chain-test
+ (testing "the first recognized cause wins, however deeply it is wrapped"
+ (let [wrapped (ex-info "outer" {} (RuntimeException. "middle" (SQLException. "missing" "42P01")))]
+ (is (= :table-not-found (sql-errors/error-kind wrapped)))
+ (is (true? (sql-errors/table-not-found? wrapped)))))
+ (testing "an unrecognized outer SQLException does not hide a recognized cause"
+ (is (= :duplicate-key (sql-errors/error-kind (SQLException. "outer" "HY000" 0 (SQLException. "dup" "23505"))))))
+ (testing "a chain with no recognized cause"
+ (let [wrapped (ex-info "outer" {} (SQLException. "syntax" "42000"))]
+ (is (nil? (sql-errors/error-kind wrapped)))
+ (is (false? (sql-errors/table-not-found? wrapped))))))
+
+(deftest ^:parallel error-kind-next-exception-chain-test
+ (testing "a recognized JDBC sibling is reached through getNextException"
+ (let [batch-error (doto (SQLException. "batch failed" "HY000")
+ (.setNextException (SQLException. "missing" "42S02")))]
+ (is (= :table-not-found (sql-errors/error-kind batch-error)))
+ (is (true? (sql-errors/table-not-found? batch-error)))))
+ (testing "cycles across cause and next-exception links terminate"
+ (let [outer (SQLException. "outer" "HY000")
+ inner (SQLException. "inner" "42000" outer)]
+ (.setNextException outer inner)
+ (is (nil? (sql-errors/error-kind outer))))))
diff --git a/test/metabase/search/api_test.clj b/test/metabase/search/api_test.clj
index 9b53979cb27..ae20c62ef88 100644
--- a/test/metabase/search/api_test.clj
+++ b/test/metabase/search/api_test.clj
@@ -2072,16 +2072,18 @@
"result count is not observed on error responses")))))))
(deftest ^:synchronized multiple-limits-test
- (when (search/supports-index?)
- ;; This test is failing with "no index" for some reason, forcing the reindex
- (mt/user-real-request :crowberto :post 200 "search/force-reindex"))
(testing "Multiple `limit` query args should be handled correctly (#45345)"
- (let [total-count (-> (mt/user-real-request :crowberto :get 200 "search?q=product")
- :data count)
- result-count (-> (mt/user-real-request :crowberto :get 200 "search?q=product&limit=1&limit=3")
- :data count)]
- (is (>= total-count result-count))
- (is (= 1 result-count)))))
+ ;; Repeated query-parameter keywords reach Ring as `limit=1&limit=3`, so the mock client exercises the same
+ ;; duplicate-parameter parsing as a real request.
+ (let [q (str "multiplelimits" (u/lower-case-en (mt/random-name)))]
+ (mt/with-temp [:model/Card _ {:name (str q " one")}
+ :model/Card _ {:name (str q " two")}]
+ (let [total-count (-> (mt/user-http-request :crowberto :get 200 "search" :q q)
+ :data count)
+ result-count (-> (mt/user-http-request :crowberto :get 200 "search" :q q :limit 1 :limit 3)
+ :data count)]
+ (is (= 2 total-count))
+ (is (= 1 result-count)))))))
(deftest ^:synchronized delete-database-hides-cards-from-search-test
(testing "When deleting a database, cards referring to that database should be hidden from search"
diff --git a/test/metabase/search/appdb/index_test.clj b/test/metabase/search/appdb/index_test.clj
index dabf7ba8580..5256ea6e65f 100644
--- a/test/metabase/search/appdb/index_test.clj
+++ b/test/metabase/search/appdb/index_test.clj
@@ -823,8 +823,10 @@
(is (true? (t2/exists? :model/User))))))
(testing "upsert"
(t2/with-transaction [_conn]
+ ;; Include a non-key column so PostgreSQL reaches the missing-table failure instead of rejecting an empty
+ ;; `DO UPDATE SET` clause.
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Currently tracked index does not exist"
(#'search.index/safe-batch-upsert! :active (constantly table-name)
- [{:model "card" :model_id "1"}])))
+ [{:model "card" :model_id "1" :name "x"}])))
(testing "\nthe enclosing transaction remains usable"
(is (true? (t2/exists? :model/User)))))))))))
diff --git a/test/metabase/search/models/search_index_metadata_test.clj b/test/metabase/search/models/search_index_metadata_test.clj
index 1f1b0f3b7bf..b31e7d8ef70 100644
--- a/test/metabase/search/models/search_index_metadata_test.clj
+++ b/test/metabase/search/models/search_index_metadata_test.clj
@@ -2,6 +2,7 @@
(:require
[clojure.test :refer :all]
[java-time.api :as t]
+ [metabase.search.db :as search.db]
[metabase.search.models.search-index-metadata :as search-index-metadata]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
@@ -28,11 +29,18 @@
(is (false? (search-index-metadata/create-pending! engine version index-2))))
(testing "You can activate an index"
(is (= index-1 (search-index-metadata/active-pending! engine version)))
- (is (= {:active index-1} (indexes))))
+ (is (= {:active index-1} (indexes)))
+ (testing "Deleting an active index is a no-op"
+ (is (zero? (search-index-metadata/delete-non-active-index! engine version index-1)))
+ (is (= {:active index-1} (indexes)))))
(testing "If there is no pending index, it will return the current index"
(is (= index-1 (search-index-metadata/active-pending! engine version))))
(testing "You can retire an index"
(is (search-index-metadata/create-pending! engine version index-2))
+ (testing "Deleting a pending index removes it"
+ (is (= 1 (search-index-metadata/delete-non-active-index! engine version index-2)))
+ (is (= {:active index-1} (indexes)))
+ (is (search-index-metadata/create-pending! engine version index-2)))
(is (= {:active index-1 :pending index-2} (indexes)))
(is (= index-2 (search-index-metadata/active-pending! engine version)))
(is (= {:retired index-1 :active index-2} (indexes))))
@@ -42,7 +50,48 @@
(is (= index-3 (search-index-metadata/active-pending! engine version)))
(is (= {:retired index-2 :active index-3} (indexes)))
(is (search-index-metadata/create-pending! engine version index-4))
- (is (= {:retired index-2 :active index-3 :pending index-4} (indexes)))))))
+ (is (= {:retired index-2 :active index-3 :pending index-4} (indexes))))
+ (testing "Deleting retired and pending indexes removes them; the active index is untouched"
+ (is (= 1 (search-index-metadata/delete-non-active-index! engine version index-2)))
+ (is (= 1 (search-index-metadata/delete-non-active-index! engine version index-4)))
+ (is (zero? (search-index-metadata/delete-non-active-index! engine version index-3)))
+ (is (= {:active index-3} (indexes)))))))
+
+(deftest delete-non-active-index-racing-with-promotion-test
+ (let [engine :something-futureproof
+ version (str (random-uuid))
+ active-index (str (random-uuid))
+ pending-index (str (random-uuid))
+ lock-acquired (promise)
+ release-lock (promise)
+ deleting (promise)
+ original-prune search.db/delete-retired-index-metadata!]
+ (try
+ (is (search-index-metadata/create-pending! engine version active-index))
+ (is (= active-index (search-index-metadata/active-pending! engine version)))
+ (is (search-index-metadata/create-pending! engine version pending-index))
+ (mt/with-dynamic-fn-redefs
+ [search.db/delete-retired-index-metadata!
+ (fn [& args]
+ (deliver lock-acquired true)
+ @release-lock
+ (apply original-prune args))]
+ (let [promotion (future (search-index-metadata/active-pending! engine version))]
+ (is (true? (deref lock-acquired 5000 false)))
+ (let [deletion (future
+ (deliver deleting true)
+ (search-index-metadata/delete-non-active-index! engine version pending-index))]
+ @deleting
+ (is (= ::blocked (deref deletion 100 ::blocked)))
+ (deliver release-lock true)
+ (is (= pending-index (deref promotion 5000 ::timeout)))
+ (is (zero? (deref deletion 5000 ::timeout)))
+ (is (= pending-index
+ (t2/select-one-fn :index_name :model/SearchIndexMetadata
+ :engine engine :version version :status :active))))))
+ (finally
+ (deliver release-lock true)
+ (t2/delete! :model/SearchIndexMetadata :engine engine :version version)))))
(deftest delete-obsolete!-test
(t2/with-transaction [_ t2.connection/*current-connectable* {:rollback-only true}]
diff --git a/test/metabase/test/util.clj b/test/metabase/test/util.clj
index 8084ee9fafd..0b36d7a1aa2 100644
--- a/test/metabase/test/util.clj
+++ b/test/metabase/test/util.clj
@@ -33,6 +33,7 @@
[metabase.premium-features.test-util :as premium-features.test-util]
[metabase.query-processor.util :as qp.util]
[metabase.search.core :as search]
+ [metabase.search.spec :as search.spec]
[metabase.settings.core :as setting]
[metabase.settings.models.setting]
[metabase.settings.models.setting.cache :as setting.cache]
@@ -951,6 +952,21 @@
model
[model (first (t2/primary-keys model))]))
+(defn- delete-new-rows!
+ "Delete the rows of `model` whose `pk` exceeds `old-max-id`, skipping Toucan hooks. Returns the row count."
+ [model pk old-max-id]
+ (t2/query-one {:delete-from (t2/table-name model)
+ :where [:and
+ ;; The first use in a test run may have no previous maximum ID.
+ (if old-max-id [:> pk old-max-id] true)
+ (with-model-cleanup-additional-conditions model)]}))
+
+(defn- search-relevant-models
+ "Models whose rows, deleted with raw SQL, can leave stale rows in the search index."
+ []
+ ;; Deleting a user cascades to their personal collection, which is indexed.
+ (conj (set (keys (search.spec/model-hooks))) :model/User))
+
(defn- reindex-search-index! []
;; Wiping and repopulating the whole index table can deadlock against a concurrent writer — search ingestion from
;; another test's writes, or another test's cleanup doing this same thing. The loser of a deadlock has lost nothing
@@ -982,17 +998,14 @@
(testing (str "\n" (pr-str (cons 'with-model-cleanup (map (comp name first) models))) "\n")
(f))
(finally
- (doseq [[model pk] models
- ;; might not have an old max ID if this is the first time the macro is used in this test run.
- :let [old-max-id (get model->old-max-id model)
- max-id-condition (if old-max-id [:> pk old-max-id] true)
- additional-conditions (with-model-cleanup-additional-conditions model)
- where-clause [:and max-id-condition additional-conditions]]]
- (t2/query-one
- {:delete-from (t2/table-name model)
- :where where-clause}))
- ;; TODO we don't (currently) have index update hooks on deletes, so we need this to ensure rollback happens.
- (reindex-search-index!)))))
+ (let [search-relevant? (search-relevant-models)
+ reindex? (some (comp search-relevant? first) models)]
+ (doseq [[model pk] models]
+ (delete-new-rows! model pk (get model->old-max-id model)))
+ ;; Search has no delete hook, so the body may already have deleted a row while leaving its indexed document
+ ;; behind. Reindex whenever the declared cleanup scope can affect search, even when the final delete is empty.
+ (when reindex?
+ (reindex-search-index!)))))))
(defmacro with-model-cleanup
"Execute `body`, then delete any *new* rows created for each model in `models`.
@@ -1042,6 +1055,14 @@
(testing "Shouldn't delete other Cards"
(is (pos? (t2/count :model/Card)))))))))
+(deftest with-model-cleanup-reindexes-search-models-test
+ (testing "a search-relevant cleanup reindexes even when the body already removed every new row"
+ (let [reindexes (atom 0)]
+ (dynamic-redefs/with-dynamic-fn-redefs
+ [reindex-search-index! #(swap! reindexes inc)]
+ (with-model-cleanup [:model/Card]))
+ (is (= 1 @reindexes)))))
+
(deftest reindex-search-index!-test
(testing "a transient appdb failure is retried"
(let [attempts (atom 0)]