The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix search index metadata deletion guard (#80309)
Chris Truter1f1643d5b8c6
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex ec0b984e419..964d6753907 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -349,11 +349,12 @@app-db{:team "UX West"- :api #{metabase.app-db.cluster-lock ; TODO FIXME+ :api #{metabase.app-db.cluster-lock ; TODO (Chris 2026-09-16) -- callers should go through app-db.coremetabase.app-db.coremetabase.app-db.initmetabase.app-db.settingmetabase.app-db.setup+ metabase.app-db.sql-errorsmetabase.app-db.transient-error}:uses #{auth-providerconnection-pooldiff --git a/src/metabase/app_db/core.clj b/src/metabase/app_db/core.cljindex 80b17e93f3a..a6fb131a962 100644--- a/src/metabase/app_db/core.clj+++ b/src/metabase/app_db/core.clj@@ -16,7 +16,6 @@[metabase.app-db.encryption :as mdb.encryption][metabase.app-db.env :as mdb.env][metabase.app-db.format]- [metabase.app-db.h2 :as mdb.h2][metabase.app-db.honeysql-guard][metabase.app-db.jdbc-protocols :as mdb.jdbc-protocols][metabase.app-db.liquibase :as liquibase]@@ -55,8 +54,6 @@broken-out-details->DataSource][mdb.envdb-file]- [mdb.h2- jdbc-sql-syntax-error-exception-classname][mdb.jdbc-protocolsclob->str][mdb.encryptiondiff --git a/src/metabase/app_db/h2.clj b/src/metabase/app_db/h2.cljindex 339e9edb6ba..67c468c3b70 100644--- a/src/metabase/app_db/h2.clj+++ b/src/metabase/app_db/h2.clj@@ -5,7 +5,3 @@"Value of `org.h2.api.ErrorCode/STATEMENT_WAS_CANCELED`, inlined so callers can recognize an H2query cancelation without the H2 library on the classpath."57014)--(def jdbc-sql-syntax-error-exception-classname- "Class name of H2's syntax-error exception, matched by name so callers need not import the class."- "org.h2.jdbc.JdbcSQLSyntaxErrorException")diff --git a/src/metabase/app_db/sql_errors.clj b/src/metabase/app_db/sql_errors.cljnew file mode 100644index 00000000000..649694213ca--- /dev/null+++ b/src/metabase/app_db/sql_errors.clj@@ -0,0 +1,82 @@+(ns metabase.app-db.sql-errors+ "Classification of SQL errors raised by the supported application databases.+ [[error-kind]] names the class of error an exception represents; [[table-not-found?]] is its common special case."+ (:import+ (java.sql SQLException)))++(set! *warn-on-reflection* true)++;; The registries name every code we recognize so the classifier reads by intent rather than by bare string.+;; Keep them cumulative: append newly recognized codes rather than removing entries that temporarily have no caller.+;; They are the shared app-db SQL vocabulary, not an inventory of what the current change happens to use.+;; Keep the missing-table states aligned with `impl-table-known-to-not-exist?` in the H2, Postgres, and MySQL+;; drivers: this module cannot depend on the driver module, so the lists are maintained in both places.+(def sql-states+ "SQLSTATE codes returned by supported application databases."+ ;; `undefined_table` (42P01) is PostgreSQL-specific. 42S02 is the ODBC/X-Open state for a missing table; H2 adds+ ;; two of its own, mapping `TABLE_OR_VIEW_NOT_FOUND_1`, `..._WITH_CANDIDATES_2`, and `..._DATABASE_EMPTY_1` in+ ;; `org.h2.api.ErrorCode` to 42S02, 42S03, and 42S04. 23000 is the standard integrity-constraint state; 23505 is+ ;; the PostgreSQL and H2 unique-violation state.+ {:undefined-table "42P01"+ :table-or-view-not-found "42S02"+ :table-or-view-not-found-with-candidates "42S03"+ :table-or-view-not-found-database-empty "42S04"+ :unique-violation "23505"+ :integrity-constraint-violation "23000"})++(def error-codes+ "Vendor-specific error codes returned by supported application databases."+ ;; MySQL and MariaDB use one SQLSTATE for every integrity-constraint failure, so `ER_DUP_ENTRY` identifies+ ;; duplicate keys. Consult a vendor code only after its SQLSTATE matched: H2 and MySQL both use small integers,+ ;; so a bare code lookup would classify the other vendor's error.+ {:mysql/duplicate-entry 1062})++(def ^:private table-not-found-states+ (into #{} (map sql-states) [:undefined-table+ :table-or-view-not-found+ :table-or-view-not-found-with-candidates+ :table-or-view-not-found-database-empty]))++(defn- sql-error-kind [^SQLException e]+ (let [state (.getSQLState e)]+ (cond+ (contains? table-not-found-states state)+ :table-not-found++ (= (sql-states :unique-violation) state)+ :duplicate-key++ (and (= (sql-states :integrity-constraint-violation) state)+ (= (error-codes :mysql/duplicate-entry) (.getErrorCode e)))+ :duplicate-key)))++(defn- exception-chain+ "Every exception reachable from `e`, following causes and JDBC's next-exception links without revisiting cycles."+ [e]+ (loop [pending [e]+ seen #{}+ found []]+ (if-let [^Throwable x (first pending)]+ (if (seen x)+ (recur (rest pending) seen found)+ (recur (into (vec (rest pending))+ (remove nil?)+ [(.getCause x)+ (when (instance? SQLException x)+ (.getNextException ^SQLException x))])+ (conj seen x)+ (conj found x)))+ found)))++(defn error-kind+ "The class of database error `e` represents: `:table-not-found`, `:duplicate-key`, or nil when unrecognized.+ Walks the full cause and JDBC next-exception chains; the first recognized class wins."+ [e]+ (some #(when (instance? SQLException %)+ (sql-error-kind %))+ (exception-chain e)))++(defn table-not-found?+ "Whether `e` was caused by querying a table that does not exist."+ [e]+ (= :table-not-found (error-kind e)))diff --git a/src/metabase/app_db/transient_error.clj b/src/metabase/app_db/transient_error.cljindex 7671fe383a1..440cb8f7541 100644--- a/src/metabase/app_db/transient_error.clj+++ b/src/metabase/app_db/transient_error.clj@@ -1,6 +1,8 @@(ns metabase.app-db.transient-error"Detection of transient application database errors (deadlocks, lock timeouts, serialization failures)that may succeed on retry."+ (:require+ [metabase.util :as u])(:import(java.sql SQLException)))@@ -30,4 +32,4 @@(boolean(some #(and (instance? SQLException %)(transient-sql-exception? codes %))- (take-while some? (iterate ex-cause e))))))+ (u/full-exception-chain e)))))diff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.cljindex 911fb4527c4..7c0732821dc 100644--- a/src/metabase/search/appdb/index.clj+++ b/src/metabase/search/appdb/index.clj@@ -4,6 +4,7 @@[clojure.string :as str][metabase.analytics-interface.core :as analytics][metabase.app-db.core :as mdb]+ [metabase.app-db.sql-errors :as sql-errors][metabase.config.core :as config][metabase.search.appdb.specialization.api :as specialization][metabase.search.appdb.specialization.h2 :as h2]@@ -19,9 +20,7 @@[metabase.util.json :as json][metabase.util.log :as log][metabase.util.string :as string]- [toucan2.core :as t2])- (:import- (org.postgresql.util PSQLException)))+ [toucan2.core :as t2]))(commenth2/keep-me@@ -170,7 +169,7 @@(log/errorf "Error creating pending index table, cleaning up metadata: %s" (ex-message e))(try(t2/with-connection [safe-conn (mdb/app-db)]- (search.db/delete-index-metadata-by-name-on-conn! safe-conn (name table-name)))+ (search.db/delete-index-metadata-by-name! safe-conn (name table-name)))(catch Exception del-e(log/warnf "Error clearing out search metadata after failure: %s" (ex-message del-e))))(sync-tracking-atoms!))))@@ -232,13 +231,6 @@(dissoc :native_query)(merge (specialization/extra-entry-fields entity)))))-(defn- table-not-found-exception? [e]- ;; Use with care, obviously this can give false positives if used with a query that's *actually* malformed.- ;; TODO we should handle the MySQL and MariaDB flavors here too- (or (instance? PSQLException (ex-cause e))- (= mdb/jdbc-sql-syntax-error-exception-classname- (some-> e ex-cause class .getName))))-(defn- retry-upsert-ex [table-type table-name-before table-name-after e-before e-after](ex-info "Failed retrying search index batch upsert"{:table-type table-type@@ -258,15 +250,12 @@(f)))(defn- safe-batch-upsert!- "A version of batch-upsert! that no-ops for missing indexes, and handles stale index tracking metadata.-- Returns the name of the table that was written to, or nil if there is none being tracked, or nil- if the upsert failed for any other reason — in which case the failure is logged at ERROR and we- continue so the rest of the reindex can finish and activate whatever was successfully written.+ "Upsert a batch into the tracked table, refreshing stale tracking once when the table turns out to be missing.- We recover gracefully the first time if the tracking atom was stale, but do not check again on retry."+ Returns the table name written, or nil if no table is tracked or the batch is skipped.+ Throws when the tracked table is missing and the refresh names the same table, when the retry hits a missing+ table again, and on interruption. Any other failure is logged and the batch is skipped."[table-type table-name-fn entries]- ;; For convenience, no-op if we are not tracking any table.(when-let [table-name (table-name-fn)](let [upsert! (fn [t](isolate-write! #(specialization/batch-upsert! t entries))@@ -278,7 +267,7 @@(throw ie))(catch Exception e;; If the failure is a legitimately non-existent table, refresh tracking and retry once.- (if (and (table-not-found-exception? e) (not (exists? table-name)))+ (if (and (sql-errors/table-not-found? e) (not (exists? table-name)))(when-let [refreshed-table-name (do (sync-tracking-atoms!) (table-name-fn))](if (= table-name refreshed-table-name)(throw (ex-info "Currently tracked index does not exist" {:table-name table-name} e))@@ -288,7 +277,7 @@(.interrupt (Thread/currentThread))(throw ie))(catch Exception e2- (if (table-not-found-exception? e2)+ (if (sql-errors/table-not-found? e2)(throw (retry-upsert-ex table-type table-name refreshed-table-name e e2))(do (analytics/inc! :metabase-search/appdb-index-batches-skipped {:table-type table-type})(log/errorf "Error upserting search index batch into %s table %s after refresh; skipping batch and continuing: %s"@@ -373,7 +362,7 @@;; The table can disappear after we read its name, especially during tests.{search-model (try (isolate-write!#(search.db/delete-index-rows! table-name search-model (set ids)))- (catch Exception e (if (table-not-found-exception? e) 0 (throw e))))})))+ (catch Exception e (if (sql-errors/table-not-found? e) 0 (throw e))))})))(apply merge-with +)(into {}))))@@ -398,9 +387,11 @@;; stop tracking any pending table(when-let [table-name (pending-table)](when-not *mocking-tables*- (let [deleted (search-index-metadata/delete-index! :appdb (search.spec/index-version-hash) table-name)]+ (let [deleted (search-index-metadata/delete-non-active-index! :appdb+ (search.spec/index-version-hash)+ table-name)](when (pos? deleted)- (log/infof "Deleted %d pending indices" deleted))))+ (log/infof "Deleted %d non-active metadata rows for index %s" deleted table-name))))(swap! *indexes* assoc :pending nil))(maybe-create-pending!)(activate-table!))]diff --git a/src/metabase/search/core.clj b/src/metabase/search/core.cljindex 5ed0db4be15..bff0a5019f6 100644--- a/src/metabase/search/core.clj+++ b/src/metabase/search/core.clj@@ -203,10 +203,13 @@(throw e)))))))(defn reindex!- "Populate a new index, and make it active. Simultaneously updates the current index.- Returns a future that will complete when the reindexing is done.- Respects `search.ingestion/*force-sync*` and waits for the future if it's true.- Alternately, if `:async?` is false, it will also run synchronously."+ "Rebuild the search index.+ By default, stages a new index and activates it once populated.+ The active index keeps receiving updates meanwhile.+ With `:in-place? true`, empties and repopulates the active index instead.+ Runs asynchronously and returns a future.+ Runs synchronously and returns a delivered promise when `:async?` is false or+ [[search.ingestion/*force-sync*]] is true."[& {:keys [async?] :or {async? true} :as opts}](let [f (fn [](trydiff --git a/src/metabase/search/db.clj b/src/metabase/search/db.cljindex a702047c954..627ccf697de 100644--- a/src/metabase/search/db.clj+++ b/src/metabase/search/db.clj@@ -29,7 +29,6 @@:native_search_terms}(remove #{:id :created_at :updated_at} search.spec/attr-columns))][column {:optional true} ::h2x/expr])))-(mu/defn spec-index-reducible-rows"A reducible of the indexable rows of `search-model` (see `metabase.search.ingestion.query/spec-index-query`)matching `where-clause`, or every row when it is nil.@@ -120,8 +119,10 @@[table-name :- [:or :keyword :string]](t2/query (sql.helpers/drop-table :if-exists table-name)))+;; `IF EXISTS` cannot reliably report whether it dropped a table: PostgreSQL emits only a JDBC warning, which Toucan+;; does not expose, and H2 emits nothing. Let an absent table throw so callers can detect races.(mu/defn drop-search-index-table!- "Drop the search index table named `table-name`."+ "Drop the search index table named `table-name`, throwing if it is already gone."[table-name :- [:or :keyword :string]](t2/query (sql.helpers/drop-table table-name)))@@ -266,19 +267,25 @@[version :- :string](t2/delete! :model/SearchIndexMetadata :version version))-(mu/defn delete-index-metadata-by-name-on-conn!- "Delete the SearchIndexMetadata rows named `index-name`, on `conn`."+(mu/defn delete-index-metadata-by-name!+ "Delete the SearchIndexMetadata rows named `index-name` using `conn`."[conn :- (ms/InstanceOfClass java.sql.Connection)index-name :- :string](t2/delete! :conn conn :model/SearchIndexMetadata :index_name index-name))-(mu/defn delete-index-metadata!- "Delete the SearchIndexMetadata row of `engine`, `version`, `lang-code`, and `index-name`."+(mu/defn delete-non-active-index-metadata!+ "Delete the SearchIndexMetadata rows of `engine`, `version`, `lang-code`, and `index-name` in any+ non-active state (pending or retired)."[engine :- :keywordversion :- :stringlang-code :- :stringindex-name :- :string]- (t2/delete! :model/SearchIndexMetadata :engine engine :version version :lang_code lang-code :index_name index-name))+ (t2/delete! :model/SearchIndexMetadata+ :engine engine+ :version version+ :lang_code lang-code+ :index_name index-name+ :status [:not= :active]))(mu/defn index-metadata"The name, status, and creation time of the active and pending SearchIndexMetadata rows of `engine`, `version`, and@@ -309,6 +316,19 @@lang-code :- :string](t2/exists? :model/SearchIndexMetadata :engine engine :version version :lang_code lang-code :status :pending))+(mu/defn lock-pending-index-metadata!+ "Lock and return the pending SearchIndexMetadata row of `engine`, `version`, and `lang-code`, if one exists.+ Must be called inside the transaction that will promote the row."+ [engine :- :keyword+ version :- :string+ lang-code :- :string]+ (t2/select-one [:model/SearchIndexMetadata :id]+ :engine engine+ :version version+ :lang_code lang-code+ :status :pending+ {:for :update}))+(mu/defn delete-retired-index-metadata!"Delete the retired SearchIndexMetadata rows of `engine`, `version`, and `lang-code`."[engine :- :keyworddiff --git a/src/metabase/search/models/search_index_metadata.clj b/src/metabase/search/models/search_index_metadata.cljindex 9d9993291ed..68233aa5ac1 100644--- a/src/metabase/search/models/search_index_metadata.clj+++ b/src/metabase/search/models/search_index_metadata.clj@@ -52,16 +52,19 @@;; We assume that failure corresponds to a unique index conflict (a pending entry already exists)false)))))-(defn delete-index!- "Delete the given pending index, as long as its still pending."+(defn delete-non-active-index!+ "Delete the metadata rows for `index-name` unless it is the active index.+ Returns the number of rows deleted."[engine version index-name]- (search.db/delete-index-metadata! engine version (i18n/site-locale-string) (name index-name)))+ (search.db/delete-non-active-index-metadata! engine version (i18n/site-locale-string) (name index-name)))(defn active-pending!"If there is 'pending' index, make it 'active'. Return the name of the active index, regardless."[engine version](t2/with-transaction [_conn]- (when (search.db/pending-index-metadata-exists? engine version (i18n/site-locale-string))+ ;; Lock the exact row before retiring the active index. A concurrent reset can otherwise delete the pending row+ ;; after the existence check, leaving the coordinate with no active index.+ (when (search.db/lock-pending-index-metadata! engine version (i18n/site-locale-string))(search.db/delete-retired-index-metadata! engine version (i18n/site-locale-string))(search.db/retire-active-index-metadata! engine version (i18n/site-locale-string))(search.db/activate-pending-index-metadata! engine version (i18n/site-locale-string)))diff --git a/test/metabase/app_db/sql_errors_test.clj b/test/metabase/app_db/sql_errors_test.cljnew file mode 100644index 00000000000..3399c0c6a9c--- /dev/null+++ b/test/metabase/app_db/sql_errors_test.clj@@ -0,0 +1,46 @@+(ns metabase.app-db.sql-errors-test+ (:require+ [clojure.test :refer :all]+ [metabase.app-db.sql-errors :as sql-errors])+ (:import+ (java.sql SQLException)))++(set! *warn-on-reflection* true)++(deftest ^:parallel error-kind-test+ (testing "missing tables"+ (are [state] (= :table-not-found (sql-errors/error-kind (SQLException. "missing" state)))+ "42P01" "42S02" "42S03" "42S04"))+ (testing "duplicate keys"+ (is (= :duplicate-key (sql-errors/error-kind (SQLException. "duplicate" "23505"))))+ (testing "MySQL and MariaDB report a catch-all state, disambiguated by the vendor code"+ (is (= :duplicate-key (sql-errors/error-kind (SQLException. "Duplicate entry" "23000" 1062))))+ (is (nil? (sql-errors/error-kind (SQLException. "Cannot add foreign key" "23000" 1452))))))+ (testing "unrecognized errors"+ (is (nil? (sql-errors/error-kind (SQLException. "syntax" "42000"))))+ (is (nil? (sql-errors/error-kind (SQLException. "no state"))))+ (is (nil? (sql-errors/error-kind (ex-info "plain" {}))))))++(deftest ^:parallel error-kind-cause-chain-test+ (testing "the first recognized cause wins, however deeply it is wrapped"+ (let [wrapped (ex-info "outer" {} (RuntimeException. "middle" (SQLException. "missing" "42P01")))]+ (is (= :table-not-found (sql-errors/error-kind wrapped)))+ (is (true? (sql-errors/table-not-found? wrapped)))))+ (testing "an unrecognized outer SQLException does not hide a recognized cause"+ (is (= :duplicate-key (sql-errors/error-kind (SQLException. "outer" "HY000" 0 (SQLException. "dup" "23505"))))))+ (testing "a chain with no recognized cause"+ (let [wrapped (ex-info "outer" {} (SQLException. "syntax" "42000"))]+ (is (nil? (sql-errors/error-kind wrapped)))+ (is (false? (sql-errors/table-not-found? wrapped))))))++(deftest ^:parallel error-kind-next-exception-chain-test+ (testing "a recognized JDBC sibling is reached through getNextException"+ (let [batch-error (doto (SQLException. "batch failed" "HY000")+ (.setNextException (SQLException. "missing" "42S02")))]+ (is (= :table-not-found (sql-errors/error-kind batch-error)))+ (is (true? (sql-errors/table-not-found? batch-error)))))+ (testing "cycles across cause and next-exception links terminate"+ (let [outer (SQLException. "outer" "HY000")+ inner (SQLException. "inner" "42000" outer)]+ (.setNextException outer inner)+ (is (nil? (sql-errors/error-kind outer))))))diff --git a/test/metabase/search/api_test.clj b/test/metabase/search/api_test.cljindex 9b53979cb27..ae20c62ef88 100644--- a/test/metabase/search/api_test.clj+++ b/test/metabase/search/api_test.clj@@ -2072,16 +2072,18 @@"result count is not observed on error responses")))))))(deftest ^:synchronized multiple-limits-test- (when (search/supports-index?)- ;; This test is failing with "no index" for some reason, forcing the reindex- (mt/user-real-request :crowberto :post 200 "search/force-reindex"))(testing "Multiple `limit` query args should be handled correctly (#45345)"- (let [total-count (-> (mt/user-real-request :crowberto :get 200 "search?q=product")- :data count)- result-count (-> (mt/user-real-request :crowberto :get 200 "search?q=product&limit=1&limit=3")- :data count)]- (is (>= total-count result-count))- (is (= 1 result-count)))))+ ;; Repeated query-parameter keywords reach Ring as `limit=1&limit=3`, so the mock client exercises the same+ ;; duplicate-parameter parsing as a real request.+ (let [q (str "multiplelimits" (u/lower-case-en (mt/random-name)))]+ (mt/with-temp [:model/Card _ {:name (str q " one")}+ :model/Card _ {:name (str q " two")}]+ (let [total-count (-> (mt/user-http-request :crowberto :get 200 "search" :q q)+ :data count)+ result-count (-> (mt/user-http-request :crowberto :get 200 "search" :q q :limit 1 :limit 3)+ :data count)]+ (is (= 2 total-count))+ (is (= 1 result-count)))))))(deftest ^:synchronized delete-database-hides-cards-from-search-test(testing "When deleting a database, cards referring to that database should be hidden from search"diff --git a/test/metabase/search/appdb/index_test.clj b/test/metabase/search/appdb/index_test.cljindex dabf7ba8580..5256ea6e65f 100644--- a/test/metabase/search/appdb/index_test.clj+++ b/test/metabase/search/appdb/index_test.clj@@ -823,8 +823,10 @@(is (true? (t2/exists? :model/User))))))(testing "upsert"(t2/with-transaction [_conn]+ ;; Include a non-key column so PostgreSQL reaches the missing-table failure instead of rejecting an empty+ ;; `DO UPDATE SET` clause.(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Currently tracked index does not exist"(#'search.index/safe-batch-upsert! :active (constantly table-name)- [{:model "card" :model_id "1"}])))+ [{:model "card" :model_id "1" :name "x"}])))(testing "\nthe enclosing transaction remains usable"(is (true? (t2/exists? :model/User)))))))))))diff --git a/test/metabase/search/models/search_index_metadata_test.clj b/test/metabase/search/models/search_index_metadata_test.cljindex 1f1b0f3b7bf..b31e7d8ef70 100644--- a/test/metabase/search/models/search_index_metadata_test.clj+++ b/test/metabase/search/models/search_index_metadata_test.clj@@ -2,6 +2,7 @@(:require[clojure.test :refer :all][java-time.api :as t]+ [metabase.search.db :as search.db][metabase.search.models.search-index-metadata :as search-index-metadata][metabase.test :as mt][metabase.test.fixtures :as fixtures]@@ -28,11 +29,18 @@(is (false? (search-index-metadata/create-pending! engine version index-2))))(testing "You can activate an index"(is (= index-1 (search-index-metadata/active-pending! engine version)))- (is (= {:active index-1} (indexes))))+ (is (= {:active index-1} (indexes)))+ (testing "Deleting an active index is a no-op"+ (is (zero? (search-index-metadata/delete-non-active-index! engine version index-1)))+ (is (= {:active index-1} (indexes)))))(testing "If there is no pending index, it will return the current index"(is (= index-1 (search-index-metadata/active-pending! engine version))))(testing "You can retire an index"(is (search-index-metadata/create-pending! engine version index-2))+ (testing "Deleting a pending index removes it"+ (is (= 1 (search-index-metadata/delete-non-active-index! engine version index-2)))+ (is (= {:active index-1} (indexes)))+ (is (search-index-metadata/create-pending! engine version index-2)))(is (= {:active index-1 :pending index-2} (indexes)))(is (= index-2 (search-index-metadata/active-pending! engine version)))(is (= {:retired index-1 :active index-2} (indexes))))@@ -42,7 +50,48 @@(is (= index-3 (search-index-metadata/active-pending! engine version)))(is (= {:retired index-2 :active index-3} (indexes)))(is (search-index-metadata/create-pending! engine version index-4))- (is (= {:retired index-2 :active index-3 :pending index-4} (indexes)))))))+ (is (= {:retired index-2 :active index-3 :pending index-4} (indexes))))+ (testing "Deleting retired and pending indexes removes them; the active index is untouched"+ (is (= 1 (search-index-metadata/delete-non-active-index! engine version index-2)))+ (is (= 1 (search-index-metadata/delete-non-active-index! engine version index-4)))+ (is (zero? (search-index-metadata/delete-non-active-index! engine version index-3)))+ (is (= {:active index-3} (indexes)))))))++(deftest delete-non-active-index-racing-with-promotion-test+ (let [engine :something-futureproof+ version (str (random-uuid))+ active-index (str (random-uuid))+ pending-index (str (random-uuid))+ lock-acquired (promise)+ release-lock (promise)+ deleting (promise)+ original-prune search.db/delete-retired-index-metadata!]+ (try+ (is (search-index-metadata/create-pending! engine version active-index))+ (is (= active-index (search-index-metadata/active-pending! engine version)))+ (is (search-index-metadata/create-pending! engine version pending-index))+ (mt/with-dynamic-fn-redefs+ [search.db/delete-retired-index-metadata!+ (fn [& args]+ (deliver lock-acquired true)+ @release-lock+ (apply original-prune args))]+ (let [promotion (future (search-index-metadata/active-pending! engine version))]+ (is (true? (deref lock-acquired 5000 false)))+ (let [deletion (future+ (deliver deleting true)+ (search-index-metadata/delete-non-active-index! engine version pending-index))]+ @deleting+ (is (= ::blocked (deref deletion 100 ::blocked)))+ (deliver release-lock true)+ (is (= pending-index (deref promotion 5000 ::timeout)))+ (is (zero? (deref deletion 5000 ::timeout)))+ (is (= pending-index+ (t2/select-one-fn :index_name :model/SearchIndexMetadata+ :engine engine :version version :status :active))))))+ (finally+ (deliver release-lock true)+ (t2/delete! :model/SearchIndexMetadata :engine engine :version version)))))(deftest delete-obsolete!-test(t2/with-transaction [_ t2.connection/*current-connectable* {:rollback-only true}]diff --git a/test/metabase/test/util.clj b/test/metabase/test/util.cljindex 8084ee9fafd..0b36d7a1aa2 100644--- a/test/metabase/test/util.clj+++ b/test/metabase/test/util.clj@@ -33,6 +33,7 @@[metabase.premium-features.test-util :as premium-features.test-util][metabase.query-processor.util :as qp.util][metabase.search.core :as search]+ [metabase.search.spec :as search.spec][metabase.settings.core :as setting][metabase.settings.models.setting][metabase.settings.models.setting.cache :as setting.cache]@@ -951,6 +952,21 @@model[model (first (t2/primary-keys model))]))+(defn- delete-new-rows!+ "Delete the rows of `model` whose `pk` exceeds `old-max-id`, skipping Toucan hooks. Returns the row count."+ [model pk old-max-id]+ (t2/query-one {:delete-from (t2/table-name model)+ :where [:and+ ;; The first use in a test run may have no previous maximum ID.+ (if old-max-id [:> pk old-max-id] true)+ (with-model-cleanup-additional-conditions model)]}))++(defn- search-relevant-models+ "Models whose rows, deleted with raw SQL, can leave stale rows in the search index."+ []+ ;; Deleting a user cascades to their personal collection, which is indexed.+ (conj (set (keys (search.spec/model-hooks))) :model/User))+(defn- reindex-search-index! [];; Wiping and repopulating the whole index table can deadlock against a concurrent writer — search ingestion from;; another test's writes, or another test's cleanup doing this same thing. The loser of a deadlock has lost nothing@@ -982,17 +998,14 @@(testing (str "\n" (pr-str (cons 'with-model-cleanup (map (comp name first) models))) "\n")(f))(finally- (doseq [[model pk] models- ;; might not have an old max ID if this is the first time the macro is used in this test run.- :let [old-max-id (get model->old-max-id model)- max-id-condition (if old-max-id [:> pk old-max-id] true)- additional-conditions (with-model-cleanup-additional-conditions model)- where-clause [:and max-id-condition additional-conditions]]]- (t2/query-one- {:delete-from (t2/table-name model)- :where where-clause}))- ;; TODO we don't (currently) have index update hooks on deletes, so we need this to ensure rollback happens.- (reindex-search-index!)))))+ (let [search-relevant? (search-relevant-models)+ reindex? (some (comp search-relevant? first) models)]+ (doseq [[model pk] models]+ (delete-new-rows! model pk (get model->old-max-id model)))+ ;; Search has no delete hook, so the body may already have deleted a row while leaving its indexed document+ ;; behind. Reindex whenever the declared cleanup scope can affect search, even when the final delete is empty.+ (when reindex?+ (reindex-search-index!)))))))(defmacro with-model-cleanup"Execute `body`, then delete any *new* rows created for each model in `models`.@@ -1042,6 +1055,14 @@(testing "Shouldn't delete other Cards"(is (pos? (t2/count :model/Card)))))))))+(deftest with-model-cleanup-reindexes-search-models-test+ (testing "a search-relevant cleanup reindexes even when the body already removed every new row"+ (let [reindexes (atom 0)]+ (dynamic-redefs/with-dynamic-fn-redefs+ [reindex-search-index! #(swap! reindexes inc)]+ (with-model-cleanup [:model/Card]))+ (is (= 1 @reindexes)))))+(deftest reindex-search-index!-test(testing "a transient appdb failure is retried"(let [attempts (atom 0)]