Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

BOT-2110: sweep orphaned search index tables when a reindex starts (#81944)

Andrei Simionescu70a3d8cb4a73
diff --git a/src/metabase/search/appdb/core.clj b/src/metabase/search/appdb/core.clj
index 15ed88f1a6c..63588f4f99f 100644
--- a/src/metabase/search/appdb/core.clj
+++ b/src/metabase/search/appdb/core.clj
@@ -280,6 +280,7 @@
(defmethod search.engine/reindex! :search.engine/appdb
[_ {:keys [in-place?]}]
(try
+ (search.index/delete-obsolete-tables!)
(search.index/ensure-ready!)
(if in-place?
(when-let [table (search.index/active-table)]
diff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.clj
index 3dde711bded..41c69d42f80 100644
--- a/src/metabase/search/appdb/index.clj
+++ b/src/metabase/search/appdb/index.clj
@@ -119,19 +119,26 @@
(map (comp keyword u/lower-case-en :table_name)
(search.db/orphan-index-table-names)))
-(defn- delete-obsolete-tables! []
- ;; Delete metadata around indexes that are no longer needed.
- (search-index-metadata/delete-obsolete! (search.spec/index-version-hash))
- ;; Drop any indexes that are no longer referenced.
- (let [dropped (volatile! [])]
- (doseq [table (orphan-indexes)]
- (try
- (search.db/drop-search-index-table! table)
- (vswap! dropped conj table)
- ;; Deletion could fail if it races with other instances
- (catch Exception e
- (log/warnf "Failed to drop stale index %s: %s" table (ex-message e)))))
- (log/infof "Dropped %d stale indexes: %s" (count @dropped) @dropped)))
+(defn delete-obsolete-tables!
+ "Drop index tables that are no longer needed. Best effort: failures are logged and never propagate. Does nothing
+ while mocking tables, where the pending table is tracked in an atom and has no metadata row to find it by."
+ []
+ (when-not *mocking-tables*
+ (try
+ ;; Delete metadata around indexes that are no longer needed.
+ (search-index-metadata/delete-obsolete! (search.spec/index-version-hash))
+ ;; Drop any indexes that are no longer referenced.
+ (let [dropped (volatile! [])]
+ (doseq [table (orphan-indexes)]
+ (try
+ (search.db/drop-search-index-table! table)
+ (vswap! dropped conj table)
+ ;; Deletion could fail if it races with other instances
+ (catch Exception e
+ (log/warnf "Failed to drop stale index %s: %s" table (ex-message e)))))
+ (log/infof "Dropped %d stale indexes: %s" (count @dropped) @dropped))
+ (catch Exception e
+ (log/warnf "Failed to clean up obsolete indexes: %s" (ex-message e))))))
(defn- ->db-type [t]
(get {:pk :int, :timestamp :timestamp-with-time-zone} t t))
diff --git a/test/metabase/search/appdb/index_test.clj b/test/metabase/search/appdb/index_test.clj
index ff42bf19dbb..dabf7ba8580 100644
--- a/test/metabase/search/appdb/index_test.clj
+++ b/test/metabase/search/appdb/index_test.clj
@@ -586,7 +586,7 @@
(is (= active-after (active-table-after period)))))
(finally
(t2/delete! :model/SearchIndexMetadata :version "auto-refresh-test")
- (#'search.index/delete-obsolete-tables!))))))
+ (search.index/delete-obsolete-tables!))))))
(deftest pending-table-expiry-test
(when (search/supports-index?)
@@ -618,7 +618,28 @@
(is (= pending-new (#'search.index/pending-table)))))
(finally
(t2/delete! :model/SearchIndexMetadata :version "pending-timeout-test")
- (#'search.index/delete-obsolete-tables!))))))
+ (search.index/delete-obsolete-tables!))))))
+
+(deftest failed-reindex-drops-orphaned-tables-test
+ (when (search/supports-index?)
+ (binding [search.spec/*testing-only-index-version-hash* "orphan-cleanup-test"]
+ (try
+ (reset! @#'search.index/next-sync-at nil)
+ (search.index/reset-index!)
+ (let [orphan (search.index/gen-table-name)]
+ (search.index/create-table! orphan)
+ (mt/with-dynamic-fn-redefs [search.ingestion/searchable-documents #(throw (ex-info "Simulated connection loss" {}))]
+ (mt/with-log-level [metabase.search.appdb.core :fatal]
+ (is (thrown-with-msg? Exception #"Simulated connection loss"
+ (search.engine/reindex! :search.engine/appdb {})))))
+ (testing "the orphan is dropped even though the reindex never reached activation"
+ (is (not (search.index/exists? orphan))))
+ (testing "the active table and the pending table left behind by the failed run are kept"
+ (is (search.index/exists? (search.index/active-table)))
+ (is (search.index/exists? (#'search.index/pending-table)))))
+ (finally
+ (t2/delete! :model/SearchIndexMetadata :version "orphan-cleanup-test")
+ (search.index/delete-obsolete-tables!))))))
(deftest strip-junk-chars-test
(let [strip @#'search.index/strip-junk-chars]
@@ -787,7 +808,7 @@
(is (= update-time (t/truncate-to (#'search.index/when-index-created) :millis))))))
(finally
(t2/delete! :model/SearchIndexMetadata :version "index-age-test")
- (#'search.index/delete-obsolete-tables!))))))
+ (search.index/delete-obsolete-tables!))))))
(deftest missing-index-table-does-not-abort-enclosing-transaction-test
(when (search/supports-index?)