Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Faster search index sync when loading snapshots (#71410)

Chris Truter40691f70d87f
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 75cf6a00e27..94f2850ae6c 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1856,7 +1856,8 @@
metabase.search.ingestion
metabase.search.init
metabase.search.scoring
- metabase.search.spec}
+ metabase.search.spec
+ metabase.search.task.search-index}
:uses #{analytics
api
app-db
diff --git a/src/metabase/search/appdb/core.clj b/src/metabase/search/appdb/core.clj
index 899ba2c4dd6..665c4f7e047 100644
--- a/src/metabase/search/appdb/core.clj
+++ b/src/metabase/search/appdb/core.clj
@@ -211,6 +211,9 @@
(log/info "Populating index")
(populate-index! (if created? :search/reindexing :search/updating)))))))
+(defmethod search.engine/sync-from-restored-db! :search.engine/appdb [_]
+ (search.index/sync-from-restored-db!))
+
(defmethod search.engine/reindex! :search.engine/appdb
[_ {:keys [in-place?]}]
(try
diff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.clj
index 6a934ecfce9..c5af8a2ce04 100644
--- a/src/metabase/search/appdb/index.clj
+++ b/src/metabase/search/appdb/index.clj
@@ -64,6 +64,13 @@
(log/debugf "Sync tracking atoms: %s" indexes)
(reset! *indexes* indexes))))
+(defn sync-from-restored-db!
+ "Re-sync tracking atoms with the current database state.
+ Used after snapshot restore where the index tables are already present."
+ []
+ (reset! next-sync-at nil)
+ (sync-tracking-atoms!))
+
;; This exists only to be mocked.
(defn- now [] (System/nanoTime))
diff --git a/src/metabase/search/core.clj b/src/metabase/search/core.clj
index 7f7518ea36a..7f678e795b8 100644
--- a/src/metabase/search/core.clj
+++ b/src/metabase/search/core.clj
@@ -157,6 +157,14 @@
(doseq [e (search.engine/active-engines)]
(search.engine/reset-tracking! e))))
+(defn sync-from-restored-db!
+ "Reconcile all search engine state with the current database.
+ Use after snapshot restore instead of reindex! to avoid redundant work."
+ []
+ (when (supports-index?)
+ (doseq [e (search.engine/active-engines)]
+ (search.engine/sync-from-restored-db! e))))
+
(defn update!
"Given a new or updated instance, put all the corresponding search entries if needed in the queue."
[instance & [always?]]
diff --git a/src/metabase/search/engine.clj b/src/metabase/search/engine.clj
index e42f4ab24a0..8911660df20 100644
--- a/src/metabase/search/engine.clj
+++ b/src/metabase/search/engine.clj
@@ -68,6 +68,13 @@
{:arglists '([engine])}
identity)
+(defmulti sync-from-restored-db!
+ "Reconcile in-memory search state with what's currently in the database.
+ Used after snapshot restore where the DB already contains valid index data.
+ Engines that store their index in the appdb can skip reindexing."
+ {:arglists '([engine])}
+ identity)
+
(defn known-engines
"List the possible search engines defined for this version, whether this instance supports them or not."
[]
diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.clj
index 3e1003104dd..8fdf979f732 100644
--- a/src/metabase/search/ingestion.clj
+++ b/src/metabase/search/ingestion.clj
@@ -321,6 +321,34 @@
(track-queue-size!)
true)))))
+(defn wait-for-idle!
+ "Block until the ingestion queue has been drained and no more items arrive.
+ Polls the queue and considers it idle when it has been empty for `settle-ms`
+ (default 500ms). Gives up after `timeout-ms` (default 30s)."
+ [& {:keys [settle-ms timeout-ms poll-ms]
+ :or {settle-ms 500, timeout-ms 30000, poll-ms 50}}]
+ (let [deadline-ns (+ (System/nanoTime) (* (long timeout-ms) 1000000))
+ settle-ns (* (long settle-ms) 1000000)
+ poll-ms (long poll-ms)
+ empty-since (atom nil)]
+ (loop []
+ (let [now-ns (System/nanoTime)
+ sz (.size queue)
+ remaining-ns (- settle-ns (- now-ns (or @empty-since now-ns)))]
+ (cond
+ (>= now-ns deadline-ns)
+ (log/warnf "wait-for-idle! timed out with %d items still in the queue" sz)
+
+ (pos? sz)
+ (do (reset! empty-since nil)
+ (Thread/sleep poll-ms)
+ (recur))
+
+ (pos? remaining-ns)
+ (do (when-not @empty-since (reset! empty-since now-ns))
+ (Thread/sleep (long (max poll-ms (quot remaining-ns 1000000))))
+ (recur)))))))
+
(defn start-listener!
"Starts the ingestion listener on the queue"
[]
diff --git a/src/metabase/search/semantic/core.clj b/src/metabase/search/semantic/core.clj
index 4da203867a8..2b36a9fd4d8 100644
--- a/src/metabase/search/semantic/core.clj
+++ b/src/metabase/search/semantic/core.clj
@@ -101,3 +101,9 @@
(defmethod search.engine/reset-tracking! :search.engine/semantic [_]
(log/debug "reset-tracking! is not supported for semantic search engine")
nil)
+
+(defmethod search.engine/sync-from-restored-db! :search.engine/semantic [_]
+ ;; Semantic search index lives outside the appdb snapshot.
+ ;; For now, fall back to reindex (which is already a no-op for this engine).
+ ;; TODO: If semantic embeddings are serialized alongside snapshots in the future, optimize this.
+ (search.engine/reindex! :search.engine/semantic {}))
diff --git a/src/metabase/search/task/search_index.clj b/src/metabase/search/task/search_index.clj
index c6826673097..bedbfda5697 100644
--- a/src/metabase/search/task/search_index.clj
+++ b/src/metabase/search/task/search_index.clj
@@ -45,8 +45,25 @@
(cluster-lock/with-cluster-lock cluster-lock-name
(search/reindex! {:async? false})))
+;; Atom holding a promise that is delivered when the background init thread finishes.
+;; nil when no init has been started — [[wait-for-init!]] returns immediately in that case.
+(defonce ^:private init-promise (atom nil))
+
+(defn wait-for-init!
+ "Block until the background search index initialization has completed.
+ No-op if init has not been started (e.g. in unit tests)."
+ []
+ (some-> @init-promise deref))
+
(defmethod startup/def-startup-logic! ::SearchIndexInit [_]
- (doto (Thread. ^Runnable init!) .start))
+ (let [p (promise)]
+ (reset! init-promise p)
+ (doto (Thread. ^Runnable (fn []
+ (try
+ (init!)
+ (finally
+ (deliver p true)))))
+ .start)))
(defmethod task/init! ::SearchIndexReindex [_]
(let [job (jobs/build
diff --git a/src/metabase/testing_api/api.clj b/src/metabase/testing_api/api.clj
index f4c2de5c5e8..71c72fb4e41 100644
--- a/src/metabase/testing_api/api.clj
+++ b/src/metabase/testing_api/api.clj
@@ -18,6 +18,7 @@
[metabase.premium-features.core :refer [defenterprise]]
[metabase.search.core :as search]
[metabase.search.ingestion :as search.ingestion]
+ [metabase.search.task.search-index :as task.search-index]
[metabase.util.date-2 :as u.date]
[metabase.util.files :as u.files]
[metabase.util.json :as json]
@@ -60,6 +61,8 @@
"Snapshot the database for testing purposes."
[{snapshot-name :name} :- [:map
[:name ms/NonBlankString]]]
+ (task.search-index/wait-for-init!)
+ (search.ingestion/wait-for-idle!)
(save-snapshot! snapshot-name)
nil)
@@ -139,7 +142,7 @@
(alter-var-root #'java-time.clock/*clock* (constantly nil))
(.clear ^Queue @#'search.ingestion/queue)
(restore-snapshot! snapshot-name)
- (search/reindex! {:async? false})
+ (search/sync-from-restored-db!)
nil)
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
diff --git a/test/metabase/testing_api/api_test.clj b/test/metabase/testing_api/api_test.clj
index 8643f414459..0204d65ecf3 100644
--- a/test/metabase/testing_api/api_test.clj
+++ b/test/metabase/testing_api/api_test.clj
@@ -6,6 +6,8 @@
[java-time.api :as t]
[java-time.clock]
[metabase.app-db.core :as mdb]
+ [metabase.search.appdb.index :as search.index]
+ [metabase.search.core :as search]
[metabase.test :as mt]
[metabase.testing-api.api :as testing]
[metabase.util :as u]
@@ -58,6 +60,28 @@
(alter-var-root #'java-time.clock/*clock* (constantly nil))
(.delete (io/file (#'testing/snapshot-path-for-name snapshot-name)))))))))
+(deftest restore-refreshes-search-index-test
+ (when (and (= (mdb/db-type) :h2) (search/supports-index?))
+ (testing "After restore, the search index tracking atoms should reflect the restored state"
+ (let [snapshot-name (munge (u/qualified-name ::search-index-snapshot))]
+ (try
+ ;; Ensure a search index exists before snapshotting
+ (mt/user-http-request :crowberto :post 200 "search/re-init")
+ (is (some? (search.index/active-table))
+ "Precondition: search index should exist before snapshot")
+ ;; Snapshot with a valid search index in place
+ (mt/user-http-request :rasta :post 204 (format "testing/snapshot/%s" snapshot-name))
+ ;; Clear the tracking atoms so we can verify they get restored
+ (reset! @#'search.index/*indexes* {:active nil, :pending nil})
+ (is (nil? (search.index/active-table))
+ "Precondition: tracking atoms should be cleared before restore")
+ ;; Restore should call sync-from-restored-db! and refresh the atoms
+ (mt/user-http-request :rasta :post 204 (format "testing/restore/%s" snapshot-name))
+ (is (some? (search.index/active-table))
+ "After restore, the active search index table should be tracked")
+ (finally
+ (.delete (io/file (#'testing/snapshot-path-for-name snapshot-name)))))))))
+
(deftest snapshot-restore-works-with-views
;; workaround for https://github.com/h2database/h2database/issues/3942, see comment in
;; `restore-app-db-from-snapshot!` for more details