The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Faster search index sync when loading snapshots (#71410)
Chris Truter40691f70d87f
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 75cf6a00e27..94f2850ae6c 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1856,7 +1856,8 @@metabase.search.ingestionmetabase.search.initmetabase.search.scoring- metabase.search.spec}+ metabase.search.spec+ metabase.search.task.search-index}:uses #{analyticsapiapp-dbdiff --git a/src/metabase/search/appdb/core.clj b/src/metabase/search/appdb/core.cljindex 899ba2c4dd6..665c4f7e047 100644--- a/src/metabase/search/appdb/core.clj+++ b/src/metabase/search/appdb/core.clj@@ -211,6 +211,9 @@(log/info "Populating index")(populate-index! (if created? :search/reindexing :search/updating)))))))+(defmethod search.engine/sync-from-restored-db! :search.engine/appdb [_]+ (search.index/sync-from-restored-db!))+(defmethod search.engine/reindex! :search.engine/appdb[_ {:keys [in-place?]}](trydiff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.cljindex 6a934ecfce9..c5af8a2ce04 100644--- a/src/metabase/search/appdb/index.clj+++ b/src/metabase/search/appdb/index.clj@@ -64,6 +64,13 @@(log/debugf "Sync tracking atoms: %s" indexes)(reset! *indexes* indexes))))+(defn sync-from-restored-db!+ "Re-sync tracking atoms with the current database state.+ Used after snapshot restore where the index tables are already present."+ []+ (reset! next-sync-at nil)+ (sync-tracking-atoms!))+;; This exists only to be mocked.(defn- now [] (System/nanoTime))diff --git a/src/metabase/search/core.clj b/src/metabase/search/core.cljindex 7f7518ea36a..7f678e795b8 100644--- a/src/metabase/search/core.clj+++ b/src/metabase/search/core.clj@@ -157,6 +157,14 @@(doseq [e (search.engine/active-engines)](search.engine/reset-tracking! e))))+(defn sync-from-restored-db!+ "Reconcile all search engine state with the current database.+ Use after snapshot restore instead of reindex! to avoid redundant work."+ []+ (when (supports-index?)+ (doseq [e (search.engine/active-engines)]+ (search.engine/sync-from-restored-db! e))))+(defn update!"Given a new or updated instance, put all the corresponding search entries if needed in the queue."[instance & [always?]]diff --git a/src/metabase/search/engine.clj b/src/metabase/search/engine.cljindex e42f4ab24a0..8911660df20 100644--- a/src/metabase/search/engine.clj+++ b/src/metabase/search/engine.clj@@ -68,6 +68,13 @@{:arglists '([engine])}identity)+(defmulti sync-from-restored-db!+ "Reconcile in-memory search state with what's currently in the database.+ Used after snapshot restore where the DB already contains valid index data.+ Engines that store their index in the appdb can skip reindexing."+ {:arglists '([engine])}+ identity)+(defn known-engines"List the possible search engines defined for this version, whether this instance supports them or not."[]diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.cljindex 3e1003104dd..8fdf979f732 100644--- a/src/metabase/search/ingestion.clj+++ b/src/metabase/search/ingestion.clj@@ -321,6 +321,34 @@(track-queue-size!)true)))))+(defn wait-for-idle!+ "Block until the ingestion queue has been drained and no more items arrive.+ Polls the queue and considers it idle when it has been empty for `settle-ms`+ (default 500ms). Gives up after `timeout-ms` (default 30s)."+ [& {:keys [settle-ms timeout-ms poll-ms]+ :or {settle-ms 500, timeout-ms 30000, poll-ms 50}}]+ (let [deadline-ns (+ (System/nanoTime) (* (long timeout-ms) 1000000))+ settle-ns (* (long settle-ms) 1000000)+ poll-ms (long poll-ms)+ empty-since (atom nil)]+ (loop []+ (let [now-ns (System/nanoTime)+ sz (.size queue)+ remaining-ns (- settle-ns (- now-ns (or @empty-since now-ns)))]+ (cond+ (>= now-ns deadline-ns)+ (log/warnf "wait-for-idle! timed out with %d items still in the queue" sz)++ (pos? sz)+ (do (reset! empty-since nil)+ (Thread/sleep poll-ms)+ (recur))++ (pos? remaining-ns)+ (do (when-not @empty-since (reset! empty-since now-ns))+ (Thread/sleep (long (max poll-ms (quot remaining-ns 1000000))))+ (recur)))))))+(defn start-listener!"Starts the ingestion listener on the queue"[]diff --git a/src/metabase/search/semantic/core.clj b/src/metabase/search/semantic/core.cljindex 4da203867a8..2b36a9fd4d8 100644--- a/src/metabase/search/semantic/core.clj+++ b/src/metabase/search/semantic/core.clj@@ -101,3 +101,9 @@(defmethod search.engine/reset-tracking! :search.engine/semantic [_](log/debug "reset-tracking! is not supported for semantic search engine")nil)++(defmethod search.engine/sync-from-restored-db! :search.engine/semantic [_]+ ;; Semantic search index lives outside the appdb snapshot.+ ;; For now, fall back to reindex (which is already a no-op for this engine).+ ;; TODO: If semantic embeddings are serialized alongside snapshots in the future, optimize this.+ (search.engine/reindex! :search.engine/semantic {}))diff --git a/src/metabase/search/task/search_index.clj b/src/metabase/search/task/search_index.cljindex c6826673097..bedbfda5697 100644--- a/src/metabase/search/task/search_index.clj+++ b/src/metabase/search/task/search_index.clj@@ -45,8 +45,25 @@(cluster-lock/with-cluster-lock cluster-lock-name(search/reindex! {:async? false})))+;; Atom holding a promise that is delivered when the background init thread finishes.+;; nil when no init has been started — [[wait-for-init!]] returns immediately in that case.+(defonce ^:private init-promise (atom nil))++(defn wait-for-init!+ "Block until the background search index initialization has completed.+ No-op if init has not been started (e.g. in unit tests)."+ []+ (some-> @init-promise deref))+(defmethod startup/def-startup-logic! ::SearchIndexInit [_]- (doto (Thread. ^Runnable init!) .start))+ (let [p (promise)]+ (reset! init-promise p)+ (doto (Thread. ^Runnable (fn []+ (try+ (init!)+ (finally+ (deliver p true)))))+ .start)))(defmethod task/init! ::SearchIndexReindex [_](let [job (jobs/builddiff --git a/src/metabase/testing_api/api.clj b/src/metabase/testing_api/api.cljindex f4c2de5c5e8..71c72fb4e41 100644--- a/src/metabase/testing_api/api.clj+++ b/src/metabase/testing_api/api.clj@@ -18,6 +18,7 @@[metabase.premium-features.core :refer [defenterprise]][metabase.search.core :as search][metabase.search.ingestion :as search.ingestion]+ [metabase.search.task.search-index :as task.search-index][metabase.util.date-2 :as u.date][metabase.util.files :as u.files][metabase.util.json :as json]@@ -60,6 +61,8 @@"Snapshot the database for testing purposes."[{snapshot-name :name} :- [:map[:name ms/NonBlankString]]]+ (task.search-index/wait-for-init!)+ (search.ingestion/wait-for-idle!)(save-snapshot! snapshot-name)nil)@@ -139,7 +142,7 @@(alter-var-root #'java-time.clock/*clock* (constantly nil))(.clear ^Queue @#'search.ingestion/queue)(restore-snapshot! snapshot-name)- (search/reindex! {:async? false})+ (search/sync-from-restored-db!)nil);; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers todiff --git a/test/metabase/testing_api/api_test.clj b/test/metabase/testing_api/api_test.cljindex 8643f414459..0204d65ecf3 100644--- a/test/metabase/testing_api/api_test.clj+++ b/test/metabase/testing_api/api_test.clj@@ -6,6 +6,8 @@[java-time.api :as t][java-time.clock][metabase.app-db.core :as mdb]+ [metabase.search.appdb.index :as search.index]+ [metabase.search.core :as search][metabase.test :as mt][metabase.testing-api.api :as testing][metabase.util :as u]@@ -58,6 +60,28 @@(alter-var-root #'java-time.clock/*clock* (constantly nil))(.delete (io/file (#'testing/snapshot-path-for-name snapshot-name)))))))))+(deftest restore-refreshes-search-index-test+ (when (and (= (mdb/db-type) :h2) (search/supports-index?))+ (testing "After restore, the search index tracking atoms should reflect the restored state"+ (let [snapshot-name (munge (u/qualified-name ::search-index-snapshot))]+ (try+ ;; Ensure a search index exists before snapshotting+ (mt/user-http-request :crowberto :post 200 "search/re-init")+ (is (some? (search.index/active-table))+ "Precondition: search index should exist before snapshot")+ ;; Snapshot with a valid search index in place+ (mt/user-http-request :rasta :post 204 (format "testing/snapshot/%s" snapshot-name))+ ;; Clear the tracking atoms so we can verify they get restored+ (reset! @#'search.index/*indexes* {:active nil, :pending nil})+ (is (nil? (search.index/active-table))+ "Precondition: tracking atoms should be cleared before restore")+ ;; Restore should call sync-from-restored-db! and refresh the atoms+ (mt/user-http-request :rasta :post 204 (format "testing/restore/%s" snapshot-name))+ (is (some? (search.index/active-table))+ "After restore, the active search index table should be tracked")+ (finally+ (.delete (io/file (#'testing/snapshot-path-for-name snapshot-name)))))))))+(deftest snapshot-restore-works-with-views;; workaround for https://github.com/h2database/h2database/issues/3942, see comment in;; `restore-app-db-from-snapshot!` for more details