Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Wait for search indexing to finish in E2E tests (#84441)

Andersonf2ba6f42a370
diff --git a/src/metabase/search/appdb/core.clj b/src/metabase/search/appdb/core.clj
index 0050c34cc49..a341608cba6 100644
--- a/src/metabase/search/appdb/core.clj
+++ b/src/metabase/search/appdb/core.clj
@@ -28,8 +28,7 @@
[metabase.util.log :as log]
[methodical.core :as methodical])
(:import
- (java.time OffsetDateTime)
- (java.util Queue)))
+ (java.time OffsetDateTime)))
;; Register the multimethods for each specialization
(comment
@@ -115,13 +114,8 @@
(tracing/with-span :search "search.appdb.query" {:search/query-length (count search-string)}
(try
(when (setting/string->boolean (:mb-experimental-search-block-on-queue env/env))
- ;; wait for a bit for the queue to be drained
- (let [pending-updates #(.size ^Queue @#'search.ingestion/queue)]
- (when-not (u/poll {:thunk pending-updates
- :done? zero?
- :timeout-ms 2000
- :interval-ms 100})
- (log/warn "Returning search results even though they may be stale. Queue size:" (pending-updates)))))
+ (when-not (search.ingestion/await-pending-updates! 2000)
+ (log/warn "Returning search results before pending index updates finished.")))
(let [weights (search.config/weights search-ctx)
percentiles (view-count-percentiles search.config/view-count-scaling-percentile)
scorer-keys (keys (search.scoring/scorers search-ctx percentiles))]
diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.clj
index 8e51d4f4cb1..fa91dfd859c 100644
--- a/src/metabase/search/ingestion.clj
+++ b/src/metabase/search/ingestion.clj
@@ -13,6 +13,7 @@
[metabase.util :as u]
[metabase.util.json :as json]
[metabase.util.log :as log]
+ [metabase.util.malli :as mu]
[metabase.util.performance :as perf]
[metabase.util.queue :as queue]
[toucan2.realize :as t2.realize])
@@ -24,6 +25,8 @@
;; Currently we use a single queue, even if multiple engines are enabled, but may want to revisit this.
(defonce ^:private ^DelayQueue queue (queue/delay-queue))
+(defonce ^:private pending-updates (atom 0))
+
;; Perhaps this config move up somewhere more visible? Conversely, we may want to specialize it per engine.
(def ^:private message-delay-ms
@@ -291,6 +294,21 @@
(defn- index-worker-exists? []
(queue/listener-exists? listener-name))
+(mu/defn await-pending-updates! :- :boolean
+ "Wait up to `timeout-ms` for queued and running index updates to finish. Return whether none remain."
+ [timeout-ms :- nat-int?]
+ (boolean (u/poll {:thunk #(deref pending-updates)
+ :done? zero?
+ :timeout-ms timeout-ms
+ :interval-ms 100})))
+
+(mu/defn clear-queue! :- :nil
+ "Discard queued updates without interrupting updates already being indexed."
+ []
+ (doseq [message (.toArray queue)]
+ (when (.remove queue message)
+ (swap! pending-updates dec))))
+
(defn ingest-maybe-async!
"Update or create any search index entries related to the given updates.
Will be async if the worker exists, otherwise it will be done synchronously on the calling thread.
@@ -308,14 +326,14 @@
(fn []
(doseq [update updates]
(log/trace "Queuing update" update)
+ (swap! pending-updates inc)
(queue/put-with-delay! queue message-delay-ms update))
(track-queue-size!)))
true)))))
(defn wait-for-idle!
- "Block until the ingestion queue has been drained and no more items arrive.
- Polls the queue and considers it idle when it has been empty for `settle-ms`
- (default 500ms). Gives up after `timeout-ms` (default 30s)."
+ "Block until no index updates are queued or being processed for `settle-ms` (default 500ms).
+ Log a warning and return after `timeout-ms` (default 30s)."
[& {:keys [settle-ms timeout-ms poll-ms]
:or {settle-ms 500, timeout-ms 30000, poll-ms 50}}]
(let [deadline-ns (+ (System/nanoTime) (* (long timeout-ms) 1000000))
@@ -323,14 +341,14 @@
poll-ms (long poll-ms)
empty-since (atom nil)]
(loop []
- (let [now-ns (System/nanoTime)
- sz (.size queue)
- remaining-ns (- settle-ns (- now-ns (or @empty-since now-ns)))]
+ (let [now-ns (System/nanoTime)
+ pending-count @pending-updates
+ remaining-ns (- settle-ns (- now-ns (or @empty-since now-ns)))]
(cond
(>= now-ns deadline-ns)
- (log/warnf "wait-for-idle! timed out with %d items still in the queue" sz)
+ (log/warnf "wait-for-idle! timed out with %d pending index updates" pending-count)
- (pos? sz)
+ (pos? pending-count)
(do (reset! empty-since nil)
(Thread/sleep poll-ms)
(recur))
@@ -340,11 +358,17 @@
(Thread/sleep (long (max poll-ms (quot remaining-ns 1000000))))
(recur)))))))
+(defn- ingest-batch! [updates]
+ (try
+ (bulk-ingest! updates)
+ (finally
+ (swap! pending-updates - (count updates)))))
+
(defn start-listener!
"Starts the ingestion listener on the queue"
[]
(when (seq (search.engine/active-engines))
- (queue/listen! listener-name queue bulk-ingest!
+ (queue/listen! listener-name queue ingest-batch!
{:success-handler (fn [_result _duration _]
(track-queue-size!))
:err-handler (fn [err _]
diff --git a/src/metabase/testing_api/api.clj b/src/metabase/testing_api/api.clj
index 4124e7b8862..627424f08be 100644
--- a/src/metabase/testing_api/api.clj
+++ b/src/metabase/testing_api/api.clj
@@ -35,7 +35,6 @@
[metabase.util.malli.schema :as ms])
(:import
(com.mchange.v2.c3p0 PoolBackedDataSource)
- (java.util Queue)
(java.util.concurrent.locks ReentrantReadWriteLock)))
(set! *warn-on-reflection* true)
@@ -163,7 +162,9 @@
[:name ms/NonBlankString]]]
;; reset the system clock, in case `/set-time` was called without cleanup
(alter-var-root #'java-time.clock/*clock* (constantly nil))
- (.clear ^Queue @#'search.ingestion/queue)
+ (search.ingestion/clear-queue!)
+ (api/check (search.ingestion/await-pending-updates! 30000)
+ [500 "Timed out waiting for search indexing before restoring a snapshot."])
(restore-snapshot! snapshot-name)
(search/sync-from-restored-db!)
nil)
diff --git a/test/metabase/search/ingestion_test.clj b/test/metabase/search/ingestion_test.clj
index 8e793b628e0..6975e844714 100644
--- a/test/metabase/search/ingestion_test.clj
+++ b/test/metabase/search/ingestion_test.clj
@@ -269,10 +269,74 @@
"Run `f` with a fresh ingestion queue swapped in, and return the updates that end up on it."
[f]
(let [test-queue (queue/delay-queue)]
- (with-redefs [search.ingestion/queue test-queue]
+ (with-redefs [search.ingestion/queue test-queue
+ search.ingestion/pending-updates (atom 0)]
(f test-queue))
(mapv :value test-queue)))
+(deftest await-pending-updates-waits-for-in-flight-batch-test
+ (let [test-queue (queue/delay-queue)
+ listener-name (str (random-uuid))
+ processing (promise)
+ release-batch (promise)
+ indexed (atom [])
+ update ["card" [:= :this.id 1]]
+ later-update ["card" [:= :this.id 2]]]
+ (with-redefs [search.ingestion/queue test-queue
+ search.ingestion/pending-updates (atom 0)
+ search.ingestion/listener-name listener-name]
+ (mt/with-dynamic-fn-redefs
+ [search.engine/active-engines (constantly [:search.engine/appdb])
+ search.ingestion/bulk-ingest! (fn [updates]
+ (deliver processing true)
+ (deref release-batch 5000 ::timeout)
+ (swap! indexed into updates))]
+ (search.ingestion/start-listener!)
+ (try
+ (search.ingestion/ingest-maybe-async! [update] false)
+ (is (true? (deref processing 2000 ::timeout)))
+ (is (empty? test-queue))
+ (is (false? (search.ingestion/await-pending-updates! 10))
+ "An empty queue does not mean the worker finished indexing")
+ (mt/with-log-messages-for-level [messages [metabase.search.ingestion :warn]]
+ (search.ingestion/wait-for-idle! :settle-ms 0 :timeout-ms 10 :poll-ms 1)
+ (is (=? [{:message #"wait-for-idle! timed out.*"}] (messages))
+ "The running batch prevents the queue from being idle"))
+ (testing "Discarding queued work preserves the running batch"
+ (search.ingestion/ingest-maybe-async! [later-update later-update] false)
+ (search.ingestion/clear-queue!)
+ (is (empty? test-queue))
+ (is (false? (search.ingestion/await-pending-updates! 10))))
+ (deliver release-batch true)
+ (is (true? (search.ingestion/await-pending-updates! 2000)))
+ (is (= [update] @indexed))
+ (testing "New updates remain tracked after clearing the queue"
+ (search.ingestion/ingest-maybe-async! [later-update] false)
+ (is (true? (search.ingestion/await-pending-updates! 2000)))
+ (is (= [update later-update] @indexed)))
+ (finally
+ (deliver release-batch true)
+ (queue/stop-listening! listener-name)))))))
+
+(deftest await-pending-updates-after-ingestion-error-test
+ (let [listener-name (str (random-uuid))
+ attempted (promise)]
+ (with-redefs [search.ingestion/queue (queue/delay-queue)
+ search.ingestion/pending-updates (atom 0)
+ search.ingestion/listener-name listener-name]
+ (mt/with-dynamic-fn-redefs
+ [search.engine/active-engines (constantly [:search.engine/appdb])
+ search.ingestion/bulk-ingest! (fn [_updates]
+ (deliver attempted true)
+ (throw (ex-info "Indexing failed" {})))]
+ (search.ingestion/start-listener!)
+ (try
+ (search.ingestion/ingest-maybe-async! [["card" [:= :this.id 1]]] false)
+ (is (true? (search.ingestion/await-pending-updates! 2000)))
+ (is (true? (deref attempted 2000 ::timeout)))
+ (finally
+ (queue/stop-listening! listener-name)))))))
+
(deftest ingest-maybe-async!-queues-after-commit-test
;; The worker reads each updated row back from the app DB. If an update reaches the queue before its transaction
;; commits, and the transaction takes longer than the queue delay, the worker finds no row and the entity is
@@ -284,7 +348,8 @@
(t2/with-transaction [_conn]
(search.ingestion/ingest-maybe-async! [update] false)
(testing "nothing is queued while the transaction is still open"
- (is (empty? test-queue))))))]
+ (is (empty? test-queue))
+ (is (true? (search.ingestion/await-pending-updates! 0)))))))]
(is (= [update] queued))))
(testing "nothing is queued when the transaction rolls back"
(let [queued (queued-updates!
@@ -292,7 +357,8 @@
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"rollback"
(t2/with-transaction [_conn]
(search.ingestion/ingest-maybe-async! [update] false)
- (throw (ex-info "rollback" {})))))))]
+ (throw (ex-info "rollback" {})))))
+ (is (true? (search.ingestion/await-pending-updates! 0)))))]
(is (= [] queued))))
(testing "outside a transaction the update is queued immediately"
(is (= [update]
diff --git a/test/metabase/testing_api/api_test.clj b/test/metabase/testing_api/api_test.clj
index 1cc8ef56a9f..9c36af3f257 100644
--- a/test/metabase/testing_api/api_test.clj
+++ b/test/metabase/testing_api/api_test.clj
@@ -9,11 +9,13 @@
[metabase.llm.settings :as llm.settings]
[metabase.search.appdb.index :as search.index]
[metabase.search.core :as search]
+ [metabase.search.ingestion :as search.ingestion]
[metabase.session.api :as session.api]
[metabase.test :as mt]
[metabase.testing-api.api :as testing]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
+ [metabase.util.queue :as queue]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -33,9 +35,42 @@
(deftest restore-test
(when (= (mdb/db-type) :h2)
- (testing "Should throw Exception if file does not exist"
- (is (= "Not found."
- (mt/user-http-request :rasta :post 404 (format "testing/restore/%s" (mt/random-name))))))))
+ (let [path (format "testing/restore/%s" (mt/random-name))]
+ (testing "Should throw Exception if file does not exist"
+ (is (= "Not found."
+ (mt/user-http-request :rasta :post 404 path))))
+ (testing "An indexing timeout prevents reaching the missing snapshot"
+ (mt/with-dynamic-fn-redefs [search.ingestion/await-pending-updates! (constantly false)]
+ (is (=? {:message "Timed out waiting for search indexing before restoring a snapshot."}
+ (mt/user-http-request :rasta :post 500 path))))))))
+
+(deftest restore-waits-for-in-flight-indexing-test
+ (mt/initialize-if-needed! :db :test-users :web-server)
+ (let [test-queue (queue/delay-queue)
+ pending (atom 2)
+ queue-cleared (promise)
+ restored (promise)]
+ (queue/put-with-delay! test-queue 60000 ["card" [:= :this.id 1]])
+ (add-watch pending ::queue-cleared
+ (fn [_key _ref _old new]
+ (when (= new 1)
+ (deliver queue-cleared true))))
+ (with-redefs [search.ingestion/queue test-queue
+ search.ingestion/pending-updates pending]
+ (mt/with-dynamic-fn-redefs [testing/restore-snapshot! (fn [_name] (deliver restored @pending))
+ search/sync-from-restored-db! (constantly nil)]
+ (let [request (future (mt/user-http-request :rasta :post 204 "testing/restore/in-flight-indexing"))]
+ (try
+ (is (true? (deref queue-cleared 2000 ::timeout)))
+ (is (empty? test-queue))
+ (is (= ::waiting (deref restored 100 ::waiting))
+ "Restoration cannot start while the remaining batch is still being indexed")
+ (reset! pending 0)
+ (is (nil? (deref request 2000 ::timeout)))
+ (is (= 0 (deref restored 2000 ::timeout)))
+ (finally
+ (reset! pending 0)
+ (future-cancel request))))))))
(deftest e2e-test
(when (= (mdb/db-type) :h2)