The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Wait for search indexing to finish in E2E tests (#84441)
Andersonf2ba6f42a370
diff --git a/src/metabase/search/appdb/core.clj b/src/metabase/search/appdb/core.cljindex 0050c34cc49..a341608cba6 100644--- a/src/metabase/search/appdb/core.clj+++ b/src/metabase/search/appdb/core.clj@@ -28,8 +28,7 @@[metabase.util.log :as log][methodical.core :as methodical])(:import- (java.time OffsetDateTime)- (java.util Queue)))+ (java.time OffsetDateTime)));; Register the multimethods for each specialization(comment@@ -115,13 +114,8 @@(tracing/with-span :search "search.appdb.query" {:search/query-length (count search-string)}(try(when (setting/string->boolean (:mb-experimental-search-block-on-queue env/env))- ;; wait for a bit for the queue to be drained- (let [pending-updates #(.size ^Queue @#'search.ingestion/queue)]- (when-not (u/poll {:thunk pending-updates- :done? zero?- :timeout-ms 2000- :interval-ms 100})- (log/warn "Returning search results even though they may be stale. Queue size:" (pending-updates)))))+ (when-not (search.ingestion/await-pending-updates! 2000)+ (log/warn "Returning search results before pending index updates finished.")))(let [weights (search.config/weights search-ctx)percentiles (view-count-percentiles search.config/view-count-scaling-percentile)scorer-keys (keys (search.scoring/scorers search-ctx percentiles))]diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.cljindex 8e51d4f4cb1..fa91dfd859c 100644--- a/src/metabase/search/ingestion.clj+++ b/src/metabase/search/ingestion.clj@@ -13,6 +13,7 @@[metabase.util :as u][metabase.util.json :as json][metabase.util.log :as log]+ [metabase.util.malli :as mu][metabase.util.performance :as perf][metabase.util.queue :as queue][toucan2.realize :as t2.realize])@@ -24,6 +25,8 @@;; Currently we use a single queue, even if multiple engines are enabled, but may want to revisit this.(defonce ^:private ^DelayQueue queue (queue/delay-queue))+(defonce ^:private pending-updates (atom 0))+;; Perhaps this config move up somewhere more visible? Conversely, we may want to specialize it per engine.(def ^:private message-delay-ms@@ -291,6 +294,21 @@(defn- index-worker-exists? [](queue/listener-exists? listener-name))+(mu/defn await-pending-updates! :- :boolean+ "Wait up to `timeout-ms` for queued and running index updates to finish. Return whether none remain."+ [timeout-ms :- nat-int?]+ (boolean (u/poll {:thunk #(deref pending-updates)+ :done? zero?+ :timeout-ms timeout-ms+ :interval-ms 100})))++(mu/defn clear-queue! :- :nil+ "Discard queued updates without interrupting updates already being indexed."+ []+ (doseq [message (.toArray queue)]+ (when (.remove queue message)+ (swap! pending-updates dec))))+(defn ingest-maybe-async!"Update or create any search index entries related to the given updates.Will be async if the worker exists, otherwise it will be done synchronously on the calling thread.@@ -308,14 +326,14 @@(fn [](doseq [update updates](log/trace "Queuing update" update)+ (swap! pending-updates inc)(queue/put-with-delay! queue message-delay-ms update))(track-queue-size!)))true)))))(defn wait-for-idle!- "Block until the ingestion queue has been drained and no more items arrive.- Polls the queue and considers it idle when it has been empty for `settle-ms`- (default 500ms). Gives up after `timeout-ms` (default 30s)."+ "Block until no index updates are queued or being processed for `settle-ms` (default 500ms).+ Log a warning and return after `timeout-ms` (default 30s)."[& {:keys [settle-ms timeout-ms poll-ms]:or {settle-ms 500, timeout-ms 30000, poll-ms 50}}](let [deadline-ns (+ (System/nanoTime) (* (long timeout-ms) 1000000))@@ -323,14 +341,14 @@poll-ms (long poll-ms)empty-since (atom nil)](loop []- (let [now-ns (System/nanoTime)- sz (.size queue)- remaining-ns (- settle-ns (- now-ns (or @empty-since now-ns)))]+ (let [now-ns (System/nanoTime)+ pending-count @pending-updates+ remaining-ns (- settle-ns (- now-ns (or @empty-since now-ns)))](cond(>= now-ns deadline-ns)- (log/warnf "wait-for-idle! timed out with %d items still in the queue" sz)+ (log/warnf "wait-for-idle! timed out with %d pending index updates" pending-count)- (pos? sz)+ (pos? pending-count)(do (reset! empty-since nil)(Thread/sleep poll-ms)(recur))@@ -340,11 +358,17 @@(Thread/sleep (long (max poll-ms (quot remaining-ns 1000000))))(recur)))))))+(defn- ingest-batch! [updates]+ (try+ (bulk-ingest! updates)+ (finally+ (swap! pending-updates - (count updates)))))+(defn start-listener!"Starts the ingestion listener on the queue"[](when (seq (search.engine/active-engines))- (queue/listen! listener-name queue bulk-ingest!+ (queue/listen! listener-name queue ingest-batch!{:success-handler (fn [_result _duration _](track-queue-size!)):err-handler (fn [err _]diff --git a/src/metabase/testing_api/api.clj b/src/metabase/testing_api/api.cljindex 4124e7b8862..627424f08be 100644--- a/src/metabase/testing_api/api.clj+++ b/src/metabase/testing_api/api.clj@@ -35,7 +35,6 @@[metabase.util.malli.schema :as ms])(:import(com.mchange.v2.c3p0 PoolBackedDataSource)- (java.util Queue)(java.util.concurrent.locks ReentrantReadWriteLock)))(set! *warn-on-reflection* true)@@ -163,7 +162,9 @@[:name ms/NonBlankString]]];; reset the system clock, in case `/set-time` was called without cleanup(alter-var-root #'java-time.clock/*clock* (constantly nil))- (.clear ^Queue @#'search.ingestion/queue)+ (search.ingestion/clear-queue!)+ (api/check (search.ingestion/await-pending-updates! 30000)+ [500 "Timed out waiting for search indexing before restoring a snapshot."])(restore-snapshot! snapshot-name)(search/sync-from-restored-db!)nil)diff --git a/test/metabase/search/ingestion_test.clj b/test/metabase/search/ingestion_test.cljindex 8e793b628e0..6975e844714 100644--- a/test/metabase/search/ingestion_test.clj+++ b/test/metabase/search/ingestion_test.clj@@ -269,10 +269,74 @@"Run `f` with a fresh ingestion queue swapped in, and return the updates that end up on it."[f](let [test-queue (queue/delay-queue)]- (with-redefs [search.ingestion/queue test-queue]+ (with-redefs [search.ingestion/queue test-queue+ search.ingestion/pending-updates (atom 0)](f test-queue))(mapv :value test-queue)))+(deftest await-pending-updates-waits-for-in-flight-batch-test+ (let [test-queue (queue/delay-queue)+ listener-name (str (random-uuid))+ processing (promise)+ release-batch (promise)+ indexed (atom [])+ update ["card" [:= :this.id 1]]+ later-update ["card" [:= :this.id 2]]]+ (with-redefs [search.ingestion/queue test-queue+ search.ingestion/pending-updates (atom 0)+ search.ingestion/listener-name listener-name]+ (mt/with-dynamic-fn-redefs+ [search.engine/active-engines (constantly [:search.engine/appdb])+ search.ingestion/bulk-ingest! (fn [updates]+ (deliver processing true)+ (deref release-batch 5000 ::timeout)+ (swap! indexed into updates))]+ (search.ingestion/start-listener!)+ (try+ (search.ingestion/ingest-maybe-async! [update] false)+ (is (true? (deref processing 2000 ::timeout)))+ (is (empty? test-queue))+ (is (false? (search.ingestion/await-pending-updates! 10))+ "An empty queue does not mean the worker finished indexing")+ (mt/with-log-messages-for-level [messages [metabase.search.ingestion :warn]]+ (search.ingestion/wait-for-idle! :settle-ms 0 :timeout-ms 10 :poll-ms 1)+ (is (=? [{:message #"wait-for-idle! timed out.*"}] (messages))+ "The running batch prevents the queue from being idle"))+ (testing "Discarding queued work preserves the running batch"+ (search.ingestion/ingest-maybe-async! [later-update later-update] false)+ (search.ingestion/clear-queue!)+ (is (empty? test-queue))+ (is (false? (search.ingestion/await-pending-updates! 10))))+ (deliver release-batch true)+ (is (true? (search.ingestion/await-pending-updates! 2000)))+ (is (= [update] @indexed))+ (testing "New updates remain tracked after clearing the queue"+ (search.ingestion/ingest-maybe-async! [later-update] false)+ (is (true? (search.ingestion/await-pending-updates! 2000)))+ (is (= [update later-update] @indexed)))+ (finally+ (deliver release-batch true)+ (queue/stop-listening! listener-name)))))))++(deftest await-pending-updates-after-ingestion-error-test+ (let [listener-name (str (random-uuid))+ attempted (promise)]+ (with-redefs [search.ingestion/queue (queue/delay-queue)+ search.ingestion/pending-updates (atom 0)+ search.ingestion/listener-name listener-name]+ (mt/with-dynamic-fn-redefs+ [search.engine/active-engines (constantly [:search.engine/appdb])+ search.ingestion/bulk-ingest! (fn [_updates]+ (deliver attempted true)+ (throw (ex-info "Indexing failed" {})))]+ (search.ingestion/start-listener!)+ (try+ (search.ingestion/ingest-maybe-async! [["card" [:= :this.id 1]]] false)+ (is (true? (search.ingestion/await-pending-updates! 2000)))+ (is (true? (deref attempted 2000 ::timeout)))+ (finally+ (queue/stop-listening! listener-name)))))))+(deftest ingest-maybe-async!-queues-after-commit-test;; The worker reads each updated row back from the app DB. If an update reaches the queue before its transaction;; commits, and the transaction takes longer than the queue delay, the worker finds no row and the entity is@@ -284,7 +348,8 @@(t2/with-transaction [_conn](search.ingestion/ingest-maybe-async! [update] false)(testing "nothing is queued while the transaction is still open"- (is (empty? test-queue))))))]+ (is (empty? test-queue))+ (is (true? (search.ingestion/await-pending-updates! 0)))))))](is (= [update] queued))))(testing "nothing is queued when the transaction rolls back"(let [queued (queued-updates!@@ -292,7 +357,8 @@(is (thrown-with-msg? clojure.lang.ExceptionInfo #"rollback"(t2/with-transaction [_conn](search.ingestion/ingest-maybe-async! [update] false)- (throw (ex-info "rollback" {})))))))]+ (throw (ex-info "rollback" {})))))+ (is (true? (search.ingestion/await-pending-updates! 0)))))](is (= [] queued))))(testing "outside a transaction the update is queued immediately"(is (= [update]diff --git a/test/metabase/testing_api/api_test.clj b/test/metabase/testing_api/api_test.cljindex 1cc8ef56a9f..9c36af3f257 100644--- a/test/metabase/testing_api/api_test.clj+++ b/test/metabase/testing_api/api_test.clj@@ -9,11 +9,13 @@[metabase.llm.settings :as llm.settings][metabase.search.appdb.index :as search.index][metabase.search.core :as search]+ [metabase.search.ingestion :as search.ingestion][metabase.session.api :as session.api][metabase.test :as mt][metabase.testing-api.api :as testing][metabase.util :as u][metabase.util.date-2 :as u.date]+ [metabase.util.queue :as queue][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -33,9 +35,42 @@(deftest restore-test(when (= (mdb/db-type) :h2)- (testing "Should throw Exception if file does not exist"- (is (= "Not found."- (mt/user-http-request :rasta :post 404 (format "testing/restore/%s" (mt/random-name))))))))+ (let [path (format "testing/restore/%s" (mt/random-name))]+ (testing "Should throw Exception if file does not exist"+ (is (= "Not found."+ (mt/user-http-request :rasta :post 404 path))))+ (testing "An indexing timeout prevents reaching the missing snapshot"+ (mt/with-dynamic-fn-redefs [search.ingestion/await-pending-updates! (constantly false)]+ (is (=? {:message "Timed out waiting for search indexing before restoring a snapshot."}+ (mt/user-http-request :rasta :post 500 path))))))))++(deftest restore-waits-for-in-flight-indexing-test+ (mt/initialize-if-needed! :db :test-users :web-server)+ (let [test-queue (queue/delay-queue)+ pending (atom 2)+ queue-cleared (promise)+ restored (promise)]+ (queue/put-with-delay! test-queue 60000 ["card" [:= :this.id 1]])+ (add-watch pending ::queue-cleared+ (fn [_key _ref _old new]+ (when (= new 1)+ (deliver queue-cleared true))))+ (with-redefs [search.ingestion/queue test-queue+ search.ingestion/pending-updates pending]+ (mt/with-dynamic-fn-redefs [testing/restore-snapshot! (fn [_name] (deliver restored @pending))+ search/sync-from-restored-db! (constantly nil)]+ (let [request (future (mt/user-http-request :rasta :post 204 "testing/restore/in-flight-indexing"))]+ (try+ (is (true? (deref queue-cleared 2000 ::timeout)))+ (is (empty? test-queue))+ (is (= ::waiting (deref restored 100 ::waiting))+ "Restoration cannot start while the remaining batch is still being indexed")+ (reset! pending 0)+ (is (nil? (deref request 2000 ::timeout)))+ (is (= 0 (deref restored 2000 ::timeout)))+ (finally+ (reset! pending 0)+ (future-cancel request))))))))(deftest e2e-test(when (= (mdb/db-type) :h2)