The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GDGT-3326: Queue search index updates after the transaction commits (#83946)
Will Frye2cbb0fc93d8
diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.cljindex 3de0a8c336b..8e51d4f4cb1 100644--- a/src/metabase/search/ingestion.clj+++ b/src/metabase/search/ingestion.clj@@ -3,6 +3,7 @@[clojure.string :as str][medley.core :as m][metabase.analytics-interface.core :as analytics]+ [metabase.app-db.core :as mdb][metabase.collections.curation :as collections.curation][metabase.lib-be.core :as lib-be][metabase.search.db :as search.db]@@ -26,8 +27,10 @@;; Perhaps this config move up somewhere more visible? Conversely, we may want to specialize it per engine.(def ^:private message-delay-ms- "The time a message should wait before coming off the queue.- This delay exists to ensure the data is fully committed before indexing."+ "The time a message waits before coming off the queue.+ Updates made in a transaction are queued when it commits, see [[ingest-maybe-async!]].+ The delay is a head start for a write in a transaction `mdb/do-after-commit` cannot see, such as a raw JDBC one.+ Such a write is still missed if its transaction stays open longer than the delay."100)(def ^:private listener-name@@ -291,7 +294,8 @@(defn ingest-maybe-async!"Update or create any search index entries related to the given updates.Will be async if the worker exists, otherwise it will be done synchronously on the calling thread.- Can also be forced to run synchronously for testing."+ Can also be forced to run synchronously for testing.+ Async updates are queued when the outermost transaction commits, and dropped if it rolls back."([updates](ingest-maybe-async! updates (or *force-sync* (not (index-worker-exists?)))))([updates sync?]@@ -299,10 +303,13 @@(if sync?(bulk-ingest! updates)(do- (doseq [update updates]- (log/trace "Queuing update" update)- (queue/put-with-delay! queue message-delay-ms update))- (track-queue-size!)+ ;; The worker reads rows back on its own connection, so it cannot see them before the commit.+ (mdb/do-after-commit+ (fn []+ (doseq [update updates]+ (log/trace "Queuing update" update)+ (queue/put-with-delay! queue message-delay-ms update))+ (track-queue-size!)))true)))))(defn wait-for-idle!diff --git a/test/metabase/search/ingestion_test.clj b/test/metabase/search/ingestion_test.cljindex c012de5e661..aa58a2d658c 100644--- a/test/metabase/search/ingestion_test.clj+++ b/test/metabase/search/ingestion_test.clj@@ -15,6 +15,7 @@[metabase.search.test-util :as search.tu][metabase.test :as mt][metabase.util.json :as json]+ [metabase.util.queue :as queue][toucan2.core :as t2]))(deftest extract-model-and-id@@ -263,3 +264,38 @@first)](is (=? {:model "table" :curated true :data_authority "authoritative" :data_layer "final"}result)))))))++(defn- queued-updates!+ "Run `f` with a fresh ingestion queue swapped in, and return the updates that end up on it."+ [f]+ (let [test-queue (queue/delay-queue)]+ (with-redefs [search.ingestion/queue test-queue]+ (f test-queue))+ (mapv :value test-queue)))++(deftest ingest-maybe-async!-queues-after-commit-test+ ;; The worker reads each updated row back from the app DB. If an update reaches the queue before its transaction+ ;; commits, and the transaction takes longer than the queue delay, the worker finds no row and the entity is+ ;; silently left out of the index (GDGT-3326).+ (let [update ["collection" [:= 1 :this.id]]]+ (testing "async updates made inside a transaction are only queued once it commits"+ (let [queued (queued-updates!+ (fn [test-queue]+ (t2/with-transaction [_conn]+ (search.ingestion/ingest-maybe-async! [update] false)+ (testing "nothing is queued while the transaction is still open"+ (is (empty? test-queue))))))]+ (is (= [update] queued))))+ (testing "nothing is queued when the transaction rolls back"+ (let [queued (queued-updates!+ (fn [_test-queue]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"rollback"+ (t2/with-transaction [_conn]+ (search.ingestion/ingest-maybe-async! [update] false)+ (throw (ex-info "rollback" {})))))))]+ (is (= [] queued))))+ (testing "outside a transaction the update is queued immediately"+ (is (= [update]+ (queued-updates!+ (fn [_test-queue]+ (search.ingestion/ingest-maybe-async! [update] false))))))))