Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

GDGT-3326: Queue search index updates after the transaction commits (#83946)

Will Frye2cbb0fc93d8
diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.clj
index 3de0a8c336b..8e51d4f4cb1 100644
--- a/src/metabase/search/ingestion.clj
+++ b/src/metabase/search/ingestion.clj
@@ -3,6 +3,7 @@
[clojure.string :as str]
[medley.core :as m]
[metabase.analytics-interface.core :as analytics]
+ [metabase.app-db.core :as mdb]
[metabase.collections.curation :as collections.curation]
[metabase.lib-be.core :as lib-be]
[metabase.search.db :as search.db]
@@ -26,8 +27,10 @@
;; Perhaps this config move up somewhere more visible? Conversely, we may want to specialize it per engine.
(def ^:private message-delay-ms
- "The time a message should wait before coming off the queue.
- This delay exists to ensure the data is fully committed before indexing."
+ "The time a message waits before coming off the queue.
+ Updates made in a transaction are queued when it commits, see [[ingest-maybe-async!]].
+ The delay is a head start for a write in a transaction `mdb/do-after-commit` cannot see, such as a raw JDBC one.
+ Such a write is still missed if its transaction stays open longer than the delay."
100)
(def ^:private listener-name
@@ -291,7 +294,8 @@
(defn ingest-maybe-async!
"Update or create any search index entries related to the given updates.
Will be async if the worker exists, otherwise it will be done synchronously on the calling thread.
- Can also be forced to run synchronously for testing."
+ Can also be forced to run synchronously for testing.
+ Async updates are queued when the outermost transaction commits, and dropped if it rolls back."
([updates]
(ingest-maybe-async! updates (or *force-sync* (not (index-worker-exists?)))))
([updates sync?]
@@ -299,10 +303,13 @@
(if sync?
(bulk-ingest! updates)
(do
- (doseq [update updates]
- (log/trace "Queuing update" update)
- (queue/put-with-delay! queue message-delay-ms update))
- (track-queue-size!)
+ ;; The worker reads rows back on its own connection, so it cannot see them before the commit.
+ (mdb/do-after-commit
+ (fn []
+ (doseq [update updates]
+ (log/trace "Queuing update" update)
+ (queue/put-with-delay! queue message-delay-ms update))
+ (track-queue-size!)))
true)))))
(defn wait-for-idle!
diff --git a/test/metabase/search/ingestion_test.clj b/test/metabase/search/ingestion_test.clj
index c012de5e661..aa58a2d658c 100644
--- a/test/metabase/search/ingestion_test.clj
+++ b/test/metabase/search/ingestion_test.clj
@@ -15,6 +15,7 @@
[metabase.search.test-util :as search.tu]
[metabase.test :as mt]
[metabase.util.json :as json]
+ [metabase.util.queue :as queue]
[toucan2.core :as t2]))
(deftest extract-model-and-id
@@ -263,3 +264,38 @@
first)]
(is (=? {:model "table" :curated true :data_authority "authoritative" :data_layer "final"}
result)))))))
+
+(defn- queued-updates!
+ "Run `f` with a fresh ingestion queue swapped in, and return the updates that end up on it."
+ [f]
+ (let [test-queue (queue/delay-queue)]
+ (with-redefs [search.ingestion/queue test-queue]
+ (f test-queue))
+ (mapv :value test-queue)))
+
+(deftest ingest-maybe-async!-queues-after-commit-test
+ ;; The worker reads each updated row back from the app DB. If an update reaches the queue before its transaction
+ ;; commits, and the transaction takes longer than the queue delay, the worker finds no row and the entity is
+ ;; silently left out of the index (GDGT-3326).
+ (let [update ["collection" [:= 1 :this.id]]]
+ (testing "async updates made inside a transaction are only queued once it commits"
+ (let [queued (queued-updates!
+ (fn [test-queue]
+ (t2/with-transaction [_conn]
+ (search.ingestion/ingest-maybe-async! [update] false)
+ (testing "nothing is queued while the transaction is still open"
+ (is (empty? test-queue))))))]
+ (is (= [update] queued))))
+ (testing "nothing is queued when the transaction rolls back"
+ (let [queued (queued-updates!
+ (fn [_test-queue]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"rollback"
+ (t2/with-transaction [_conn]
+ (search.ingestion/ingest-maybe-async! [update] false)
+ (throw (ex-info "rollback" {})))))))]
+ (is (= [] queued))))
+ (testing "outside a transaction the update is queued immediately"
+ (is (= [update]
+ (queued-updates!
+ (fn [_test-queue]
+ (search.ingestion/ingest-maybe-async! [update] false))))))))