Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix appdb reindex misdirecting writes to the live index (#76199)

Nathan Voxland16080fabce63
diff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.clj
index c55118c4c89..eb64558612a 100644
--- a/src/metabase/search/appdb/index.clj
+++ b/src/metabase/search/appdb/index.clj
@@ -342,8 +342,15 @@
nil)))))))
(defn- batch-update!
- "Create the given search index entries in bulk. Commits after each batch"
- [context documents]
+ "Create the given search index entries in bulk. Commits after each batch.
+
+ `reindex-table`, when non-nil, is the destination captured once at the start of a full reindex (see
+ [[index-docs!]]) -- the pending table when a rebuild is staging one, otherwise the active table for an
+ initial build. During a reindex we write ONLY to that captured table, so a concurrent resync that
+ transiently blanks the tracking atom can't redirect writes elsewhere mid-rebuild. For incremental and
+ in-place updates (`reindex-table` nil) we dual-write to the active and pending tables so an in-progress
+ rebuild stays current."
+ [reindex-table documents]
;; Protect against tests that nuke the appdb
(when config/is-test?
(when-let [table (active-table)]
@@ -355,15 +362,20 @@
(log/warnf "Unable to find table %s and no longer tracking it as pending", table)
(swap! *indexes* assoc :pending nil))))
- (let [reindexing? (and (= :search/reindexing context) (not search.ingestion/*force-sync*))
+ (let [reindexing? (some? reindex-table)
do-writes (fn []
- (let [entries (map document->entry documents)
- ;; No need to update the active index if we are doing a full index, as this table will be
- ;; swapped out soon. Most updates would be no-ops anyway.
- active-updated (when-not (and reindexing? (pending-table))
- (safe-batch-upsert! :active active-table entries))
- pending-updated (safe-batch-upsert! :pending pending-table entries)]
- (when (or active-updated pending-updated)
+ (let [entries (map document->entry documents)
+ updated (if reindexing?
+ ;; Full reindex: write only to the table captured for the whole run, so a
+ ;; transiently-blanked tracking atom can't redirect writes into the live
+ ;; active table (which would silently drop documents from the new index).
+ (safe-batch-upsert! :pending (constantly reindex-table) entries)
+ ;; Incremental / in-place: dual-write so an in-progress rebuild stays
+ ;; current. Either table may legitimately be absent.
+ (let [active-updated (safe-batch-upsert! :active active-table entries)
+ pending-updated (safe-batch-upsert! :pending pending-table entries)]
+ (or active-updated pending-updated)))]
+ (when updated
(u/prog1 (->> entries (map :model) frequencies)
(when reindexing?
(t2/query ["commit"]))
@@ -379,10 +391,17 @@
Context should be :search/updating or :search/reindexing to help control how to manage the updates"
[context document-reducible]
(tracing/with-span :search "search.appdb.index-docs" {:search/context (name context)}
- (transduce (comp (partition-all insert-batch-size)
- (map (partial batch-update! context)))
- (partial merge-with +)
- document-reducible)))
+ (let [reindexing? (and (= :search/reindexing context) (not search.ingestion/*force-sync*))
+ ;; Capture the destination table ONCE for the whole reindex: the pending table when a rebuild is
+ ;; staging one, otherwise the active table (initial creation populates the freshly-activated table
+ ;; directly, with no pending). Resolving this per batch is unsafe -- a concurrent TTL resync of the
+ ;; tracking atom can transiently blank :pending, which would otherwise flip the write target to the
+ ;; live active table mid-rebuild and silently drop documents from the index we are about to activate.
+ reindex-table (when reindexing? (or (pending-table) (active-table)))]
+ (transduce (comp (partition-all insert-batch-size)
+ (map (partial batch-update! reindex-table)))
+ (partial merge-with +)
+ document-reducible))))
(defmethod search.engine/update! :search.engine/appdb [_engine document-reducible]
(index-docs! :search/updating document-reducible))
diff --git a/test/metabase/search/appdb/index_test.clj b/test/metabase/search/appdb/index_test.clj
index e6917392e19..7b1d3bfe16c 100644
--- a/test/metabase/search/appdb/index_test.clj
+++ b/test/metabase/search/appdb/index_test.clj
@@ -8,6 +8,7 @@
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.search.appdb.index :as search.index]
+ [metabase.search.appdb.specialization.api :as specialization]
[metabase.search.core :as search]
[metabase.search.engine :as search.engine]
[metabase.search.ingestion :as search.ingestion]
@@ -706,6 +707,67 @@
(search.engine/reindex! :search.engine/appdb {:in-place? true})
(is (= 1 (t2/count (search.index/active-table) :model "card" :model_id (str card-id)))))))))
+(deftest reindex-does-not-misdirect-writes-to-active-when-pending-tracking-lost-test
+ ;; Regression for the reindex write-misdirection race. A full reindex resolved its destination table
+ ;; per batch from the tracking atom. If a concurrent TTL resync transiently blanked the :pending entry
+ ;; mid-reindex, later batches fell through to the LIVE active table instead of the pending one -- so
+ ;; the index that then got activated was silently missing whole models (real incident: indexed-entity
+ ;; and dataset, the last models processed, were written into the outgoing active table).
+ (when (search/supports-index?)
+ (search.tu/with-temp-index-table
+ (let [active-tbl (search.index/active-table)
+ pending-tbl (search.index/gen-table-name)
+ pending-lost? (atom false)
+ real-upsert specialization/batch-upsert!
+ docs (mapv (fn [i] {:model "card" :id (str i) :name (str "Reindex doc " i)
+ :display_data {} :legacy_input {} :archived false})
+ (range 1 6))]
+ (try
+ (search.index/create-table! pending-tbl)
+ (is (zero? (t2/count active-tbl)) "active index starts empty")
+ (with-redefs [search.index/insert-batch-size 1
+ ;; Real pending table until the first batch is written, then the tracking atom
+ ;; "loses" it -- exactly what a background resync did mid-reindex in the incident.
+ search.index/pending-table (fn [] (when-not @pending-lost? pending-tbl))
+ specialization/batch-upsert! (fn [t entries]
+ (reset! pending-lost? true)
+ (real-upsert t entries))]
+ ;; *force-sync* false so we exercise the real reindex write path (not the dual-write path)
+ (binding [search.ingestion/*force-sync* false]
+ (#'search.index/index-docs! :search/reindexing docs)))
+ (testing "no batch is written to the live active table"
+ (is (zero? (t2/count active-tbl))))
+ (testing "every document lands in the pending table captured at the start of the reindex"
+ (is (= (count docs) (t2/count pending-tbl))))
+ (finally
+ (#'search.index/drop-table! pending-tbl)))))))
+
+(deftest reindex-without-pending-populates-active-table-test
+ ;; The initial-build path populates the freshly-activated table directly with :search/reindexing context
+ ;; and NO pending table. That must keep writing to the active table (it is the captured destination), even
+ ;; if the tracking atom is blanked mid-build -- it must NOT be treated as an error.
+ (when (search/supports-index?)
+ (search.tu/with-temp-index-table
+ (let [active-tbl (search.index/active-table)
+ atom-blank? (atom false)
+ real-upsert specialization/batch-upsert!
+ docs (mapv (fn [i] {:model "card" :id (str i) :name (str "Init doc " i)
+ :display_data {} :legacy_input {} :archived false})
+ (range 1 4))]
+ (is (zero? (t2/count active-tbl)) "active index starts empty")
+ (with-redefs [search.index/insert-batch-size 1
+ ;; No pending table at all; the active reference is "lost" after the first write to
+ ;; prove the destination captured at the start is used for the whole build.
+ search.index/pending-table (constantly nil)
+ search.index/active-table (fn [] (when-not @atom-blank? active-tbl))
+ specialization/batch-upsert! (fn [t entries]
+ (reset! atom-blank? true)
+ (real-upsert t entries))]
+ (binding [search.ingestion/*force-sync* false]
+ (#'search.index/index-docs! :search/reindexing docs)))
+ (testing "every document lands in the active table that was captured at the start of the build"
+ (is (= (count docs) (t2/count active-tbl))))))))
+
(deftest when-index-created
(when (search/supports-index?)
(binding [search.spec/*testing-only-index-version-hash* "index-age-test"]