The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix appdb reindex misdirecting writes to the live index (#76199)
Nathan Voxland16080fabce63
diff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.cljindex c55118c4c89..eb64558612a 100644--- a/src/metabase/search/appdb/index.clj+++ b/src/metabase/search/appdb/index.clj@@ -342,8 +342,15 @@nil)))))))(defn- batch-update!- "Create the given search index entries in bulk. Commits after each batch"- [context documents]+ "Create the given search index entries in bulk. Commits after each batch.++ `reindex-table`, when non-nil, is the destination captured once at the start of a full reindex (see+ [[index-docs!]]) -- the pending table when a rebuild is staging one, otherwise the active table for an+ initial build. During a reindex we write ONLY to that captured table, so a concurrent resync that+ transiently blanks the tracking atom can't redirect writes elsewhere mid-rebuild. For incremental and+ in-place updates (`reindex-table` nil) we dual-write to the active and pending tables so an in-progress+ rebuild stays current."+ [reindex-table documents];; Protect against tests that nuke the appdb(when config/is-test?(when-let [table (active-table)]@@ -355,15 +362,20 @@(log/warnf "Unable to find table %s and no longer tracking it as pending", table)(swap! *indexes* assoc :pending nil))))- (let [reindexing? (and (= :search/reindexing context) (not search.ingestion/*force-sync*))+ (let [reindexing? (some? reindex-table)do-writes (fn []- (let [entries (map document->entry documents)- ;; No need to update the active index if we are doing a full index, as this table will be- ;; swapped out soon. Most updates would be no-ops anyway.- active-updated (when-not (and reindexing? (pending-table))- (safe-batch-upsert! :active active-table entries))- pending-updated (safe-batch-upsert! :pending pending-table entries)]- (when (or active-updated pending-updated)+ (let [entries (map document->entry documents)+ updated (if reindexing?+ ;; Full reindex: write only to the table captured for the whole run, so a+ ;; transiently-blanked tracking atom can't redirect writes into the live+ ;; active table (which would silently drop documents from the new index).+ (safe-batch-upsert! :pending (constantly reindex-table) entries)+ ;; Incremental / in-place: dual-write so an in-progress rebuild stays+ ;; current. Either table may legitimately be absent.+ (let [active-updated (safe-batch-upsert! :active active-table entries)+ pending-updated (safe-batch-upsert! :pending pending-table entries)]+ (or active-updated pending-updated)))]+ (when updated(u/prog1 (->> entries (map :model) frequencies)(when reindexing?(t2/query ["commit"]))@@ -379,10 +391,17 @@Context should be :search/updating or :search/reindexing to help control how to manage the updates"[context document-reducible](tracing/with-span :search "search.appdb.index-docs" {:search/context (name context)}- (transduce (comp (partition-all insert-batch-size)- (map (partial batch-update! context)))- (partial merge-with +)- document-reducible)))+ (let [reindexing? (and (= :search/reindexing context) (not search.ingestion/*force-sync*))+ ;; Capture the destination table ONCE for the whole reindex: the pending table when a rebuild is+ ;; staging one, otherwise the active table (initial creation populates the freshly-activated table+ ;; directly, with no pending). Resolving this per batch is unsafe -- a concurrent TTL resync of the+ ;; tracking atom can transiently blank :pending, which would otherwise flip the write target to the+ ;; live active table mid-rebuild and silently drop documents from the index we are about to activate.+ reindex-table (when reindexing? (or (pending-table) (active-table)))]+ (transduce (comp (partition-all insert-batch-size)+ (map (partial batch-update! reindex-table)))+ (partial merge-with +)+ document-reducible))))(defmethod search.engine/update! :search.engine/appdb [_engine document-reducible](index-docs! :search/updating document-reducible))diff --git a/test/metabase/search/appdb/index_test.clj b/test/metabase/search/appdb/index_test.cljindex e6917392e19..7b1d3bfe16c 100644--- a/test/metabase/search/appdb/index_test.clj+++ b/test/metabase/search/appdb/index_test.clj@@ -8,6 +8,7 @@[metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.search.appdb.index :as search.index]+ [metabase.search.appdb.specialization.api :as specialization][metabase.search.core :as search][metabase.search.engine :as search.engine][metabase.search.ingestion :as search.ingestion]@@ -706,6 +707,67 @@(search.engine/reindex! :search.engine/appdb {:in-place? true})(is (= 1 (t2/count (search.index/active-table) :model "card" :model_id (str card-id)))))))))+(deftest reindex-does-not-misdirect-writes-to-active-when-pending-tracking-lost-test+ ;; Regression for the reindex write-misdirection race. A full reindex resolved its destination table+ ;; per batch from the tracking atom. If a concurrent TTL resync transiently blanked the :pending entry+ ;; mid-reindex, later batches fell through to the LIVE active table instead of the pending one -- so+ ;; the index that then got activated was silently missing whole models (real incident: indexed-entity+ ;; and dataset, the last models processed, were written into the outgoing active table).+ (when (search/supports-index?)+ (search.tu/with-temp-index-table+ (let [active-tbl (search.index/active-table)+ pending-tbl (search.index/gen-table-name)+ pending-lost? (atom false)+ real-upsert specialization/batch-upsert!+ docs (mapv (fn [i] {:model "card" :id (str i) :name (str "Reindex doc " i)+ :display_data {} :legacy_input {} :archived false})+ (range 1 6))]+ (try+ (search.index/create-table! pending-tbl)+ (is (zero? (t2/count active-tbl)) "active index starts empty")+ (with-redefs [search.index/insert-batch-size 1+ ;; Real pending table until the first batch is written, then the tracking atom+ ;; "loses" it -- exactly what a background resync did mid-reindex in the incident.+ search.index/pending-table (fn [] (when-not @pending-lost? pending-tbl))+ specialization/batch-upsert! (fn [t entries]+ (reset! pending-lost? true)+ (real-upsert t entries))]+ ;; *force-sync* false so we exercise the real reindex write path (not the dual-write path)+ (binding [search.ingestion/*force-sync* false]+ (#'search.index/index-docs! :search/reindexing docs)))+ (testing "no batch is written to the live active table"+ (is (zero? (t2/count active-tbl))))+ (testing "every document lands in the pending table captured at the start of the reindex"+ (is (= (count docs) (t2/count pending-tbl))))+ (finally+ (#'search.index/drop-table! pending-tbl)))))))++(deftest reindex-without-pending-populates-active-table-test+ ;; The initial-build path populates the freshly-activated table directly with :search/reindexing context+ ;; and NO pending table. That must keep writing to the active table (it is the captured destination), even+ ;; if the tracking atom is blanked mid-build -- it must NOT be treated as an error.+ (when (search/supports-index?)+ (search.tu/with-temp-index-table+ (let [active-tbl (search.index/active-table)+ atom-blank? (atom false)+ real-upsert specialization/batch-upsert!+ docs (mapv (fn [i] {:model "card" :id (str i) :name (str "Init doc " i)+ :display_data {} :legacy_input {} :archived false})+ (range 1 4))]+ (is (zero? (t2/count active-tbl)) "active index starts empty")+ (with-redefs [search.index/insert-batch-size 1+ ;; No pending table at all; the active reference is "lost" after the first write to+ ;; prove the destination captured at the start is used for the whole build.+ search.index/pending-table (constantly nil)+ search.index/active-table (fn [] (when-not @atom-blank? active-tbl))+ specialization/batch-upsert! (fn [t entries]+ (reset! atom-blank? true)+ (real-upsert t entries))]+ (binding [search.ingestion/*force-sync* false]+ (#'search.index/index-docs! :search/reindexing docs)))+ (testing "every document lands in the active table that was captured at the start of the build"+ (is (= (count docs) (t2/count active-tbl))))))))+(deftest when-index-created(when (search/supports-index?)(binding [search.spec/*testing-only-index-version-hash* "index-age-test"]