Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Better de-duping of items to index to avoid `ON CONFLICT DO UPDATE` error (#74160)

Nathan Voxland48b86050c15c
diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.clj
index 333604c9de8..78e829bb03a 100644
--- a/src/metabase/search/ingestion.clj
+++ b/src/metabase/search/ingestion.clj
@@ -192,13 +192,13 @@
(sql.helpers/where where-clause)))
(defn- spec-index-reducible [search-model & [where-clause]]
+ ;; Joins in the spec (e.g. card → revision on most_recent = true) can produce duplicate rows when the
+ ;; joined side has its own integrity violations. Downstream upserts hit a unique constraint on
+ ;; (model, model_id), so dedup here at the streaming boundary — bounded by per-model row count.
(->> (spec-index-query-where search-model where-clause)
mdb/streaming-reducible-query
- (eduction (cond-> (map #(assoc % :model search-model))
- ;; It's possible to get redundant entries from the indexed-entities table.
- ;; We deduplicate only that model to avoid an unbounded set over all documents.
- (= "indexed-entity" search-model)
- (comp (m/distinct-by (juxt :id :model)))))))
+ (eduction (comp (map #(assoc % :model search-model))
+ (m/distinct-by (juxt :id :model))))))
(defn- search-items-reducible []
(reduce u/rconcat [] (map spec-index-reducible search.spec/search-models)))
diff --git a/test/metabase/search/appdb/index_test.clj b/test/metabase/search/appdb/index_test.clj
index 9f66a81d15a..6444b9fdc73 100644
--- a/test/metabase/search/appdb/index_test.clj
+++ b/test/metabase/search/appdb/index_test.clj
@@ -688,6 +688,33 @@
(is (false? (.contains json-str ^CharSequence NUL)) "encoded JSON has no raw NUL")
(is (false? (.contains json-str ^CharSequence FF)) "encoded JSON has no raw form feed"))))))
+(deftest reindex-survives-duplicate-most-recent-revisions-test
+ ;; Two `revision` rows with `most_recent = true` for the same card cause the spec's LEFT
+ ;; JOIN to produce two rows per card. Before this fix, the resulting duplicate (model, model_id)
+ ;; pair in one upsert batch tripped a unique constraint, failing the batch (and the whole reindex).
+ (when (search/supports-index?)
+ (with-index
+ (let [card-id (t2/select-one-pk :model/Card :name "Customer Satisfaction")
+ insert-rev! (fn []
+ ;; Raw SQL to bypass the before/after-insert hooks that would normally
+ ;; demote older revisions to most_recent=false.
+ (t2/query {:insert-into [(t2/table-name :model/Revision)]
+ :values [{:model "Card"
+ :model_id card-id
+ :user_id (mt/user->id :rasta)
+ :object "{}"
+ :is_creation false
+ :is_reversion false
+ :most_recent true
+ :timestamp :%now}]}))]
+ (insert-rev!)
+ (insert-rev!)
+ (testing "two revision rows with most_recent=true exist for the card"
+ (is (= 2 (t2/count :model/Revision :model "Card" :model_id card-id :most_recent true))))
+ (testing "reindex completes and writes a single row for the card"
+ (search.engine/reindex! :search.engine/appdb {:in-place? true})
+ (is (= 1 (t2/count (search.index/active-table) :model "card" :model_id (str card-id)))))))))
+
(deftest when-index-created
(when (search/supports-index?)
(binding [search.spec/*testing-only-index-version-hash* "index-age-test"]