The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Better de-duping of items to index to avoid `ON CONFLICT DO UPDATE` error (#74160)
Nathan Voxland48b86050c15c
diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.cljindex 333604c9de8..78e829bb03a 100644--- a/src/metabase/search/ingestion.clj+++ b/src/metabase/search/ingestion.clj@@ -192,13 +192,13 @@(sql.helpers/where where-clause)))(defn- spec-index-reducible [search-model & [where-clause]]+ ;; Joins in the spec (e.g. card → revision on most_recent = true) can produce duplicate rows when the+ ;; joined side has its own integrity violations. Downstream upserts hit a unique constraint on+ ;; (model, model_id), so dedup here at the streaming boundary — bounded by per-model row count.(->> (spec-index-query-where search-model where-clause)mdb/streaming-reducible-query- (eduction (cond-> (map #(assoc % :model search-model))- ;; It's possible to get redundant entries from the indexed-entities table.- ;; We deduplicate only that model to avoid an unbounded set over all documents.- (= "indexed-entity" search-model)- (comp (m/distinct-by (juxt :id :model)))))))+ (eduction (comp (map #(assoc % :model search-model))+ (m/distinct-by (juxt :id :model))))))(defn- search-items-reducible [](reduce u/rconcat [] (map spec-index-reducible search.spec/search-models)))diff --git a/test/metabase/search/appdb/index_test.clj b/test/metabase/search/appdb/index_test.cljindex 9f66a81d15a..6444b9fdc73 100644--- a/test/metabase/search/appdb/index_test.clj+++ b/test/metabase/search/appdb/index_test.clj@@ -688,6 +688,33 @@(is (false? (.contains json-str ^CharSequence NUL)) "encoded JSON has no raw NUL")(is (false? (.contains json-str ^CharSequence FF)) "encoded JSON has no raw form feed"))))))+(deftest reindex-survives-duplicate-most-recent-revisions-test+ ;; Two `revision` rows with `most_recent = true` for the same card cause the spec's LEFT+ ;; JOIN to produce two rows per card. Before this fix, the resulting duplicate (model, model_id)+ ;; pair in one upsert batch tripped a unique constraint, failing the batch (and the whole reindex).+ (when (search/supports-index?)+ (with-index+ (let [card-id (t2/select-one-pk :model/Card :name "Customer Satisfaction")+ insert-rev! (fn []+ ;; Raw SQL to bypass the before/after-insert hooks that would normally+ ;; demote older revisions to most_recent=false.+ (t2/query {:insert-into [(t2/table-name :model/Revision)]+ :values [{:model "Card"+ :model_id card-id+ :user_id (mt/user->id :rasta)+ :object "{}"+ :is_creation false+ :is_reversion false+ :most_recent true+ :timestamp :%now}]}))]+ (insert-rev!)+ (insert-rev!)+ (testing "two revision rows with most_recent=true exist for the card"+ (is (= 2 (t2/count :model/Revision :model "Card" :model_id card-id :most_recent true))))+ (testing "reindex completes and writes a single row for the card"+ (search.engine/reindex! :search.engine/appdb {:in-place? true})+ (is (= 1 (t2/count (search.index/active-table) :model "card" :model_id (str card-id)))))))))+(deftest when-index-created(when (search/supports-index?)(binding [search.spec/*testing-only-index-version-hash* "index-age-test"]