Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix search related testing race conditions (#71290)

Chris Truterd34d1dc9ff64
diff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.clj
index 8ac64def192..6a934ecfce9 100644
--- a/src/metabase/search/appdb/index.clj
+++ b/src/metabase/search/appdb/index.clj
@@ -105,8 +105,8 @@
(defn- drop-table! [table]
(boolean
- (when (and table (exists? table))
- (t2/query (sql.helpers/drop-table (keyword (table-name table)))))))
+ (when table
+ (t2/query (sql.helpers/drop-table :if-exists (keyword (table-name table)))))))
(defn- orphan-indexes []
(map (comp keyword u/lower-case-en :table_name)
@@ -119,6 +119,8 @@
;; legacy table names
[:in [:lower :table_name]
(mapv #(vector :inline %) ["search_index" "search_index_next" "search_index_retired"])]]
+ ;; Exclude temp tables — they are managed by with-temp-index-table
+ [:not-like [:lower :table_name] [:inline "%\\_temp"]]
[:not-in [:lower :table_name]
[:raw
(str "("
@@ -261,22 +263,37 @@
(or (instance? PSQLException (ex-cause e))
(instance? JdbcSQLSyntaxErrorException (ex-cause e))))
-(defn- safe-batch-upsert! [table-name entries]
+(defn- retry-upsert-ex [table-type table-name-before table-name-after e-before e-after]
+ (ex-info "Failed retrying search index batch upsert"
+ {:table-type table-type
+ :table-name-before-refresh table-name-before
+ :table-name-after-refresh table-name-after
+ :initial-exception-class (class e-before)
+ :initial-exception-message (ex-message e-before)}
+ e-after))
+
+(defn- safe-batch-upsert!
+ "A version of batch-upsert! that no-ops for missing indexes, and handles stale index tracking metadata.
+
+ Returns the name of the table that was written to, or nil if there is none being tracked.
+ We recover gracefully the first time if the tracking atom was stale, but do not check again on retry."
+ [table-type table-name-fn entries]
;; For convenience, no-op if we are not tracking any table.
- (when table-name
- (try
- (specialization/batch-upsert! table-name entries)
- (catch Exception e
- (if (table-not-found-exception? e)
- ;; If resetting tracking atoms resolves the issue (which is likely happened because of stale tracking data),
- ;; suppress the issue - but throw it all the way to the caller if the issue persists
- (try
- (sync-tracking-atoms!)
- (specialization/batch-upsert! table-name entries)
- (catch Exception e2
- (log/error e2 "Error syncing index tracking atoms after table not found exception")
- (throw e)))
- (throw e))))))
+ (when-let [table-name (table-name-fn)]
+ (let [upsert! (fn [t] (specialization/batch-upsert! t entries) t)]
+ (try
+ (upsert! table-name)
+ (catch Exception e
+ ;; Only suppress failures related to a legitimately non-existent table
+ (if (or (not (table-not-found-exception? e)) (exists? table-name))
+ (throw e)
+ (when-let [refreshed-table-name (do (sync-tracking-atoms!) (table-name-fn))]
+ (if (= table-name refreshed-table-name)
+ (throw (ex-info "Currently tracked index does not exist" e {:table-name table-name}))
+ (try
+ (upsert! refreshed-table-name)
+ (catch Exception e2
+ (retry-upsert-ex table-type table-name refreshed-table-name e e2)))))))))))
(defn- batch-update!
"Create the given search index entries in bulk. Commits after each batch"
@@ -294,19 +311,22 @@
(let [reindexing? (and (= :search/reindexing context) (not search.ingestion/*force-sync*))
do-writes (fn []
- (let [active-table (active-table)
- entries (map document->entry documents)
- ;; No need to update the active index if we are doing a full index and it will be swapped out soon. Most updates are no-ops anyway.
- active-updated? (when-not (and active-table (pending-table) reindexing?)
- (safe-batch-upsert! active-table entries))
- pending-updated? (safe-batch-upsert! (pending-table) entries)]
- (when (or active-updated? pending-updated?)
+ (let [entries (map document->entry documents)
+ ;; No need to update the active index if we are doing a full index, as this table will be
+ ;; swapped out soon. Most updates would be no-ops anyway.
+ active-updated (when-not (and reindexing? (pending-table))
+ (safe-batch-upsert! :active active-table entries))
+ pending-updated (safe-batch-upsert! :pending pending-table entries)]
+ (when (or active-updated pending-updated)
(u/prog1 (->> entries (map :model) frequencies)
(when reindexing?
(t2/query ["commit"]))
(log/trace "indexed documents for " <>)
- (when active-updated?
- (analytics/set! :metabase-search/appdb-index-size (t2/count (name active-table))))))))]
+ (when active-updated
+ (try
+ (analytics/set! :metabase-search/appdb-index-size (t2/count (name active-updated)))
+ (catch Exception e
+ (log/warnf e "Unable to measure active search index size (%s)" active-updated))))))))]
(if reindexing?
;; New connection used for performing the updates which commit periodically without impacting any outer transactions.
(t2/with-connection [_conn (mdb/data-source)]