Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix some indexing race conditions (#64783)

Nathan Voxland4f9dc213eddf
diff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.clj
index 7c33fa62207..d32e093f6ea 100644
--- a/src/metabase/search/appdb/index.clj
+++ b/src/metabase/search/appdb/index.clj
@@ -43,7 +43,9 @@
(defonce ^:private next-sync-at (atom nil))
-(defonce ^:dynamic ^:private *indexes* (atom {:active nil, :pending nil}))
+(defonce ^:dynamic ^:private ^{:doc "This atom is often reset! in threads, so modifications should be done only when locking it first."}
+ *indexes*
+ (atom {:active nil, :pending nil}))
(def ^:private ^:dynamic *mocking-tables* false)
@@ -52,24 +54,31 @@
(declare exists?)
-(defn- sync-tracking-atoms! []
- (let [indexes (into {}
- (for [[status table-name] (search-index-metadata/indexes :appdb *index-version-id*)]
- (if (exists? table-name)
- [status (keyword table-name)]
- ;; For debugging, make it clear why we are not tracking the given metadata.
- [(keyword (name status) "not-found") (keyword table-name)])))]
- (log/debugf "Sync tracking atoms: %s" indexes)
- (reset! *indexes* indexes)))
+(defn- sync-tracking-atoms!
+ "Sync the *indexes* atom with the current database metadata state."
+ []
+ ;; Locks the indexes so the reset! doesn't lose data written to the db by a different thread between the read and write
+ (locking *indexes*
+ (let [indexes (into {}
+ (for [[status table-name] (search-index-metadata/indexes :appdb *index-version-id*)]
+ (if (exists? table-name)
+ [status (keyword table-name)]
+ ;; For debugging, make it clear why we are not tracking the given metadata.
+ [(keyword (name status) "not-found") (keyword table-name)])))]
+ (log/debugf "Sync tracking atoms: %s" indexes)
+ (reset! *indexes* indexes))))
;; This exists only to be mocked.
(defn- now [] (System/nanoTime))
(defn- sync-tracking-atoms-if-stale! []
(when-not *mocking-tables*
- (when (or (not @next-sync-at) (> (now) @next-sync-at))
- (reset! next-sync-at (+ (now) sync-tracking-period))
- (sync-tracking-atoms!))))
+ (let [current @next-sync-at
+ now-ns (now)]
+ (when (or (nil? current) (> now-ns current))
+ ;; Use compare-and-set! to ensure only one thread wins the race and syncs
+ (when (compare-and-set! next-sync-at current (+ now-ns sync-tracking-period))
+ (sync-tracking-atoms!))))))
(defn active-table
"The table against which we should currently make search queries."
@@ -187,52 +196,54 @@
(defn maybe-create-pending!
"Create a search index table if one doesn't exist. Record and return the name of the table, regardless."
[]
- (if *mocking-tables*
- ;; The atoms are the only source of truth, create a new table if necessary.
- (or (pending-table)
- (let [table-name (gen-table-name)]
- (create-table! table-name)
- (swap! *indexes* assoc :pending table-name)))
- ;; The database is the source of truth
- (let [{:keys [pending]} (sync-tracking-atoms!)]
- (or pending
+ (locking *indexes*
+ (if *mocking-tables*
+ ;; In a test where the atoms are the source of truth, create a new table if necessary.
+ (or (pending-table)
(let [table-name (gen-table-name)]
- (log/infof "Creating pending index %s for lang %s" table-name (i18n/site-locale-string))
+ (create-table! table-name)
+ (swap! *indexes* assoc :pending table-name) table-name))
+ ;; The database is the source of truth
+ (let [{:keys [pending]} (sync-tracking-atoms!)]
+ (or pending
+ (let [table-name (gen-table-name)]
+ (log/infof "Creating pending index %s for lang %s" table-name (i18n/site-locale-string))
;; We may fail to insert a new metadata row if we lose a race with another instance.
- (when (search-index-metadata/create-pending! :appdb *index-version-id* table-name)
- (try
- (create-table! table-name)
- (catch Exception e
- (log/error e "Error creating pending index table, cleaning up metadata")
- (try
- (t2/with-connection [safe-conn (mdb/app-db)]
- (t2/delete! :conn safe-conn :model/SearchIndexMetadata :index_name (name table-name)))
- (catch Exception del-e
- (log/warn del-e "Error clearing out search metadata after failure")))
- (sync-tracking-atoms!))))
- (let [pending (:pending (sync-tracking-atoms!))]
- (log/infof "New pending index %s" pending)
- pending))))))
+ (when (search-index-metadata/create-pending! :appdb *index-version-id* table-name)
+ (try
+ (create-table! table-name)
+ (catch Exception e
+ (log/error e "Error creating pending index table, cleaning up metadata")
+ (try
+ (t2/with-connection [safe-conn (mdb/app-db)]
+ (t2/delete! :conn safe-conn :model/SearchIndexMetadata :index_name (name table-name)))
+ (catch Exception del-e
+ (log/warn del-e "Error clearing out search metadata after failure")))
+ (sync-tracking-atoms!))))
+ (let [pending (:pending (sync-tracking-atoms!))]
+ (log/infof "New pending index %s" pending)
+ pending)))))))
(defn activate-table!
"Make the pending index active if it exists. Returns true if it did so."
[]
- (if *mocking-tables*
- ;; The atoms are the only source of truth, we must not update the metadata.
- (boolean
- (when-let [pending (:pending @*indexes*)]
- (reset! *indexes* {:pending nil, :active pending})))
- ;; Ensure the metadata is updated and pruned.
- (let [{:keys [pending]} (sync-tracking-atoms!)]
- (log/infof "Activating pending index %s" pending)
- (when pending
- (let [active (keyword (search-index-metadata/active-pending! :appdb *index-version-id*))]
- (reset! *indexes* {:pending nil :active active})
- (log/infof "Activated pending index %s" active)))
- ;; Clean up while we're here
- (delete-obsolete-tables!)
- ;; Did *we* do a rotation?
- (boolean pending))))
+ (locking *indexes*
+ (if *mocking-tables*
+ ;; The atoms are the only source of truth, we must not update the metadata.
+ (boolean
+ (when-let [pending (:pending @*indexes*)]
+ (reset! *indexes* {:pending nil, :active pending}) true))
+ ;; Ensure the metadata is updated and pruned.
+ (let [{:keys [pending]} (sync-tracking-atoms!)]
+ (log/infof "Activating pending index %s" pending)
+ (when pending
+ (let [active (keyword (search-index-metadata/active-pending! :appdb *index-version-id*))]
+ (reset! *indexes* {:pending nil :active active})
+ (log/infof "Activated pending index %s" active)))
+ ;; Clean up while we're here
+ (delete-obsolete-tables!)
+ ;; Did *we* do a rotation?
+ (boolean pending)))))
(defn- document->entry [entity]
(-> entity
diff --git a/src/metabase/search/core.clj b/src/metabase/search/core.clj
index 0622d63d6ad..4d7d5a96c94 100644
--- a/src/metabase/search/core.clj
+++ b/src/metabase/search/core.clj
@@ -133,7 +133,13 @@
Respects `search.ingestion/*force-sync*` and waits for the future if it's true.
Alternately, if `:async?` is false, it will also run synchronously."
[& {:keys [async?] :or {async? true} :as opts}]
- (let [f #(reindex-logic! opts)]
+ (let [f (fn []
+ (try
+ (reindex-logic! opts)
+ (catch Exception e
+ (log/error e "Reindex failed")
+ (analytics/inc! :metabase-search/index-error)
+ (throw e))))]
(if (or search.ingestion/*force-sync* (not async?))
(doto (promise) (deliver (f)))
(future (f)))))