The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix some indexing race conditions (#64783)
Nathan Voxland4f9dc213eddf
diff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.cljindex 7c33fa62207..d32e093f6ea 100644--- a/src/metabase/search/appdb/index.clj+++ b/src/metabase/search/appdb/index.clj@@ -43,7 +43,9 @@(defonce ^:private next-sync-at (atom nil))-(defonce ^:dynamic ^:private *indexes* (atom {:active nil, :pending nil}))+(defonce ^:dynamic ^:private ^{:doc "This atom is often reset! in threads, so modifications should be done only when locking it first."}+ *indexes*+ (atom {:active nil, :pending nil}))(def ^:private ^:dynamic *mocking-tables* false)@@ -52,24 +54,31 @@(declare exists?)-(defn- sync-tracking-atoms! []- (let [indexes (into {}- (for [[status table-name] (search-index-metadata/indexes :appdb *index-version-id*)]- (if (exists? table-name)- [status (keyword table-name)]- ;; For debugging, make it clear why we are not tracking the given metadata.- [(keyword (name status) "not-found") (keyword table-name)])))]- (log/debugf "Sync tracking atoms: %s" indexes)- (reset! *indexes* indexes)))+(defn- sync-tracking-atoms!+ "Sync the *indexes* atom with the current database metadata state."+ []+ ;; Locks the indexes so the reset! doesn't lose data written to the db by a different thread between the read and write+ (locking *indexes*+ (let [indexes (into {}+ (for [[status table-name] (search-index-metadata/indexes :appdb *index-version-id*)]+ (if (exists? table-name)+ [status (keyword table-name)]+ ;; For debugging, make it clear why we are not tracking the given metadata.+ [(keyword (name status) "not-found") (keyword table-name)])))]+ (log/debugf "Sync tracking atoms: %s" indexes)+ (reset! *indexes* indexes))));; This exists only to be mocked.(defn- now [] (System/nanoTime))(defn- sync-tracking-atoms-if-stale! [](when-not *mocking-tables*- (when (or (not @next-sync-at) (> (now) @next-sync-at))- (reset! next-sync-at (+ (now) sync-tracking-period))- (sync-tracking-atoms!))))+ (let [current @next-sync-at+ now-ns (now)]+ (when (or (nil? current) (> now-ns current))+ ;; Use compare-and-set! to ensure only one thread wins the race and syncs+ (when (compare-and-set! next-sync-at current (+ now-ns sync-tracking-period))+ (sync-tracking-atoms!))))))(defn active-table"The table against which we should currently make search queries."@@ -187,52 +196,54 @@(defn maybe-create-pending!"Create a search index table if one doesn't exist. Record and return the name of the table, regardless."[]- (if *mocking-tables*- ;; The atoms are the only source of truth, create a new table if necessary.- (or (pending-table)- (let [table-name (gen-table-name)]- (create-table! table-name)- (swap! *indexes* assoc :pending table-name)))- ;; The database is the source of truth- (let [{:keys [pending]} (sync-tracking-atoms!)]- (or pending+ (locking *indexes*+ (if *mocking-tables*+ ;; In a test where the atoms are the source of truth, create a new table if necessary.+ (or (pending-table)(let [table-name (gen-table-name)]- (log/infof "Creating pending index %s for lang %s" table-name (i18n/site-locale-string))+ (create-table! table-name)+ (swap! *indexes* assoc :pending table-name) table-name))+ ;; The database is the source of truth+ (let [{:keys [pending]} (sync-tracking-atoms!)]+ (or pending+ (let [table-name (gen-table-name)]+ (log/infof "Creating pending index %s for lang %s" table-name (i18n/site-locale-string));; We may fail to insert a new metadata row if we lose a race with another instance.- (when (search-index-metadata/create-pending! :appdb *index-version-id* table-name)- (try- (create-table! table-name)- (catch Exception e- (log/error e "Error creating pending index table, cleaning up metadata")- (try- (t2/with-connection [safe-conn (mdb/app-db)]- (t2/delete! :conn safe-conn :model/SearchIndexMetadata :index_name (name table-name)))- (catch Exception del-e- (log/warn del-e "Error clearing out search metadata after failure")))- (sync-tracking-atoms!))))- (let [pending (:pending (sync-tracking-atoms!))]- (log/infof "New pending index %s" pending)- pending))))))+ (when (search-index-metadata/create-pending! :appdb *index-version-id* table-name)+ (try+ (create-table! table-name)+ (catch Exception e+ (log/error e "Error creating pending index table, cleaning up metadata")+ (try+ (t2/with-connection [safe-conn (mdb/app-db)]+ (t2/delete! :conn safe-conn :model/SearchIndexMetadata :index_name (name table-name)))+ (catch Exception del-e+ (log/warn del-e "Error clearing out search metadata after failure")))+ (sync-tracking-atoms!))))+ (let [pending (:pending (sync-tracking-atoms!))]+ (log/infof "New pending index %s" pending)+ pending)))))))(defn activate-table!"Make the pending index active if it exists. Returns true if it did so."[]- (if *mocking-tables*- ;; The atoms are the only source of truth, we must not update the metadata.- (boolean- (when-let [pending (:pending @*indexes*)]- (reset! *indexes* {:pending nil, :active pending})))- ;; Ensure the metadata is updated and pruned.- (let [{:keys [pending]} (sync-tracking-atoms!)]- (log/infof "Activating pending index %s" pending)- (when pending- (let [active (keyword (search-index-metadata/active-pending! :appdb *index-version-id*))]- (reset! *indexes* {:pending nil :active active})- (log/infof "Activated pending index %s" active)))- ;; Clean up while we're here- (delete-obsolete-tables!)- ;; Did *we* do a rotation?- (boolean pending))))+ (locking *indexes*+ (if *mocking-tables*+ ;; The atoms are the only source of truth, we must not update the metadata.+ (boolean+ (when-let [pending (:pending @*indexes*)]+ (reset! *indexes* {:pending nil, :active pending}) true))+ ;; Ensure the metadata is updated and pruned.+ (let [{:keys [pending]} (sync-tracking-atoms!)]+ (log/infof "Activating pending index %s" pending)+ (when pending+ (let [active (keyword (search-index-metadata/active-pending! :appdb *index-version-id*))]+ (reset! *indexes* {:pending nil :active active})+ (log/infof "Activated pending index %s" active)))+ ;; Clean up while we're here+ (delete-obsolete-tables!)+ ;; Did *we* do a rotation?+ (boolean pending)))))(defn- document->entry [entity](-> entitydiff --git a/src/metabase/search/core.clj b/src/metabase/search/core.cljindex 0622d63d6ad..4d7d5a96c94 100644--- a/src/metabase/search/core.clj+++ b/src/metabase/search/core.clj@@ -133,7 +133,13 @@Respects `search.ingestion/*force-sync*` and waits for the future if it's true.Alternately, if `:async?` is false, it will also run synchronously."[& {:keys [async?] :or {async? true} :as opts}]- (let [f #(reindex-logic! opts)]+ (let [f (fn []+ (try+ (reindex-logic! opts)+ (catch Exception e+ (log/error e "Reindex failed")+ (analytics/inc! :metabase-search/index-error)+ (throw e))))](if (or search.ingestion/*force-sync* (not async?))(doto (promise) (deliver (f)))(future (f)))))