The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
automatic search index versions (#66898)
John Swansonc6efa84b64d4
diff --git a/src/metabase/search/appdb/core.clj b/src/metabase/search/appdb/core.cljindex 12685ebd5aa..30aa8e278ef 100644--- a/src/metabase/search/appdb/core.clj+++ b/src/metabase/search/appdb/core.clj@@ -16,6 +16,7 @@[metabase.search.ingestion :as search.ingestion][metabase.search.permissions :as search.permissions][metabase.search.settings :as search.settings]+ [metabase.search.spec :as search.spec][metabase.search.util :as search.util][metabase.settings.core :as setting][metabase.util :as u]@@ -137,7 +138,7 @@(throw (ex-info "Search Index not found."{:search-engine search-engine:db-type (mdb/db-type)- :version @#'search.index/*index-version-id*+ :version (search.spec/index-version-hash):lang_code (i18n/site-locale-string):forced-init? init-now?:index-state-before index-statediff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.cljindex d172f4b9716..800c1204ecf 100644--- a/src/metabase/search/appdb/index.clj+++ b/src/metabase/search/appdb/index.clj@@ -36,12 +36,6 @@(def ^:private sync-tracking-period (long (* 5 #_minutes 60e9)))-; The version ID MUST be updated whenever there is an incompatible change to the schema OR content the index table-; The id can be any string, but a simple incrementing number is normally easiest to manage while giving some semantic meaning.-; When this version changes, on startup metabase will not use the existing index table and instead reindex everything to a new table marked with the new version.-; It is dynamic to allow tests to override it-(defonce ^:dynamic ^:private *index-version-id* "3")-(defonce ^:private next-sync-at (atom nil))(defonce ^:dynamic ^:private ^{:doc "This atom is often reset! in threads, so modifications should be done only when locking it first."}@@ -61,7 +55,7 @@;; Locks the indexes so the reset! doesn't lose data written to the db by a different thread between the read and write(locking *indexes*(let [indexes (into {}- (for [[status table-name] (search-index-metadata/indexes :appdb *index-version-id*)]+ (for [[status table-name] (search-index-metadata/indexes :appdb (search.spec/index-version-hash))](if (exists? table-name)[status (keyword table-name)];; For debugging, make it clear why we are not tracking the given metadata.@@ -134,7 +128,7 @@(defn- delete-obsolete-tables! [];; Delete metadata around indexes that are no longer needed.- (search-index-metadata/delete-obsolete! *index-version-id*)+ (search-index-metadata/delete-obsolete! (search.spec/index-version-hash));; Drop any indexes that are no longer referenced.(let [dropped (volatile! [])](doseq [table (orphan-indexes)]@@ -212,7 +206,7 @@(let [table-name (gen-table-name)](log/infof "Creating pending index %s for lang %s" table-name (i18n/site-locale-string));; We may fail to insert a new metadata row if we lose a race with another instance.- (when (search-index-metadata/create-pending! :appdb *index-version-id* table-name)+ (when (search-index-metadata/create-pending! :appdb (search.spec/index-version-hash) table-name)(try(create-table! table-name)(catch Exception e@@ -240,7 +234,7 @@(let [{:keys [pending]} (sync-tracking-atoms!)](log/infof "Activating pending index %s" pending)(when pending- (let [active (keyword (search-index-metadata/active-pending! :appdb *index-version-id*))]+ (let [active (keyword (search-index-metadata/active-pending! :appdb (search.spec/index-version-hash)))](reset! *indexes* {:pending nil :active active})(log/infof "Activated pending index %s" active)));; Clean up while we're here@@ -353,7 +347,7 @@(t2/select-one-fn :created_at:model/SearchIndexMetadata:engine :appdb- :version *index-version-id*+ :version (search.spec/index-version-hash):lang_code (i18n/site-locale-string):status :active{:order-by [[:created_at :desc]]}))@@ -375,13 +369,13 @@(defn reset-index!"Ensure we have a blank slate; in case the table schema or stored data format has changed."[]- (log/infof "Resetting appdb index for version %s, active table: %s" *index-version-id*+ (log/infof "Resetting appdb index for version %s, active table: %s" (search.spec/index-version-hash)(pr-str (active-table)))(letfn [(reset-logic [];; stop tracking any pending table(when-let [table-name (pending-table)](when-not *mocking-tables*- (let [deleted (search-index-metadata/delete-index! :appdb *index-version-id* table-name)]+ (let [deleted (search-index-metadata/delete-index! :appdb (search.spec/index-version-hash) table-name)](when (pos? deleted)(log/infof "Deleted %d pending indices" deleted))))(swap! *indexes* assoc :pending nil))diff --git a/src/metabase/search/spec.clj b/src/metabase/search/spec.cljindex 5c5d1c53ee7..6ced6e2d1aa 100644--- a/src/metabase/search/spec.clj+++ b/src/metabase/search/spec.clj@@ -1,5 +1,7 @@(ns metabase.search.spec(:require+ [buddy.core.codecs :as codecs]+ [buddy.core.hash :as buddy-hash][clojure.set :as set][clojure.string :as str][clojure.walk :as walk]@@ -7,6 +9,7 @@[metabase.config.core :as config][metabase.search.config :as search.config][metabase.util :as u]+ [metabase.util.json :as json][metabase.util.malli.registry :as mr][toucan2.core :as t2][toucan2.tools.transformed :as t2.transformed]))@@ -19,6 +22,11 @@;; table and indexed-entity moved to the end because there can be a large number of themtrue (conj "table" "indexed-entity" "metric" "card" "dataset")))+(def raw-spec-forms+ "Stores the raw (unevaluated) spec forms captured at macro expansion time.+ Used to compute a deterministic hash for index versioning."+ (atom {}))+(def ^:private search-model->toucan-model(into {}(map (fn [search-model]@@ -350,13 +358,16 @@- `:column_name` - Use specified database column- `{:fn function :fields [:field1 :field2]}` - Execute a clojure funtion at index time with the given fields"[search-model spec]- `(let [spec# (-> ~spec- (assoc :name ~search-model)- (update :visibility #(or % :all))- (update :attrs #(merge ~default-attrs %)))]- (validate-spec! spec#)- (derive (:model spec#) :hook/search-index)- (defmethod spec* ~search-model [~'_] spec#)))+ `(do+ ;; Capture raw form before evaluation (symbols stay as symbols, not function objects)+ (swap! raw-spec-forms assoc ~search-model '~spec)+ (let [spec# (-> ~spec+ (assoc :name ~search-model)+ (update :visibility #(or % :all))+ (update :attrs #(merge ~default-attrs %)))]+ (validate-spec! spec#)+ (derive (:model spec#) :hook/search-index)+ (defmethod spec* ~search-model [~'_] spec#))));; TODO we should memoize this for production (based on spec values)(defn model-hooks@@ -410,3 +421,40 @@"Transform CamelCase into 'CamelCase Camel Case' so that every word can be searchable"[s](str s " " (str/replace s #"([a-z])([A-Z])" "$1 $2")))++;;;; index version hashing++(defn- canonicalize+ "Convert a form to a canonical, JSON-serializable representation.+ Symbols and keywords become strings, maps are sorted."+ [form]+ (cond+ (symbol? form) (str form)+ (keyword? form) (str form)+ (map? form) (into (sorted-map)+ (for [[k v] form]+ [(canonicalize k) (canonicalize v)]))+ (sequential? form) (mapv canonicalize form)+ :else form))++(def ^:dynamic *testing-only-index-version-hash*+ "Override for tests that need a specific index version."+ nil)++(def ^{:arglists '([testing-only-index-version-hash]) :private true} index-version-hash*+ (memoize (fn [testing-only-index-version-hash]+ (or testing-only-index-version-hash+ (let [data {:specs @raw-spec-forms+ :default-attrs default-attrs+ :attr-types attr-types}]+ (-> data+ canonicalize+ json/encode+ buddy-hash/sha256+ codecs/bytes->hex))))))++(defn index-version-hash+ "Compute a deterministic hash of all search specifications.+ Includes raw spec forms, default-attrs, and attr-types."+ []+ (index-version-hash* *testing-only-index-version-hash*))diff --git a/test/metabase/search/appdb/index_test.clj b/test/metabase/search/appdb/index_test.cljindex 6236db9a6c9..1c7544c0056 100644--- a/test/metabase/search/appdb/index_test.clj+++ b/test/metabase/search/appdb/index_test.clj@@ -570,7 +570,7 @@(deftest auto-refresh-test(when (search/supports-index?)- (binding [search.index/*index-version-id* "auto-refresh-test"]+ (binding [search.spec/*testing-only-index-version-hash* "auto-refresh-test"](try(reset! @#'search.index/next-sync-at nil)(search.index/reset-index!)@@ -578,7 +578,7 @@active-after (search.index/gen-table-name)pending-after (search.index/gen-table-name)period @#'search.index/sync-tracking-period- version @#'search.index/*index-version-id*]+ version (search.spec/index-version-hash)](search-index-metadata/create-pending! :appdb version active-after)(search.index/create-table! active-after)(search-index-metadata/active-pending! :appdb version)@@ -595,14 +595,14 @@(deftest pending-table-expiry-test(when (search/supports-index?)- (binding [search.index/*index-version-id* "pending-timeout-test"]+ (binding [search.spec/*testing-only-index-version-hash* "pending-timeout-test"](try(reset! @#'search.index/next-sync-at nil)(search.index/reset-index!)(let [active-table (search.index/active-table)pending-old (search.index/gen-table-name)pending-new (search.index/gen-table-name)- version @#'search.index/*index-version-id*]+ version (search.spec/index-version-hash)];; Set up old pending table (more than a day old)(search.index/create-table! pending-old)@@ -632,10 +632,10 @@(deftest when-index-created(when (search/supports-index?)- (binding [search.index/*index-version-id* "index-age-test"]+ (binding [search.spec/*testing-only-index-version-hash* "index-age-test"](try(let [table-name (search.index/gen-table-name)- version @#'search.index/*index-version-id*]+ version (search.spec/index-version-hash)](testing "Nil age if no active table"(is (nil? (#'search.index/when-index-created))))diff --git a/test/metabase/search/spec_test.clj b/test/metabase/search/spec_test.cljindex b39566a0936..08d0c4ccf76 100644--- a/test/metabase/search/spec_test.clj+++ b/test/metabase/search/spec_test.clj@@ -143,3 +143,11 @@(is (actual-models em))))(testing "... and nothing else does"(is (empty? (sort-by name (remove expected-models actual-models))))))))++(deftest ^:parallel index-version-hash-test+ (testing "index-version-hash returns a consistent value"+ (let [hash1 (search.spec/index-version-hash)+ hash2 (search.spec/index-version-hash)]+ (is (string? hash1))+ (is (= 64 (count hash1)) "SHA-256 hex string should be 64 characters")+ (is (= hash1 hash2) "Hash should be deterministic"))))