Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

automatic search index versions (#66898)

John Swansonc6efa84b64d4
diff --git a/src/metabase/search/appdb/core.clj b/src/metabase/search/appdb/core.clj
index 12685ebd5aa..30aa8e278ef 100644
--- a/src/metabase/search/appdb/core.clj
+++ b/src/metabase/search/appdb/core.clj
@@ -16,6 +16,7 @@
[metabase.search.ingestion :as search.ingestion]
[metabase.search.permissions :as search.permissions]
[metabase.search.settings :as search.settings]
+ [metabase.search.spec :as search.spec]
[metabase.search.util :as search.util]
[metabase.settings.core :as setting]
[metabase.util :as u]
@@ -137,7 +138,7 @@
(throw (ex-info "Search Index not found."
{:search-engine search-engine
:db-type (mdb/db-type)
- :version @#'search.index/*index-version-id*
+ :version (search.spec/index-version-hash)
:lang_code (i18n/site-locale-string)
:forced-init? init-now?
:index-state-before index-state
diff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.clj
index d172f4b9716..800c1204ecf 100644
--- a/src/metabase/search/appdb/index.clj
+++ b/src/metabase/search/appdb/index.clj
@@ -36,12 +36,6 @@
(def ^:private sync-tracking-period (long (* 5 #_minutes 60e9)))
-; The version ID MUST be updated whenever there is an incompatible change to the schema OR content the index table
-; The id can be any string, but a simple incrementing number is normally easiest to manage while giving some semantic meaning.
-; When this version changes, on startup metabase will not use the existing index table and instead reindex everything to a new table marked with the new version.
-; It is dynamic to allow tests to override it
-(defonce ^:dynamic ^:private *index-version-id* "3")
-
(defonce ^:private next-sync-at (atom nil))
(defonce ^:dynamic ^:private ^{:doc "This atom is often reset! in threads, so modifications should be done only when locking it first."}
@@ -61,7 +55,7 @@
;; Locks the indexes so the reset! doesn't lose data written to the db by a different thread between the read and write
(locking *indexes*
(let [indexes (into {}
- (for [[status table-name] (search-index-metadata/indexes :appdb *index-version-id*)]
+ (for [[status table-name] (search-index-metadata/indexes :appdb (search.spec/index-version-hash))]
(if (exists? table-name)
[status (keyword table-name)]
;; For debugging, make it clear why we are not tracking the given metadata.
@@ -134,7 +128,7 @@
(defn- delete-obsolete-tables! []
;; Delete metadata around indexes that are no longer needed.
- (search-index-metadata/delete-obsolete! *index-version-id*)
+ (search-index-metadata/delete-obsolete! (search.spec/index-version-hash))
;; Drop any indexes that are no longer referenced.
(let [dropped (volatile! [])]
(doseq [table (orphan-indexes)]
@@ -212,7 +206,7 @@
(let [table-name (gen-table-name)]
(log/infof "Creating pending index %s for lang %s" table-name (i18n/site-locale-string))
;; We may fail to insert a new metadata row if we lose a race with another instance.
- (when (search-index-metadata/create-pending! :appdb *index-version-id* table-name)
+ (when (search-index-metadata/create-pending! :appdb (search.spec/index-version-hash) table-name)
(try
(create-table! table-name)
(catch Exception e
@@ -240,7 +234,7 @@
(let [{:keys [pending]} (sync-tracking-atoms!)]
(log/infof "Activating pending index %s" pending)
(when pending
- (let [active (keyword (search-index-metadata/active-pending! :appdb *index-version-id*))]
+ (let [active (keyword (search-index-metadata/active-pending! :appdb (search.spec/index-version-hash)))]
(reset! *indexes* {:pending nil :active active})
(log/infof "Activated pending index %s" active)))
;; Clean up while we're here
@@ -353,7 +347,7 @@
(t2/select-one-fn :created_at
:model/SearchIndexMetadata
:engine :appdb
- :version *index-version-id*
+ :version (search.spec/index-version-hash)
:lang_code (i18n/site-locale-string)
:status :active
{:order-by [[:created_at :desc]]}))
@@ -375,13 +369,13 @@
(defn reset-index!
"Ensure we have a blank slate; in case the table schema or stored data format has changed."
[]
- (log/infof "Resetting appdb index for version %s, active table: %s" *index-version-id*
+ (log/infof "Resetting appdb index for version %s, active table: %s" (search.spec/index-version-hash)
(pr-str (active-table)))
(letfn [(reset-logic []
;; stop tracking any pending table
(when-let [table-name (pending-table)]
(when-not *mocking-tables*
- (let [deleted (search-index-metadata/delete-index! :appdb *index-version-id* table-name)]
+ (let [deleted (search-index-metadata/delete-index! :appdb (search.spec/index-version-hash) table-name)]
(when (pos? deleted)
(log/infof "Deleted %d pending indices" deleted))))
(swap! *indexes* assoc :pending nil))
diff --git a/src/metabase/search/spec.clj b/src/metabase/search/spec.clj
index 5c5d1c53ee7..6ced6e2d1aa 100644
--- a/src/metabase/search/spec.clj
+++ b/src/metabase/search/spec.clj
@@ -1,5 +1,7 @@
(ns metabase.search.spec
(:require
+ [buddy.core.codecs :as codecs]
+ [buddy.core.hash :as buddy-hash]
[clojure.set :as set]
[clojure.string :as str]
[clojure.walk :as walk]
@@ -7,6 +9,7 @@
[metabase.config.core :as config]
[metabase.search.config :as search.config]
[metabase.util :as u]
+ [metabase.util.json :as json]
[metabase.util.malli.registry :as mr]
[toucan2.core :as t2]
[toucan2.tools.transformed :as t2.transformed]))
@@ -19,6 +22,11 @@
;; table and indexed-entity moved to the end because there can be a large number of them
true (conj "table" "indexed-entity" "metric" "card" "dataset")))
+(def raw-spec-forms
+ "Stores the raw (unevaluated) spec forms captured at macro expansion time.
+ Used to compute a deterministic hash for index versioning."
+ (atom {}))
+
(def ^:private search-model->toucan-model
(into {}
(map (fn [search-model]
@@ -350,13 +358,16 @@
- `:column_name` - Use specified database column
- `{:fn function :fields [:field1 :field2]}` - Execute a clojure funtion at index time with the given fields"
[search-model spec]
- `(let [spec# (-> ~spec
- (assoc :name ~search-model)
- (update :visibility #(or % :all))
- (update :attrs #(merge ~default-attrs %)))]
- (validate-spec! spec#)
- (derive (:model spec#) :hook/search-index)
- (defmethod spec* ~search-model [~'_] spec#)))
+ `(do
+ ;; Capture raw form before evaluation (symbols stay as symbols, not function objects)
+ (swap! raw-spec-forms assoc ~search-model '~spec)
+ (let [spec# (-> ~spec
+ (assoc :name ~search-model)
+ (update :visibility #(or % :all))
+ (update :attrs #(merge ~default-attrs %)))]
+ (validate-spec! spec#)
+ (derive (:model spec#) :hook/search-index)
+ (defmethod spec* ~search-model [~'_] spec#))))
;; TODO we should memoize this for production (based on spec values)
(defn model-hooks
@@ -410,3 +421,40 @@
"Transform CamelCase into 'CamelCase Camel Case' so that every word can be searchable"
[s]
(str s " " (str/replace s #"([a-z])([A-Z])" "$1 $2")))
+
+;;;; index version hashing
+
+(defn- canonicalize
+ "Convert a form to a canonical, JSON-serializable representation.
+ Symbols and keywords become strings, maps are sorted."
+ [form]
+ (cond
+ (symbol? form) (str form)
+ (keyword? form) (str form)
+ (map? form) (into (sorted-map)
+ (for [[k v] form]
+ [(canonicalize k) (canonicalize v)]))
+ (sequential? form) (mapv canonicalize form)
+ :else form))
+
+(def ^:dynamic *testing-only-index-version-hash*
+ "Override for tests that need a specific index version."
+ nil)
+
+(def ^{:arglists '([testing-only-index-version-hash]) :private true} index-version-hash*
+ (memoize (fn [testing-only-index-version-hash]
+ (or testing-only-index-version-hash
+ (let [data {:specs @raw-spec-forms
+ :default-attrs default-attrs
+ :attr-types attr-types}]
+ (-> data
+ canonicalize
+ json/encode
+ buddy-hash/sha256
+ codecs/bytes->hex))))))
+
+(defn index-version-hash
+ "Compute a deterministic hash of all search specifications.
+ Includes raw spec forms, default-attrs, and attr-types."
+ []
+ (index-version-hash* *testing-only-index-version-hash*))
diff --git a/test/metabase/search/appdb/index_test.clj b/test/metabase/search/appdb/index_test.clj
index 6236db9a6c9..1c7544c0056 100644
--- a/test/metabase/search/appdb/index_test.clj
+++ b/test/metabase/search/appdb/index_test.clj
@@ -570,7 +570,7 @@
(deftest auto-refresh-test
(when (search/supports-index?)
- (binding [search.index/*index-version-id* "auto-refresh-test"]
+ (binding [search.spec/*testing-only-index-version-hash* "auto-refresh-test"]
(try
(reset! @#'search.index/next-sync-at nil)
(search.index/reset-index!)
@@ -578,7 +578,7 @@
active-after (search.index/gen-table-name)
pending-after (search.index/gen-table-name)
period @#'search.index/sync-tracking-period
- version @#'search.index/*index-version-id*]
+ version (search.spec/index-version-hash)]
(search-index-metadata/create-pending! :appdb version active-after)
(search.index/create-table! active-after)
(search-index-metadata/active-pending! :appdb version)
@@ -595,14 +595,14 @@
(deftest pending-table-expiry-test
(when (search/supports-index?)
- (binding [search.index/*index-version-id* "pending-timeout-test"]
+ (binding [search.spec/*testing-only-index-version-hash* "pending-timeout-test"]
(try
(reset! @#'search.index/next-sync-at nil)
(search.index/reset-index!)
(let [active-table (search.index/active-table)
pending-old (search.index/gen-table-name)
pending-new (search.index/gen-table-name)
- version @#'search.index/*index-version-id*]
+ version (search.spec/index-version-hash)]
;; Set up old pending table (more than a day old)
(search.index/create-table! pending-old)
@@ -632,10 +632,10 @@
(deftest when-index-created
(when (search/supports-index?)
- (binding [search.index/*index-version-id* "index-age-test"]
+ (binding [search.spec/*testing-only-index-version-hash* "index-age-test"]
(try
(let [table-name (search.index/gen-table-name)
- version @#'search.index/*index-version-id*]
+ version (search.spec/index-version-hash)]
(testing "Nil age if no active table"
(is (nil? (#'search.index/when-index-created))))
diff --git a/test/metabase/search/spec_test.clj b/test/metabase/search/spec_test.clj
index b39566a0936..08d0c4ccf76 100644
--- a/test/metabase/search/spec_test.clj
+++ b/test/metabase/search/spec_test.clj
@@ -143,3 +143,11 @@
(is (actual-models em))))
(testing "... and nothing else does"
(is (empty? (sort-by name (remove expected-models actual-models))))))))
+
+(deftest ^:parallel index-version-hash-test
+ (testing "index-version-hash returns a consistent value"
+ (let [hash1 (search.spec/index-version-hash)
+ hash2 (search.spec/index-version-hash)]
+ (is (string? hash1))
+ (is (= 64 (count hash1)) "SHA-256 hex string should be 64 characters")
+ (is (= hash1 hash2) "Hash should be deterministic"))))