Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Create HNSW index only when explicitly using it (#76387)

Chris Truter6ca4f480ffd2
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index cf1b48e98a0..297b0c2824c 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -3458,6 +3458,7 @@
collections
config
connection-pool
+ events
llm
metabot
models
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
index 2dce56af79d..42adcc8ee17 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
@@ -11,6 +11,7 @@
[metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
[metabase-enterprise.semantic-search.repair :as semantic.repair]
[metabase-enterprise.semantic-search.settings :as semantic.settings]
+ [metabase-enterprise.semantic-search.util :as semantic.util]
[metabase.analytics-interface.core :as analytics]
[metabase.premium-features.core :refer [defenterprise]]
[metabase.search.config :as search.config]
@@ -45,6 +46,21 @@
(some? semantic.db.datasource/db-url)
(semantic.settings/semantic-search-enabled)))
+(defn build-hnsw-index-async!
+ "Build the HNSW index on the active semantic search index in the background, returning promptly.
+
+ No-ops when semantic search isn't available on this instance. Backs the just-in-time HNSW build, which
+ runs only when an instance is configured to the `:hnsw` vector-search strategy."
+ []
+ (when (semantic.util/semantic-search-available?)
+ (future
+ (try
+ (semantic.pgvector-api/ensure-active-hnsw-index! (semantic.env/get-pgvector-datasource!)
+ (semantic.env/get-index-metadata))
+ (catch Throwable t
+ (log/error t "Failed to build HNSW index for semantic search")))))
+ nil)
+
(defn- with-zero-semantic-distance-score
"Record a 0 `:semantic-distance` score on `results` that lack it, for a consistent merged-result score breakdown."
[search-ctx results]
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/events.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/events.clj
new file mode 100644
index 00000000000..072357f36d7
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/events.clj
@@ -0,0 +1,13 @@
+(ns metabase-enterprise.semantic-search.events
+ "Event handlers for the semantic-search module."
+ (:require
+ [metabase-enterprise.semantic-search.core :as semantic.core]
+ [metabase.events.core :as events]
+ [methodical.core :as methodical]))
+
+(derive :event/semantic-search-hnsw-enabled ::build-hnsw-index)
+
+;; Handled via an event so settings stays decoupled from the index-building code, which requires settings.
+(methodical/defmethod events/publish-event! ::build-hnsw-index
+ [_topic _event]
+ (semantic.core/build-hnsw-index-async!))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
index 6bb4dbb45fb..b7718e4a250 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
@@ -12,6 +12,7 @@
[metabase-enterprise.semantic-search.embedding :as embedding]
[metabase-enterprise.semantic-search.scoring :as scoring]
[metabase-enterprise.semantic-search.settings :as semantic-settings]
+ [metabase-enterprise.semantic-search.util :as semantic.util]
[metabase.analytics-interface.core :as analytics]
[metabase.models.interface :as mi]
[metabase.permissions.core :as perms]
@@ -466,9 +467,33 @@
[index]
(index-name index "_content_idx"))
+(defn create-hnsw-index-if-not-exists!
+ "Create the HNSW index on the `embedding` column of `index`'s table, if it does not already exist.
+
+ HNSW indexes are expensive to build and maintain, so we only create one when an instance is configured to
+ use the `:hnsw` vector-search strategy (see [[create-index-table-if-not-exists!]] and the async build
+ triggered by [[metabase-enterprise.semantic-search.settings/semantic-search-vector-strategy]]).
+
+ Pass `concurrently? true` to build with `CREATE INDEX CONCURRENTLY` (for a populated table whose writes
+ shouldn't be locked out); it must run outside a transaction."
+ [connectable index & {:keys [concurrently?] :or {concurrently? false}}]
+ (let [{:keys [table-name]} index
+ ;; HoneySQL emits CONCURRENTLY in the wrong position (before INDEX), so splice it into the
+ ;; formatted statement instead: `CREATE INDEX CONCURRENTLY IF NOT EXISTS ...` is valid Postgres.
+ [sql & params] (sql-format-quoted
+ (sql.helpers/create-index
+ [(keyword (hnsw-index-name index)) :if-not-exists]
+ [(keyword table-name) :using-hnsw [[:raw "embedding vector_cosine_ops"]]]))
+ sql (cond-> sql
+ concurrently? (str/replace-first "CREATE INDEX " "CREATE INDEX CONCURRENTLY "))]
+ (jdbc/execute! connectable (into [sql] params))))
+
(defn create-index-table-if-not-exists!
"Ensure that the index table exists and is ready to be populated. If
- force-reset? is true, drops and recreates the table if it exists."
+ force-reset? is true, drops and recreates the table if it exists.
+
+ The HNSW index is only created when the instance is configured for the `:hnsw` vector-search strategy;
+ otherwise it is built just-in-time when the strategy is configured (see [[create-hnsw-index-if-not-exists!]])."
[connectable index & {:keys [force-reset?] :or {force-reset? false}}]
(try
(let [{:keys [embedding-model table-name]} index
@@ -483,12 +508,8 @@
(-> (sql.helpers/create-table (keyword table-name) :if-not-exists)
(sql.helpers/with-columns (index-table-schema vector-dimensions))
sql-format-quoted))
- (jdbc/execute!
- connectable
- (-> (sql.helpers/create-index
- [(keyword (hnsw-index-name index)) :if-not-exists]
- [(keyword table-name) :using-hnsw [[:raw "embedding vector_cosine_ops"]]])
- sql-format-quoted))
+ (when (= :hnsw (semantic-settings/semantic-search-vector-strategy))
+ (create-hnsw-index-if-not-exists! connectable index))
(jdbc/execute!
connectable
(-> (sql.helpers/create-index
@@ -910,72 +931,78 @@
search-string (:search-string search-context)]
(if (str/blank? search-string)
{:results [] :raw-count 0}
- (let [timer (u/start-timer)
-
- ;; Warm the pool concurrently with the embedding round-trip: the pool holds zero idle
- ;; connections by default (see semantic-search.db.datasource), so the first search after an
- ;; idle period would otherwise pay the connection handshake serially on top of the embedding
- ;; latency.
- _ (warm-connection-pool-async! db)
- embedding (tracing/with-span :search "search.semantic.embedding"
- {:search.semantic/provider (:provider embedding-model)
- :search.semantic/model-name (:model-name embedding-model)}
- (embedding/get-embedding embedding-model
- (embedding/prefix-search-query embedding-model search-string)
- {:type :query :record-tokens? true}))
- embedding-time-ms (u/since-ms timer)
-
- db-timer (u/start-timer)
- weights (search.config/weights search-context)
- scorers (scoring/semantic-scorers (:table-name index) search-context)
- query (scored-search-query index embedding search-context scorers)
- xform (comp (map decode-legacy-input)
- (map (partial legacy-input-with-score weights (keys scorers))))
- reducible (reducible-search-query db query)
- raw-results (tracing/with-span :search "search.semantic.db-query"
- {:search/query-length (count search-string)}
- (into [] xform reducible))
- db-query-time-ms (u/since-ms db-timer)
-
- filter-timer (u/start-timer)
- filtered-results (tracing/with-span :search "search.semantic.permission-filter"
- {:search.semantic/raw-count (count raw-results)}
- (->> raw-results
- filter-read-permitted
- (apply-collection-id-filter search-context)
- (mapv search/collapse-id)))
- filter-time-ms (u/since-ms filter-timer)
-
- appdb-scorers (scoring/appdb-scorers search-context)
- appdb-scores-timer (u/start-timer)
- final-results (->> filtered-results
- (scoring/with-appdb-scores search-context appdb-scorers weights))
- appdb-scores-time-ms (u/since-ms appdb-scores-timer)
- total-time-ms (u/since-ms timer)]
- (log/debug "Semantic search"
- {:search-string-length (count search-string)
- :raw-results-count (count raw-results)
- :final-results-count (count final-results)
- :embedding-time-ms embedding-time-ms
- :db-query-time-ms db-query-time-ms
- :filter-time-ms filter-time-ms
- :appdb-scores-time-ms appdb-scores-time-ms
- :total-time-ms total-time-ms})
- (analytics/inc! :metabase-search/semantic-embedding-ms
- {:embedding-model (:name embedding-model)}
- embedding-time-ms)
- (analytics/inc! :metabase-search/semantic-db-query-ms
- {:embedding-model (:name embedding-model)}
- db-query-time-ms)
- (analytics/inc! :metabase-search/semantic-appdb-scores-ms
- appdb-scores-time-ms)
- (analytics/inc! :metabase-search/semantic-search-ms
- {:embedding-model (:name embedding-model)}
- total-time-ms)
- (comment
- (jdbc/execute! db (sql-format-quoted query)))
- {:results final-results
- :raw-count (count raw-results)}))))
+ (do
+ (when (and (= :hnsw (vector-search-strategy search-context))
+ (not (semantic.util/index-exists? db (hnsw-index-name index))))
+ (throw (ex-info (str "HNSW vector-search strategy requested but no HNSW index exists. "
+ "Set the semantic-search-vector-strategy setting to :hnsw to build it.")
+ {:table-name (:table-name index)})))
+ (let [timer (u/start-timer)
+
+ ;; Warm the pool concurrently with the embedding round-trip: the pool holds zero idle
+ ;; connections by default (see semantic-search.db.datasource), so the first search after an
+ ;; idle period would otherwise pay the connection handshake serially on top of the embedding
+ ;; latency.
+ _ (warm-connection-pool-async! db)
+ embedding (tracing/with-span :search "search.semantic.embedding"
+ {:search.semantic/provider (:provider embedding-model)
+ :search.semantic/model-name (:model-name embedding-model)}
+ (embedding/get-embedding embedding-model
+ (embedding/prefix-search-query embedding-model search-string)
+ {:type :query :record-tokens? true}))
+ embedding-time-ms (u/since-ms timer)
+
+ db-timer (u/start-timer)
+ weights (search.config/weights search-context)
+ scorers (scoring/semantic-scorers (:table-name index) search-context)
+ query (scored-search-query index embedding search-context scorers)
+ xform (comp (map decode-legacy-input)
+ (map (partial legacy-input-with-score weights (keys scorers))))
+ reducible (reducible-search-query db query)
+ raw-results (tracing/with-span :search "search.semantic.db-query"
+ {:search/query-length (count search-string)}
+ (into [] xform reducible))
+ db-query-time-ms (u/since-ms db-timer)
+
+ filter-timer (u/start-timer)
+ filtered-results (tracing/with-span :search "search.semantic.permission-filter"
+ {:search.semantic/raw-count (count raw-results)}
+ (->> raw-results
+ filter-read-permitted
+ (apply-collection-id-filter search-context)
+ (mapv search/collapse-id)))
+ filter-time-ms (u/since-ms filter-timer)
+
+ appdb-scorers (scoring/appdb-scorers search-context)
+ appdb-scores-timer (u/start-timer)
+ final-results (->> filtered-results
+ (scoring/with-appdb-scores search-context appdb-scorers weights))
+ appdb-scores-time-ms (u/since-ms appdb-scores-timer)
+ total-time-ms (u/since-ms timer)]
+ (log/debug "Semantic search"
+ {:search-string-length (count search-string)
+ :raw-results-count (count raw-results)
+ :final-results-count (count final-results)
+ :embedding-time-ms embedding-time-ms
+ :db-query-time-ms db-query-time-ms
+ :filter-time-ms filter-time-ms
+ :appdb-scores-time-ms appdb-scores-time-ms
+ :total-time-ms total-time-ms})
+ (analytics/inc! :metabase-search/semantic-embedding-ms
+ {:embedding-model (:name embedding-model)}
+ embedding-time-ms)
+ (analytics/inc! :metabase-search/semantic-db-query-ms
+ {:embedding-model (:name embedding-model)}
+ db-query-time-ms)
+ (analytics/inc! :metabase-search/semantic-appdb-scores-ms
+ appdb-scores-time-ms)
+ (analytics/inc! :metabase-search/semantic-search-ms
+ {:embedding-model (:name embedding-model)}
+ total-time-ms)
+ (comment
+ (jdbc/execute! db (sql-format-quoted query)))
+ {:results final-results
+ :raw-count (count raw-results)})))))
(defn- row-present?
"Whether a single `(model, id)` row survives `where` against the index `table`."
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj
index 30e92d534c0..330cec96293 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj
@@ -1,5 +1,6 @@
(ns metabase-enterprise.semantic-search.init
(:require
+ [metabase-enterprise.semantic-search.events]
[metabase-enterprise.semantic-search.settings]
[metabase-enterprise.semantic-search.task.index-cleanup]
[metabase-enterprise.semantic-search.task.index-repair]
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.clj
index 8fbf30b7f20..7282469e950 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.clj
@@ -61,6 +61,18 @@
(semantic.index-metadata/activate-index! tx index-metadata index-id))
index))
+(defn ensure-active-hnsw-index!
+ "Build the HNSW index on the active index table if it does not already exist.
+
+ Called when an instance is (re)configured to the `:hnsw` vector-search strategy. No-ops when there is no
+ active index. Builds the index with `CREATE INDEX CONCURRENTLY` since the table is typically populated."
+ [pgvector index-metadata]
+ (if-let [{:keys [index]} (semantic.index-metadata/get-active-index-state pgvector index-metadata)]
+ (do
+ (log/info "Building HNSW index for active semantic search index" (:table-name index))
+ (semantic.index/create-hnsw-index-if-not-exists! pgvector index {:concurrently? true}))
+ (log/info "No active semantic search index; skipping HNSW index build")))
+
(defn init-semantic-search!
"Initialises a pgvector database for semantic search if it does not exist and creates an index for the provided
embedding model (if it does not exist).
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj
index ff2d055b6fb..a74a52e09cf 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj
@@ -1,11 +1,16 @@
(ns metabase-enterprise.semantic-search.settings
(:require
+ [metabase.events.core :as events]
[metabase.llm.settings :as llm-settings]
[metabase.premium-features.core :as premium-features]
[metabase.search.config :as search.config]
[metabase.settings.core :as setting :refer [defsetting]]
[metabase.util.i18n :refer [deferred-tru]]))
+;; Topic for the just-in-time HNSW build, handled in metabase-enterprise.semantic-search.events. Declared
+;; here, not there, so it's valid wherever the setter runs regardless of handler-namespace load order.
+(derive :event/semantic-search-hnsw-enabled :metabase/event)
+
(def ^:private valid-embedding-providers
"The set of valid embedding provider names."
#{"ai-service" "openai" "ollama"})
@@ -120,21 +125,26 @@
(deferred-tru
(str "Default vector-search strategy for semantic search: `hnsw` (approximate, HNSW-index-backed) or "
"`brute-force` (exact, applies non-vector filters first then computes cosine distance over the "
- "survivors). Individual requests may override this via the `vector_search_strategy` API parameter."))
+ "survivors). Defaults to `brute-force`, which needs no index; selecting `hnsw` builds the HNSW index "
+ "just-in-time. Individual requests may override this via the `vector_search_strategy` API parameter."))
:type :keyword
- :default :hnsw
+ :default :brute-force
:encryption :no
:export? false
:visibility :internal
:doc false
:setter (fn [new-value]
- (let [kw (some-> new-value keyword)]
+ (let [kw (some-> new-value keyword)
+ old (setting/get-value-of-type :keyword :semantic-search-vector-strategy)]
(when (and kw (not (contains? valid-vector-search-strategies kw)))
(throw (ex-info (str "Invalid vector-search strategy: " (pr-str new-value)
". Valid strategies are: " (pr-str valid-vector-search-strategies))
{:invalid-value new-value
:valid-values valid-vector-search-strategies})))
- (setting/set-value-of-type! :keyword :semantic-search-vector-strategy kw))))
+ (setting/set-value-of-type! :keyword :semantic-search-vector-strategy kw)
+ ;; Gated on the transition (not every set) so re-setting :hnsw doesn't rebuild the index.
+ (when (and (= kw :hnsw) (not= old :hnsw))
+ (events/publish-event! :event/semantic-search-hnsw-enabled {})))))
(defsetting semantic-search-min-results-threshold
(deferred-tru "Minimum number of semantic search results required before falling back to other engines.")
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj
index 50aed9377da..bec65c03f0c 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj
@@ -3,8 +3,10 @@
[clojurewerkz.quartzite.jobs :as jobs]
[clojurewerkz.quartzite.schedule.simple :as simple]
[clojurewerkz.quartzite.triggers :as triggers]
+ [metabase-enterprise.semantic-search.core :as semantic.core]
[metabase-enterprise.semantic-search.env :as semantic.env]
[metabase-enterprise.semantic-search.indexer :as semantic-search.indexer]
+ [metabase-enterprise.semantic-search.settings :as semantic.settings]
[metabase-enterprise.semantic-search.util :as semantic.u]
[metabase.search.engine :as search.engine]
[metabase.task.core :as task]
@@ -66,4 +68,8 @@
(simple/schedule
(simple/with-interval-in-milliseconds (.toMillis run-frequency))
(simple/repeat-forever))))]
- (task/schedule-task! job trigger))))
+ (task/schedule-task! job trigger))
+ ;; Safety net: an instance that booted already configured for :hnsw (strategy set via env var, or set
+ ;; before an active index existed) never saw the setter's transition event, so build the index now.
+ (when (= :hnsw (semantic.settings/semantic-search-vector-strategy))
+ (semantic.core/build-hnsw-index-async!))))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj
index 6615f590be5..7bf0e5dba45 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj
@@ -14,6 +14,15 @@
{:builder-fn jdbc.rs/as-unqualified-lower-maps})
(:table_exists false)))
+(defn index-exists?
+ "Does an index named `index-name` exist in the pgvector DB's pg_indexes?"
+ [pgvector index-name]
+ (-> (jdbc/execute-one! pgvector
+ ["SELECT exists (select 1 FROM pg_indexes WHERE indexname = ?) index_exists"
+ index-name]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})
+ (:index_exists false)))
+
(defn semantic-search-available?
"Predicate to check whether semantic search is available on the instance."
[]
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
index 3c8d40a412b..699e3629012 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
@@ -4,8 +4,11 @@
[clojure.string :as str]
[clojure.test :refer :all]
[honey.sql :as sql]
+ [metabase-enterprise.semantic-search.core :as semantic.core]
[metabase-enterprise.semantic-search.embedding :as semantic.embedding]
[metabase-enterprise.semantic-search.env :as semantic.env]
+ ;; loaded for its :event/semantic-search-hnsw-enabled handler, which the setter test exercises
+ [metabase-enterprise.semantic-search.events]
[metabase-enterprise.semantic-search.index :as semantic.index]
[metabase-enterprise.semantic-search.settings :as semantic.settings]
[metabase-enterprise.semantic-search.spec-trace-test-util :as spec-trace]
@@ -79,24 +82,71 @@
(testing "valid strategies round-trip"
(doseq [strategy [:hnsw :brute-force]]
(mt/with-temporary-setting-values [semantic.settings/semantic-search-vector-strategy strategy]
- (is (= strategy (semantic.settings/semantic-search-vector-strategy)))))))
+ (is (= strategy (semantic.settings/semantic-search-vector-strategy))))))
+ (testing "the setter kicks off the background build job only on the transition into :hnsw"
+ ;; This asserts *when* the build is triggered (the transition gating), not what the build does -- the
+ ;; build itself is covered by pgvector-api-test/ensure-active-hnsw-index!-test. The setter publishes
+ ;; :event/semantic-search-hnsw-enabled, whose handler (semantic-search.events) calls the async build; we
+ ;; spy on that build so the trigger is verified deterministically without spawning a real future.
+ (mt/with-premium-features #{:semantic-search}
+ (let [triggers (atom 0)]
+ (mt/with-dynamic-fn-redefs [semantic.core/build-hnsw-index-async! (fn [] (swap! triggers inc))]
+ (mt/with-temporary-setting-values [semantic.settings/semantic-search-vector-strategy :brute-force]
+ (semantic.settings/semantic-search-vector-strategy! :hnsw)
+ (is (= 1 @triggers) "transitioning :brute-force -> :hnsw kicks off a build")
+ (semantic.settings/semantic-search-vector-strategy! :hnsw)
+ (is (= 1 @triggers) "setting :hnsw again when already :hnsw does not re-trigger")
+ (semantic.settings/semantic-search-vector-strategy! :brute-force)
+ (is (= 1 @triggers) "switching away from :hnsw does not trigger")
+ (semantic.settings/semantic-search-vector-strategy! :hnsw)
+ (is (= 2 @triggers) "transitioning back into :hnsw triggers again")))))))
(deftest create-index-table!-test
(mt/with-premium-features #{:semantic-search}
- (with-open [index-ref (semantic.tu/open-temp-index!)]
- ;; open-temp-index-table! creates the temp table, so drop it in order to test create!.
+ (with-open [index-ref (semantic.tu/open-temp-index! :hnsw? false)]
+ ;; open-temp-index! creates the temp table, so drop it in order to test create!.
(semantic.index/drop-index-table! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index)
(testing "index table is not present before create!"
(is (not (semantic.tu/table-exists-in-db? (:table-name @index-ref))))
(is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref))))
(is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-index-name @index-ref))))
(is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-native-index-name @index-ref)))))
- (testing "index table is present after create!"
- (semantic.index/create-index-table-if-not-exists! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index {:force-reset? false})
+ (testing "under the default :brute-force strategy, create! builds the table and FTS indexes but not the HNSW index"
+ (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly :brute-force)]
+ (semantic.index/create-index-table-if-not-exists! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index {:force-reset? true}))
(is (semantic.tu/table-exists-in-db? (:table-name @index-ref)))
- (is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref)))
+ (is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref))))
(is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-index-name @index-ref)))
- (is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-native-index-name @index-ref)))))))
+ (is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-native-index-name @index-ref))))
+ (testing "when configured for the :hnsw strategy, create! also builds the HNSW index"
+ (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly :hnsw)]
+ (semantic.index/create-index-table-if-not-exists! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index {:force-reset? true}))
+ (is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref)))))))
+
+(deftest create-hnsw-index-if-not-exists!-test
+ (mt/with-premium-features #{:semantic-search}
+ (with-open [index-ref (semantic.tu/open-temp-index! :hnsw? false)]
+ (testing "HNSW index is absent until built explicitly"
+ (is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref))))
+ (semantic.index/create-hnsw-index-if-not-exists! (semantic.env/get-pgvector-datasource!) @index-ref)
+ (is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref))))
+ (testing "is idempotent: a second call does no work (reuses the existing index, no rebuild)"
+ (let [before (semantic.tu/index-relfilenode (semantic.index/hnsw-index-name @index-ref))]
+ (is (some? before) "the index exists before the second call")
+ (semantic.index/create-hnsw-index-if-not-exists! (semantic.env/get-pgvector-datasource!) @index-ref)
+ (is (= before (semantic.tu/index-relfilenode (semantic.index/hnsw-index-name @index-ref)))
+ "relfilenode is unchanged, so the index was not dropped, rebuilt, or reindexed"))))))
+
+(deftest query-index-hnsw-without-index-throws-test
+ (testing "a query using the :hnsw strategy fails fast when no HNSW index exists, rather than silently scanning"
+ (mt/with-premium-features #{:semantic-search}
+ (with-open [index-ref (semantic.tu/open-temp-index! :hnsw? false)]
+ (is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref))))
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo #"no HNSW index exists"
+ (semantic.index/query-index (semantic.env/get-pgvector-datasource!)
+ @index-ref
+ {:search-string "puppy" :vector-search-strategy :hnsw})))))))
(deftest drop-index-table!-test
(mt/with-premium-features #{:semantic-search}
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj
index 9c134ce8069..7edfded2bd1 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj
@@ -86,6 +86,32 @@
(semantic.pgvector-api/init-semantic-search! pgvector index-metadata model1)
(is (true? (@#'semantic.index-metadata/index-table-exists? pgvector index))))))))
+(deftest ensure-active-hnsw-index!-test
+ (mt/with-premium-features #{:semantic-search}
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ index-metadata (semantic.tu/unique-index-metadata)
+ model semantic.tu/mock-embedding-model
+ cleanup (fn [_] (semantic.tu/cleanup-index-metadata! pgvector index-metadata))]
+ (with-open [index-ref ^Closeable (semantic.tu/closeable
+ (semantic.pgvector-api/init-semantic-search! pgvector index-metadata model)
+ cleanup)]
+ (let [index @index-ref
+ hnsw-name (semantic.index/hnsw-index-name index)]
+ (testing "builds the HNSW index on the active index table when it is missing"
+ ;; init only builds the HNSW index when configured for :hnsw, so drop it for a deterministic
+ ;; starting point regardless of the ambient strategy -- mimics a brute-force-configured instance.
+ (jdbc/execute! pgvector [(str "DROP INDEX IF EXISTS \"" hnsw-name "\"")])
+ (is (not (semantic.tu/table-has-index? (:table-name index) hnsw-name)))
+ (semantic.pgvector-api/ensure-active-hnsw-index! pgvector index-metadata)
+ (is (semantic.tu/table-has-index? (:table-name index) hnsw-name)))
+ (testing "is a noop when the HNSW index already exists"
+ (let [before (semantic.tu/index-relfilenode hnsw-name)]
+ (semantic.pgvector-api/ensure-active-hnsw-index! pgvector index-metadata)
+ (is (= before (semantic.tu/index-relfilenode hnsw-name))
+ "relfilenode unchanged => the existing index was reused, not rebuilt")))))
+ (testing "is a noop when there is no active index"
+ (is (nil? (semantic.pgvector-api/ensure-active-hnsw-index! pgvector index-metadata)))))))
+
(defn- open-semantic-search! ^Closeable [pgvector index-metadata embedding-model]
(semantic.tu/closeable
(semantic.pgvector-api/init-semantic-search! pgvector index-metadata embedding-model)
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj
new file mode 100644
index 00000000000..52362c63951
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj
@@ -0,0 +1,29 @@
+(ns metabase-enterprise.semantic-search.task.indexer-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.semantic-search.core :as semantic.core]
+ [metabase-enterprise.semantic-search.settings :as semantic.settings]
+ [metabase-enterprise.semantic-search.task.indexer :as sut]
+ [metabase-enterprise.semantic-search.util :as semantic.u]
+ [metabase.task.core :as task]
+ [metabase.test :as mt]))
+
+(set! *warn-on-reflection* true)
+
+(deftest startup-hnsw-safety-net-test
+ (testing "the indexer task's startup builds the HNSW index only when configured for :hnsw"
+ ;; Covers instances that boot already configured for :hnsw (e.g. strategy set via env var), where the
+ ;; setter's transition event never fired -- see the safety net in sut/init!. Redefs avoid scheduling a
+ ;; real quartz job or spawning a real build future.
+ (let [builds (atom 0)]
+ (mt/with-dynamic-fn-redefs [semantic.u/semantic-search-available? (constantly true)
+ task/schedule-task! (fn [& _] nil)
+ semantic.core/build-hnsw-index-async! (fn [] (swap! builds inc))]
+ (testing ":brute-force does not trigger a build"
+ (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly :brute-force)]
+ (task/init! ::sut/SemanticSearchIndexer))
+ (is (zero? @builds)))
+ (testing ":hnsw triggers a build"
+ (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly :hnsw)]
+ (task/init! ::sut/SemanticSearchIndexer))
+ (is (= 1 @builds)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
index 4a9968201e3..bfdd4c2b31a 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
@@ -15,6 +15,7 @@
[metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
[metabase-enterprise.semantic-search.indexer :as semantic.indexer]
[metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
+ [metabase-enterprise.semantic-search.settings :as semantic.settings]
[metabase-enterprise.semantic-search.util :as semantic.util]
[metabase.search.config :as search.config]
[metabase.search.ingestion :as search.ingestion]
@@ -85,8 +86,11 @@
semantic.index/model-table-suffix mock-table-suffix]
(let [pgvector (semantic.env/get-pgvector-datasource!)
index-metadata (semantic.env/get-index-metadata)
- embedding-model (semantic.env/get-configured-embedding-model)]
- (semantic.pgvector-api/init-semantic-search! pgvector index-metadata embedding-model)
+ embedding-model (semantic.env/get-configured-embedding-model)
+ index (semantic.pgvector-api/init-semantic-search! pgvector index-metadata embedding-model)]
+ ;; The HNSW index is only built when configured for the :hnsw strategy; tests exercise both strategies
+ ;; via per-query overrides, so build it unconditionally here.
+ (semantic.index/create-hnsw-index-if-not-exists! pgvector index)
(thunk))))
;; TODO: declare with macro (the do- less version) throws weird errors -- investigate!
@@ -361,9 +365,14 @@
Closeable
(close [_] (close-fn o))))
-(defn open-temp-index! ^Closeable [& {:keys [index] :or {index mock-index}}]
+(defn open-temp-index! ^Closeable [& {:keys [index hnsw?] :or {index mock-index hnsw? true}}]
(closeable
- (do (semantic.index/create-index-table-if-not-exists! (semantic.env/get-pgvector-datasource!) index {:force-reset? true})
+ ;; The HNSW index is only built when configured for the :hnsw strategy. Pin the strategy during creation
+ ;; so the table deterministically has (`:hnsw? true`, the default) or lacks (`:hnsw? false`) the index,
+ ;; regardless of the ambient setting.
+ (do (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy
+ (constantly (if hnsw? :hnsw :brute-force))]
+ (semantic.index/create-index-table-if-not-exists! (semantic.env/get-pgvector-datasource!) index {:force-reset? true}))
index)
(fn cleanup-temp-index-table! [{:keys [table-name] :as index}]
(try
@@ -505,6 +514,17 @@
(-> result first vals first))
(catch Exception _ false))))
+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
+(defn index-relfilenode
+ "Return the on-disk relfilenode for the index named `index-name`, or nil if it doesn't exist. A stable
+ relfilenode across two `CREATE INDEX IF NOT EXISTS` calls proves the second did no work: had it dropped,
+ rebuilt, or reindexed, the relfilenode would change."
+ [index-name]
+ (-> (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)
+ ["SELECT relfilenode FROM pg_class WHERE relname = ?" (name index-name)]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})
+ :relfilenode))
+
(defn get-metadata-rows [pgvector index-metadata]
(jdbc/execute! pgvector
(-> {:select [:*]
@@ -546,7 +566,10 @@
"Create an index table and return a closeable that will drop it when closed."
^Closeable [pgvector index]
(closeable
- (semantic.index/create-index-table-if-not-exists! pgvector index)
+ (do (semantic.index/create-index-table-if-not-exists! pgvector index)
+ ;; Build the HNSW index for tests; it is otherwise only created when configured for the :hnsw strategy.
+ (semantic.index/create-hnsw-index-if-not-exists! pgvector index)
+ index)
(fn [_] (semantic.index/drop-index-table! pgvector index))))
(defn- decode-column