The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Create HNSW index only when explicitly using it (#76387)
Chris Truter6ca4f480ffd2
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex cf1b48e98a0..297b0c2824c 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -3458,6 +3458,7 @@collectionsconfigconnection-pool+ eventsllmmetabotmodelsdiff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/core.cljindex 2dce56af79d..42adcc8ee17 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj@@ -11,6 +11,7 @@[metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api][metabase-enterprise.semantic-search.repair :as semantic.repair][metabase-enterprise.semantic-search.settings :as semantic.settings]+ [metabase-enterprise.semantic-search.util :as semantic.util][metabase.analytics-interface.core :as analytics][metabase.premium-features.core :refer [defenterprise]][metabase.search.config :as search.config]@@ -45,6 +46,21 @@(some? semantic.db.datasource/db-url)(semantic.settings/semantic-search-enabled)))+(defn build-hnsw-index-async!+ "Build the HNSW index on the active semantic search index in the background, returning promptly.++ No-ops when semantic search isn't available on this instance. Backs the just-in-time HNSW build, which+ runs only when an instance is configured to the `:hnsw` vector-search strategy."+ []+ (when (semantic.util/semantic-search-available?)+ (future+ (try+ (semantic.pgvector-api/ensure-active-hnsw-index! (semantic.env/get-pgvector-datasource!)+ (semantic.env/get-index-metadata))+ (catch Throwable t+ (log/error t "Failed to build HNSW index for semantic search")))))+ nil)+(defn- with-zero-semantic-distance-score"Record a 0 `:semantic-distance` score on `results` that lack it, for a consistent merged-result score breakdown."[search-ctx results]diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/events.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/events.cljnew file mode 100644index 00000000000..072357f36d7--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/events.clj@@ -0,0 +1,13 @@+(ns metabase-enterprise.semantic-search.events+ "Event handlers for the semantic-search module."+ (:require+ [metabase-enterprise.semantic-search.core :as semantic.core]+ [metabase.events.core :as events]+ [methodical.core :as methodical]))++(derive :event/semantic-search-hnsw-enabled ::build-hnsw-index)++;; Handled via an event so settings stays decoupled from the index-building code, which requires settings.+(methodical/defmethod events/publish-event! ::build-hnsw-index+ [_topic _event]+ (semantic.core/build-hnsw-index-async!))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.cljindex 6bb4dbb45fb..b7718e4a250 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj@@ -12,6 +12,7 @@[metabase-enterprise.semantic-search.embedding :as embedding][metabase-enterprise.semantic-search.scoring :as scoring][metabase-enterprise.semantic-search.settings :as semantic-settings]+ [metabase-enterprise.semantic-search.util :as semantic.util][metabase.analytics-interface.core :as analytics][metabase.models.interface :as mi][metabase.permissions.core :as perms]@@ -466,9 +467,33 @@[index](index-name index "_content_idx"))+(defn create-hnsw-index-if-not-exists!+ "Create the HNSW index on the `embedding` column of `index`'s table, if it does not already exist.++ HNSW indexes are expensive to build and maintain, so we only create one when an instance is configured to+ use the `:hnsw` vector-search strategy (see [[create-index-table-if-not-exists!]] and the async build+ triggered by [[metabase-enterprise.semantic-search.settings/semantic-search-vector-strategy]]).++ Pass `concurrently? true` to build with `CREATE INDEX CONCURRENTLY` (for a populated table whose writes+ shouldn't be locked out); it must run outside a transaction."+ [connectable index & {:keys [concurrently?] :or {concurrently? false}}]+ (let [{:keys [table-name]} index+ ;; HoneySQL emits CONCURRENTLY in the wrong position (before INDEX), so splice it into the+ ;; formatted statement instead: `CREATE INDEX CONCURRENTLY IF NOT EXISTS ...` is valid Postgres.+ [sql & params] (sql-format-quoted+ (sql.helpers/create-index+ [(keyword (hnsw-index-name index)) :if-not-exists]+ [(keyword table-name) :using-hnsw [[:raw "embedding vector_cosine_ops"]]]))+ sql (cond-> sql+ concurrently? (str/replace-first "CREATE INDEX " "CREATE INDEX CONCURRENTLY "))]+ (jdbc/execute! connectable (into [sql] params))))+(defn create-index-table-if-not-exists!"Ensure that the index table exists and is ready to be populated. If- force-reset? is true, drops and recreates the table if it exists."+ force-reset? is true, drops and recreates the table if it exists.++ The HNSW index is only created when the instance is configured for the `:hnsw` vector-search strategy;+ otherwise it is built just-in-time when the strategy is configured (see [[create-hnsw-index-if-not-exists!]])."[connectable index & {:keys [force-reset?] :or {force-reset? false}}](try(let [{:keys [embedding-model table-name]} index@@ -483,12 +508,8 @@(-> (sql.helpers/create-table (keyword table-name) :if-not-exists)(sql.helpers/with-columns (index-table-schema vector-dimensions))sql-format-quoted))- (jdbc/execute!- connectable- (-> (sql.helpers/create-index- [(keyword (hnsw-index-name index)) :if-not-exists]- [(keyword table-name) :using-hnsw [[:raw "embedding vector_cosine_ops"]]])- sql-format-quoted))+ (when (= :hnsw (semantic-settings/semantic-search-vector-strategy))+ (create-hnsw-index-if-not-exists! connectable index))(jdbc/execute!connectable(-> (sql.helpers/create-index@@ -910,72 +931,78 @@search-string (:search-string search-context)](if (str/blank? search-string){:results [] :raw-count 0}- (let [timer (u/start-timer)-- ;; Warm the pool concurrently with the embedding round-trip: the pool holds zero idle- ;; connections by default (see semantic-search.db.datasource), so the first search after an- ;; idle period would otherwise pay the connection handshake serially on top of the embedding- ;; latency.- _ (warm-connection-pool-async! db)- embedding (tracing/with-span :search "search.semantic.embedding"- {:search.semantic/provider (:provider embedding-model)- :search.semantic/model-name (:model-name embedding-model)}- (embedding/get-embedding embedding-model- (embedding/prefix-search-query embedding-model search-string)- {:type :query :record-tokens? true}))- embedding-time-ms (u/since-ms timer)-- db-timer (u/start-timer)- weights (search.config/weights search-context)- scorers (scoring/semantic-scorers (:table-name index) search-context)- query (scored-search-query index embedding search-context scorers)- xform (comp (map decode-legacy-input)- (map (partial legacy-input-with-score weights (keys scorers))))- reducible (reducible-search-query db query)- raw-results (tracing/with-span :search "search.semantic.db-query"- {:search/query-length (count search-string)}- (into [] xform reducible))- db-query-time-ms (u/since-ms db-timer)-- filter-timer (u/start-timer)- filtered-results (tracing/with-span :search "search.semantic.permission-filter"- {:search.semantic/raw-count (count raw-results)}- (->> raw-results- filter-read-permitted- (apply-collection-id-filter search-context)- (mapv search/collapse-id)))- filter-time-ms (u/since-ms filter-timer)-- appdb-scorers (scoring/appdb-scorers search-context)- appdb-scores-timer (u/start-timer)- final-results (->> filtered-results- (scoring/with-appdb-scores search-context appdb-scorers weights))- appdb-scores-time-ms (u/since-ms appdb-scores-timer)- total-time-ms (u/since-ms timer)]- (log/debug "Semantic search"- {:search-string-length (count search-string)- :raw-results-count (count raw-results)- :final-results-count (count final-results)- :embedding-time-ms embedding-time-ms- :db-query-time-ms db-query-time-ms- :filter-time-ms filter-time-ms- :appdb-scores-time-ms appdb-scores-time-ms- :total-time-ms total-time-ms})- (analytics/inc! :metabase-search/semantic-embedding-ms- {:embedding-model (:name embedding-model)}- embedding-time-ms)- (analytics/inc! :metabase-search/semantic-db-query-ms- {:embedding-model (:name embedding-model)}- db-query-time-ms)- (analytics/inc! :metabase-search/semantic-appdb-scores-ms- appdb-scores-time-ms)- (analytics/inc! :metabase-search/semantic-search-ms- {:embedding-model (:name embedding-model)}- total-time-ms)- (comment- (jdbc/execute! db (sql-format-quoted query)))- {:results final-results- :raw-count (count raw-results)}))))+ (do+ (when (and (= :hnsw (vector-search-strategy search-context))+ (not (semantic.util/index-exists? db (hnsw-index-name index))))+ (throw (ex-info (str "HNSW vector-search strategy requested but no HNSW index exists. "+ "Set the semantic-search-vector-strategy setting to :hnsw to build it.")+ {:table-name (:table-name index)})))+ (let [timer (u/start-timer)++ ;; Warm the pool concurrently with the embedding round-trip: the pool holds zero idle+ ;; connections by default (see semantic-search.db.datasource), so the first search after an+ ;; idle period would otherwise pay the connection handshake serially on top of the embedding+ ;; latency.+ _ (warm-connection-pool-async! db)+ embedding (tracing/with-span :search "search.semantic.embedding"+ {:search.semantic/provider (:provider embedding-model)+ :search.semantic/model-name (:model-name embedding-model)}+ (embedding/get-embedding embedding-model+ (embedding/prefix-search-query embedding-model search-string)+ {:type :query :record-tokens? true}))+ embedding-time-ms (u/since-ms timer)++ db-timer (u/start-timer)+ weights (search.config/weights search-context)+ scorers (scoring/semantic-scorers (:table-name index) search-context)+ query (scored-search-query index embedding search-context scorers)+ xform (comp (map decode-legacy-input)+ (map (partial legacy-input-with-score weights (keys scorers))))+ reducible (reducible-search-query db query)+ raw-results (tracing/with-span :search "search.semantic.db-query"+ {:search/query-length (count search-string)}+ (into [] xform reducible))+ db-query-time-ms (u/since-ms db-timer)++ filter-timer (u/start-timer)+ filtered-results (tracing/with-span :search "search.semantic.permission-filter"+ {:search.semantic/raw-count (count raw-results)}+ (->> raw-results+ filter-read-permitted+ (apply-collection-id-filter search-context)+ (mapv search/collapse-id)))+ filter-time-ms (u/since-ms filter-timer)++ appdb-scorers (scoring/appdb-scorers search-context)+ appdb-scores-timer (u/start-timer)+ final-results (->> filtered-results+ (scoring/with-appdb-scores search-context appdb-scorers weights))+ appdb-scores-time-ms (u/since-ms appdb-scores-timer)+ total-time-ms (u/since-ms timer)]+ (log/debug "Semantic search"+ {:search-string-length (count search-string)+ :raw-results-count (count raw-results)+ :final-results-count (count final-results)+ :embedding-time-ms embedding-time-ms+ :db-query-time-ms db-query-time-ms+ :filter-time-ms filter-time-ms+ :appdb-scores-time-ms appdb-scores-time-ms+ :total-time-ms total-time-ms})+ (analytics/inc! :metabase-search/semantic-embedding-ms+ {:embedding-model (:name embedding-model)}+ embedding-time-ms)+ (analytics/inc! :metabase-search/semantic-db-query-ms+ {:embedding-model (:name embedding-model)}+ db-query-time-ms)+ (analytics/inc! :metabase-search/semantic-appdb-scores-ms+ appdb-scores-time-ms)+ (analytics/inc! :metabase-search/semantic-search-ms+ {:embedding-model (:name embedding-model)}+ total-time-ms)+ (comment+ (jdbc/execute! db (sql-format-quoted query)))+ {:results final-results+ :raw-count (count raw-results)})))))(defn- row-present?"Whether a single `(model, id)` row survives `where` against the index `table`."diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/init.cljindex 30e92d534c0..330cec96293 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj@@ -1,5 +1,6 @@(ns metabase-enterprise.semantic-search.init(:require+ [metabase-enterprise.semantic-search.events][metabase-enterprise.semantic-search.settings][metabase-enterprise.semantic-search.task.index-cleanup][metabase-enterprise.semantic-search.task.index-repair]diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.cljindex 8fbf30b7f20..7282469e950 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.clj@@ -61,6 +61,18 @@(semantic.index-metadata/activate-index! tx index-metadata index-id))index))+(defn ensure-active-hnsw-index!+ "Build the HNSW index on the active index table if it does not already exist.++ Called when an instance is (re)configured to the `:hnsw` vector-search strategy. No-ops when there is no+ active index. Builds the index with `CREATE INDEX CONCURRENTLY` since the table is typically populated."+ [pgvector index-metadata]+ (if-let [{:keys [index]} (semantic.index-metadata/get-active-index-state pgvector index-metadata)]+ (do+ (log/info "Building HNSW index for active semantic search index" (:table-name index))+ (semantic.index/create-hnsw-index-if-not-exists! pgvector index {:concurrently? true}))+ (log/info "No active semantic search index; skipping HNSW index build")))+(defn init-semantic-search!"Initialises a pgvector database for semantic search if it does not exist and creates an index for the providedembedding model (if it does not exist).diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.cljindex ff2d055b6fb..a74a52e09cf 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj@@ -1,11 +1,16 @@(ns metabase-enterprise.semantic-search.settings(:require+ [metabase.events.core :as events][metabase.llm.settings :as llm-settings][metabase.premium-features.core :as premium-features][metabase.search.config :as search.config][metabase.settings.core :as setting :refer [defsetting]][metabase.util.i18n :refer [deferred-tru]]))+;; Topic for the just-in-time HNSW build, handled in metabase-enterprise.semantic-search.events. Declared+;; here, not there, so it's valid wherever the setter runs regardless of handler-namespace load order.+(derive :event/semantic-search-hnsw-enabled :metabase/event)+(def ^:private valid-embedding-providers"The set of valid embedding provider names."#{"ai-service" "openai" "ollama"})@@ -120,21 +125,26 @@(deferred-tru(str "Default vector-search strategy for semantic search: `hnsw` (approximate, HNSW-index-backed) or ""`brute-force` (exact, applies non-vector filters first then computes cosine distance over the "- "survivors). Individual requests may override this via the `vector_search_strategy` API parameter."))+ "survivors). Defaults to `brute-force`, which needs no index; selecting `hnsw` builds the HNSW index "+ "just-in-time. Individual requests may override this via the `vector_search_strategy` API parameter.")):type :keyword- :default :hnsw+ :default :brute-force:encryption :no:export? false:visibility :internal:doc false:setter (fn [new-value]- (let [kw (some-> new-value keyword)]+ (let [kw (some-> new-value keyword)+ old (setting/get-value-of-type :keyword :semantic-search-vector-strategy)](when (and kw (not (contains? valid-vector-search-strategies kw)))(throw (ex-info (str "Invalid vector-search strategy: " (pr-str new-value)". Valid strategies are: " (pr-str valid-vector-search-strategies)){:invalid-value new-value:valid-values valid-vector-search-strategies})))- (setting/set-value-of-type! :keyword :semantic-search-vector-strategy kw))))+ (setting/set-value-of-type! :keyword :semantic-search-vector-strategy kw)+ ;; Gated on the transition (not every set) so re-setting :hnsw doesn't rebuild the index.+ (when (and (= kw :hnsw) (not= old :hnsw))+ (events/publish-event! :event/semantic-search-hnsw-enabled {})))))(defsetting semantic-search-min-results-threshold(deferred-tru "Minimum number of semantic search results required before falling back to other engines.")diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.cljindex 50aed9377da..bec65c03f0c 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj@@ -3,8 +3,10 @@[clojurewerkz.quartzite.jobs :as jobs][clojurewerkz.quartzite.schedule.simple :as simple][clojurewerkz.quartzite.triggers :as triggers]+ [metabase-enterprise.semantic-search.core :as semantic.core][metabase-enterprise.semantic-search.env :as semantic.env][metabase-enterprise.semantic-search.indexer :as semantic-search.indexer]+ [metabase-enterprise.semantic-search.settings :as semantic.settings][metabase-enterprise.semantic-search.util :as semantic.u][metabase.search.engine :as search.engine][metabase.task.core :as task]@@ -66,4 +68,8 @@(simple/schedule(simple/with-interval-in-milliseconds (.toMillis run-frequency))(simple/repeat-forever))))]- (task/schedule-task! job trigger))))+ (task/schedule-task! job trigger))+ ;; Safety net: an instance that booted already configured for :hnsw (strategy set via env var, or set+ ;; before an active index existed) never saw the setter's transition event, so build the index now.+ (when (= :hnsw (semantic.settings/semantic-search-vector-strategy))+ (semantic.core/build-hnsw-index-async!))))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/util.cljindex 6615f590be5..7bf0e5dba45 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj@@ -14,6 +14,15 @@{:builder-fn jdbc.rs/as-unqualified-lower-maps})(:table_exists false)))+(defn index-exists?+ "Does an index named `index-name` exist in the pgvector DB's pg_indexes?"+ [pgvector index-name]+ (-> (jdbc/execute-one! pgvector+ ["SELECT exists (select 1 FROM pg_indexes WHERE indexname = ?) index_exists"+ index-name]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})+ (:index_exists false)))+(defn semantic-search-available?"Predicate to check whether semantic search is available on the instance."[]diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.cljindex 3c8d40a412b..699e3629012 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj@@ -4,8 +4,11 @@[clojure.string :as str][clojure.test :refer :all][honey.sql :as sql]+ [metabase-enterprise.semantic-search.core :as semantic.core][metabase-enterprise.semantic-search.embedding :as semantic.embedding][metabase-enterprise.semantic-search.env :as semantic.env]+ ;; loaded for its :event/semantic-search-hnsw-enabled handler, which the setter test exercises+ [metabase-enterprise.semantic-search.events][metabase-enterprise.semantic-search.index :as semantic.index][metabase-enterprise.semantic-search.settings :as semantic.settings][metabase-enterprise.semantic-search.spec-trace-test-util :as spec-trace]@@ -79,24 +82,71 @@(testing "valid strategies round-trip"(doseq [strategy [:hnsw :brute-force]](mt/with-temporary-setting-values [semantic.settings/semantic-search-vector-strategy strategy]- (is (= strategy (semantic.settings/semantic-search-vector-strategy)))))))+ (is (= strategy (semantic.settings/semantic-search-vector-strategy))))))+ (testing "the setter kicks off the background build job only on the transition into :hnsw"+ ;; This asserts *when* the build is triggered (the transition gating), not what the build does -- the+ ;; build itself is covered by pgvector-api-test/ensure-active-hnsw-index!-test. The setter publishes+ ;; :event/semantic-search-hnsw-enabled, whose handler (semantic-search.events) calls the async build; we+ ;; spy on that build so the trigger is verified deterministically without spawning a real future.+ (mt/with-premium-features #{:semantic-search}+ (let [triggers (atom 0)]+ (mt/with-dynamic-fn-redefs [semantic.core/build-hnsw-index-async! (fn [] (swap! triggers inc))]+ (mt/with-temporary-setting-values [semantic.settings/semantic-search-vector-strategy :brute-force]+ (semantic.settings/semantic-search-vector-strategy! :hnsw)+ (is (= 1 @triggers) "transitioning :brute-force -> :hnsw kicks off a build")+ (semantic.settings/semantic-search-vector-strategy! :hnsw)+ (is (= 1 @triggers) "setting :hnsw again when already :hnsw does not re-trigger")+ (semantic.settings/semantic-search-vector-strategy! :brute-force)+ (is (= 1 @triggers) "switching away from :hnsw does not trigger")+ (semantic.settings/semantic-search-vector-strategy! :hnsw)+ (is (= 2 @triggers) "transitioning back into :hnsw triggers again")))))))(deftest create-index-table!-test(mt/with-premium-features #{:semantic-search}- (with-open [index-ref (semantic.tu/open-temp-index!)]- ;; open-temp-index-table! creates the temp table, so drop it in order to test create!.+ (with-open [index-ref (semantic.tu/open-temp-index! :hnsw? false)]+ ;; open-temp-index! creates the temp table, so drop it in order to test create!.(semantic.index/drop-index-table! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index)(testing "index table is not present before create!"(is (not (semantic.tu/table-exists-in-db? (:table-name @index-ref))))(is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref))))(is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-index-name @index-ref))))(is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-native-index-name @index-ref)))))- (testing "index table is present after create!"- (semantic.index/create-index-table-if-not-exists! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index {:force-reset? false})+ (testing "under the default :brute-force strategy, create! builds the table and FTS indexes but not the HNSW index"+ (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly :brute-force)]+ (semantic.index/create-index-table-if-not-exists! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index {:force-reset? true}))(is (semantic.tu/table-exists-in-db? (:table-name @index-ref)))- (is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref)))+ (is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref))))(is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-index-name @index-ref)))- (is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-native-index-name @index-ref)))))))+ (is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-native-index-name @index-ref))))+ (testing "when configured for the :hnsw strategy, create! also builds the HNSW index"+ (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly :hnsw)]+ (semantic.index/create-index-table-if-not-exists! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index {:force-reset? true}))+ (is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref)))))))++(deftest create-hnsw-index-if-not-exists!-test+ (mt/with-premium-features #{:semantic-search}+ (with-open [index-ref (semantic.tu/open-temp-index! :hnsw? false)]+ (testing "HNSW index is absent until built explicitly"+ (is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref))))+ (semantic.index/create-hnsw-index-if-not-exists! (semantic.env/get-pgvector-datasource!) @index-ref)+ (is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref))))+ (testing "is idempotent: a second call does no work (reuses the existing index, no rebuild)"+ (let [before (semantic.tu/index-relfilenode (semantic.index/hnsw-index-name @index-ref))]+ (is (some? before) "the index exists before the second call")+ (semantic.index/create-hnsw-index-if-not-exists! (semantic.env/get-pgvector-datasource!) @index-ref)+ (is (= before (semantic.tu/index-relfilenode (semantic.index/hnsw-index-name @index-ref)))+ "relfilenode is unchanged, so the index was not dropped, rebuilt, or reindexed"))))))++(deftest query-index-hnsw-without-index-throws-test+ (testing "a query using the :hnsw strategy fails fast when no HNSW index exists, rather than silently scanning"+ (mt/with-premium-features #{:semantic-search}+ (with-open [index-ref (semantic.tu/open-temp-index! :hnsw? false)]+ (is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref))))+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo #"no HNSW index exists"+ (semantic.index/query-index (semantic.env/get-pgvector-datasource!)+ @index-ref+ {:search-string "puppy" :vector-search-strategy :hnsw})))))))(deftest drop-index-table!-test(mt/with-premium-features #{:semantic-search}diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.cljindex 9c134ce8069..7edfded2bd1 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj@@ -86,6 +86,32 @@(semantic.pgvector-api/init-semantic-search! pgvector index-metadata model1)(is (true? (@#'semantic.index-metadata/index-table-exists? pgvector index))))))))+(deftest ensure-active-hnsw-index!-test+ (mt/with-premium-features #{:semantic-search}+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ index-metadata (semantic.tu/unique-index-metadata)+ model semantic.tu/mock-embedding-model+ cleanup (fn [_] (semantic.tu/cleanup-index-metadata! pgvector index-metadata))]+ (with-open [index-ref ^Closeable (semantic.tu/closeable+ (semantic.pgvector-api/init-semantic-search! pgvector index-metadata model)+ cleanup)]+ (let [index @index-ref+ hnsw-name (semantic.index/hnsw-index-name index)]+ (testing "builds the HNSW index on the active index table when it is missing"+ ;; init only builds the HNSW index when configured for :hnsw, so drop it for a deterministic+ ;; starting point regardless of the ambient strategy -- mimics a brute-force-configured instance.+ (jdbc/execute! pgvector [(str "DROP INDEX IF EXISTS \"" hnsw-name "\"")])+ (is (not (semantic.tu/table-has-index? (:table-name index) hnsw-name)))+ (semantic.pgvector-api/ensure-active-hnsw-index! pgvector index-metadata)+ (is (semantic.tu/table-has-index? (:table-name index) hnsw-name)))+ (testing "is a noop when the HNSW index already exists"+ (let [before (semantic.tu/index-relfilenode hnsw-name)]+ (semantic.pgvector-api/ensure-active-hnsw-index! pgvector index-metadata)+ (is (= before (semantic.tu/index-relfilenode hnsw-name))+ "relfilenode unchanged => the existing index was reused, not rebuilt")))))+ (testing "is a noop when there is no active index"+ (is (nil? (semantic.pgvector-api/ensure-active-hnsw-index! pgvector index-metadata)))))))+(defn- open-semantic-search! ^Closeable [pgvector index-metadata embedding-model](semantic.tu/closeable(semantic.pgvector-api/init-semantic-search! pgvector index-metadata embedding-model)diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.cljnew file mode 100644index 00000000000..52362c63951--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj@@ -0,0 +1,29 @@+(ns metabase-enterprise.semantic-search.task.indexer-test+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.semantic-search.core :as semantic.core]+ [metabase-enterprise.semantic-search.settings :as semantic.settings]+ [metabase-enterprise.semantic-search.task.indexer :as sut]+ [metabase-enterprise.semantic-search.util :as semantic.u]+ [metabase.task.core :as task]+ [metabase.test :as mt]))++(set! *warn-on-reflection* true)++(deftest startup-hnsw-safety-net-test+ (testing "the indexer task's startup builds the HNSW index only when configured for :hnsw"+ ;; Covers instances that boot already configured for :hnsw (e.g. strategy set via env var), where the+ ;; setter's transition event never fired -- see the safety net in sut/init!. Redefs avoid scheduling a+ ;; real quartz job or spawning a real build future.+ (let [builds (atom 0)]+ (mt/with-dynamic-fn-redefs [semantic.u/semantic-search-available? (constantly true)+ task/schedule-task! (fn [& _] nil)+ semantic.core/build-hnsw-index-async! (fn [] (swap! builds inc))]+ (testing ":brute-force does not trigger a build"+ (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly :brute-force)]+ (task/init! ::sut/SemanticSearchIndexer))+ (is (zero? @builds)))+ (testing ":hnsw triggers a build"+ (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly :hnsw)]+ (task/init! ::sut/SemanticSearchIndexer))+ (is (= 1 @builds)))))))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.cljindex 4a9968201e3..bfdd4c2b31a 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj@@ -15,6 +15,7 @@[metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata][metabase-enterprise.semantic-search.indexer :as semantic.indexer][metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]+ [metabase-enterprise.semantic-search.settings :as semantic.settings][metabase-enterprise.semantic-search.util :as semantic.util][metabase.search.config :as search.config][metabase.search.ingestion :as search.ingestion]@@ -85,8 +86,11 @@semantic.index/model-table-suffix mock-table-suffix](let [pgvector (semantic.env/get-pgvector-datasource!)index-metadata (semantic.env/get-index-metadata)- embedding-model (semantic.env/get-configured-embedding-model)]- (semantic.pgvector-api/init-semantic-search! pgvector index-metadata embedding-model)+ embedding-model (semantic.env/get-configured-embedding-model)+ index (semantic.pgvector-api/init-semantic-search! pgvector index-metadata embedding-model)]+ ;; The HNSW index is only built when configured for the :hnsw strategy; tests exercise both strategies+ ;; via per-query overrides, so build it unconditionally here.+ (semantic.index/create-hnsw-index-if-not-exists! pgvector index)(thunk))));; TODO: declare with macro (the do- less version) throws weird errors -- investigate!@@ -361,9 +365,14 @@Closeable(close [_] (close-fn o))))-(defn open-temp-index! ^Closeable [& {:keys [index] :or {index mock-index}}]+(defn open-temp-index! ^Closeable [& {:keys [index hnsw?] :or {index mock-index hnsw? true}}](closeable- (do (semantic.index/create-index-table-if-not-exists! (semantic.env/get-pgvector-datasource!) index {:force-reset? true})+ ;; The HNSW index is only built when configured for the :hnsw strategy. Pin the strategy during creation+ ;; so the table deterministically has (`:hnsw? true`, the default) or lacks (`:hnsw? false`) the index,+ ;; regardless of the ambient setting.+ (do (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy+ (constantly (if hnsw? :hnsw :brute-force))]+ (semantic.index/create-index-table-if-not-exists! (semantic.env/get-pgvector-datasource!) index {:force-reset? true}))index)(fn cleanup-temp-index-table! [{:keys [table-name] :as index}](try@@ -505,6 +514,17 @@(-> result first vals first))(catch Exception _ false))))+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}+(defn index-relfilenode+ "Return the on-disk relfilenode for the index named `index-name`, or nil if it doesn't exist. A stable+ relfilenode across two `CREATE INDEX IF NOT EXISTS` calls proves the second did no work: had it dropped,+ rebuilt, or reindexed, the relfilenode would change."+ [index-name]+ (-> (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)+ ["SELECT relfilenode FROM pg_class WHERE relname = ?" (name index-name)]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})+ :relfilenode))+(defn get-metadata-rows [pgvector index-metadata](jdbc/execute! pgvector(-> {:select [:*]@@ -546,7 +566,10 @@"Create an index table and return a closeable that will drop it when closed."^Closeable [pgvector index](closeable- (semantic.index/create-index-table-if-not-exists! pgvector index)+ (do (semantic.index/create-index-table-if-not-exists! pgvector index)+ ;; Build the HNSW index for tests; it is otherwise only created when configured for the :hnsw strategy.+ (semantic.index/create-hnsw-index-if-not-exists! pgvector index)+ index)(fn [_] (semantic.index/drop-index-table! pgvector index))))(defn- decode-column