The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Recover invalid semantic search HNSW indexes (#78428)
Chris Truter0d2b72368769
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.ednindex d1e05ec5329..7024ebb56cd 100644--- a/.clj-kondo/ratchets.edn+++ b/.clj-kondo/ratchets.edn@@ -22,7 +22,7 @@:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests 14:metabase/modules 8:metabase/namespace-name 4- :metabase/prefer-with-dynamic-fn-redefs 25+ :metabase/prefer-with-dynamic-fn-redefs 27:metabase/test-helpers-use-non-thread-safe-functions 17:metabase/validate-defendpoint-has-response-schema 441:metabase/validate-defendpoint-query-params-use-kebab-case 40diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/core.cljindex 5b41441e212..3e16252b003 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj@@ -53,19 +53,24 @@(and (semantic.util/semantic-search-available?)(semantic.embedding/embedding-supported? (semantic.embedding/get-configured-model))))+(defonce ^:private hnsw-index-build-running? (atom false))+(defn build-hnsw-index-async!"Build the HNSW index on the active semantic search index in the background, returning promptly.- No-ops when semantic search isn't active on this instance. Backs the just-in-time HNSW build, which- runs only when an instance is configured to the `:hnsw` vector-search strategy."+ No-ops when semantic search isn't active or this process already has a build running. Database-level+ coordination in [[semantic.pgvector-api/ensure-active-hnsw-index!]] serializes builds across instances."[]- (when (semantic.util/semantic-search-active?)+ (when (and (semantic.util/semantic-search-active?)+ (compare-and-set! hnsw-index-build-running? false true))(future(try(semantic.pgvector-api/ensure-active-hnsw-index! (semantic.env/get-pgvector-datasource!)(semantic.env/get-index-metadata))(catch Throwable t- (log/error t "Failed to build HNSW index for semantic search")))))+ (log/error t "Failed to build HNSW index for semantic search"))+ (finally+ (reset! hnsw-index-build-running? false)))))nil)(defn- with-zero-semantic-distance-scorediff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/health.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/health.cljindex 04ac4c6d72b..5796b7bd2a5 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/health.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/health.clj@@ -45,7 +45,7 @@(try(or (not (contains? search.config/hnsw-index-backed-strategies(semantic.settings/semantic-search-vector-strategy)))- (semantic.util/index-ready?+ (semantic.util/index-exists?pgvector(semantic.index/schema-qualified-index-name index (semantic.index/hnsw-index-name index))))(catch InterruptedException ediff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.cljindex 0bf4d0ec53d..3c45443f7f4 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj@@ -525,6 +525,12 @@concurrently? (str/replace-first "CREATE INDEX " "CREATE INDEX CONCURRENTLY "))](jdbc/execute! connectable (into [sql] params))))+(defn drop-index-concurrently-if-exists!+ "Drop `index-name` without blocking writes. Must run outside a transaction."+ [connectable index-name]+ (jdbc/execute! connectable+ [(str "DROP INDEX CONCURRENTLY IF EXISTS " (semantic.util/quote-table index-name))]))+(defn create-index-table-if-not-exists!"Ensure that the index table exists and is ready to be populated. Ifforce-reset? is true, drops and recreates the table if it exists.@@ -1179,16 +1185,23 @@search-string (:search-string search-context)](if (str/blank? search-string){:results [] :raw-count 0}- (do+ (let [index-name (schema-qualified-index-name index (hnsw-index-name index))+ index-state (when (contains? search.config/hnsw-index-backed-strategies+ (vector-search-strategy search-context))+ (semantic.util/index-state db index-name))+ search-context (cond-> search-context+ (= :building index-state) (assoc :vector-search-strategy :brute-force))];; `:vector-search-allow-missing-index?` is a deliberate opt-out for callers that want the inner;; query to run without the HNSW index (e.g. the strategy matrix test probing the exact seq-scan- ;; path); production traffic leaves it unset and gets the fail-fast.+ ;; path). A concurrent build also uses that exact path until PostgreSQL marks the index ready; an+ ;; absent or abandoned invalid index fails fast.(when (and (contains? search.config/hnsw-index-backed-strategies (vector-search-strategy search-context))(not (:vector-search-allow-missing-index? search-context))- (not (semantic.util/index-exists? db (schema-qualified-index-name index (hnsw-index-name index)))))- (throw (ex-info (str "HNSW-index-backed vector-search strategy requested but no HNSW index exists. "- "Set the semantic-search-vector-strategy setting to an index-backed strategy "- "(:hnsw or :hnsw-iterative-*) to build it.")+ (contains? #{nil :invalid} index-state))+ (throw (ex-info (str "HNSW-index-backed vector-search strategy requested but no usable HNSW index exists. "+ "The index is absent or was abandoned invalid. It will be rebuilt by the next "+ "maintenance pass; retry shortly, or pass :vector-search-allow-missing-index? true "+ "to bypass this check."){:table-name (:table-name index):strategy (vector-search-strategy search-context)})))(let [timer (u/start-timer)diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.cljindex 5939f5d55ca..3abebe30f75 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.clj@@ -16,9 +16,13 @@[metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata][metabase-enterprise.semantic-search.repair :as semantic.repair][metabase-enterprise.semantic-search.settings :as semantic.settings]+ [metabase-enterprise.semantic-search.util :as semantic.util][metabase.util :as u]- [metabase.util.log :as log])- (:import (java.time Instant)))+ [metabase.util.log :as log]+ [next.jdbc :as jdbc])+ (:import+ (java.sql Connection)+ (java.time Instant)))(set! *warn-on-reflection* true)@@ -60,16 +64,51 @@(semantic.index-metadata/activate-index! tx index-metadata index-id))index))+(defn- with-hnsw-index-lock+ [pgvector index-name f]+ (with-open [^Connection conn (jdbc/get-connection pgvector)]+ (let [autocommit (.getAutoCommit conn)+ lock-name (str "metabase-semantic-hnsw:" index-name)]+ (.setAutoCommit conn true)+ (try+ (jdbc/execute-one! conn ["SELECT pg_advisory_lock(hashtextextended(?, 0))" lock-name])+ (try+ (f conn)+ (finally+ (jdbc/execute-one! conn ["SELECT pg_advisory_unlock(hashtextextended(?, 0))" lock-name])))+ (finally+ (.setAutoCommit conn autocommit))))))++(defn- ensure-hnsw-index-under-lock!+ [conn index]+ (let [index-name (semantic.index/schema-qualified-index-name index (semantic.index/hnsw-index-name index))]+ (case (semantic.util/index-state conn index-name)+ :ready+ (log/debug "HNSW index is already ready" index-name)+ :building+ (log/info "HNSW index build is already in progress" index-name)+ :invalid+ (do+ (log/warn "Replacing invalid HNSW index" index-name)+ (semantic.index/drop-index-concurrently-if-exists! conn index-name)+ (semantic.index/create-hnsw-index-if-not-exists! conn index {:concurrently? true}))+ nil+ (do+ (log/info "Building HNSW index" index-name)+ (semantic.index/create-hnsw-index-if-not-exists! conn index {:concurrently? true})))))+(defn ensure-active-hnsw-index!- "Build the HNSW index on the active index table if it does not already exist.+ "Ensure that the active index has a usable HNSW index.Called when an instance is (re)configured to the `:hnsw` vector-search strategy. No-ops when there is no- active index. Builds the index with `CREATE INDEX CONCURRENTLY` since the table is typically populated."+ active index. Serializes maintenance across instances, leaves active concurrent builds alone, and replaces+ abandoned invalid indexes. Builds with concurrent DDL because the table is typically populated."[pgvector index-metadata](if-let [{:keys [index]} (semantic.index-metadata/get-active-index-state pgvector index-metadata)](do- (log/info "Building HNSW index for active semantic search index" (:table-name index))- (semantic.index/create-hnsw-index-if-not-exists! pgvector index {:concurrently? true}))+ (let [index-name (semantic.index/schema-qualified-index-name index (semantic.index/hnsw-index-name index))]+ (with-hnsw-index-lock pgvector index-name #(ensure-hnsw-index-under-lock! % index)))+ nil)(log/info "No active semantic search index; skipping HNSW index build")))(defn init-semantic-search!@@ -177,8 +216,7 @@(def index (:index index-state));; warning: deletes everything- (require 'next.jdbc)- (next.jdbc/execute! pgvector ((requiring-resolve 'honey.sql/format) {:delete-from (keyword (:table-name index))} :quoted true))+ (jdbc/execute! pgvector ((requiring-resolve 'honey.sql/format) {:delete-from (keyword (:table-name index))} :quoted true))(def searchable-documents (vec ((requiring-resolve 'metabase.search.ingestion/searchable-documents))))(index-documents! pgvector index-metadata searchable-documents)(mt/as-admin (query pgvector index-metadata {:search-string "sharp objects"})))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.cljindex 2a331c5eaba..af3b246bceb 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj@@ -49,12 +49,12 @@(if-let [{:keys [index]} (semantic.index-metadata/get-active-index-state pgvector index-metadata)](do;; The strategy setter's build event no-ops while semantic is inactive, so an index-backed- ;; strategy configured before (re)activation arrives here with no HNSW index. CONCURRENTLY- ;; registers the index in pg_indexes as soon as the build starts, so this fires only once.+ ;; strategy configured before (re)activation arrives here with no HNSW index. Retry absent+ ;; or abandoned invalid indexes; an active concurrent build is left alone.(let [hnsw-index (semantic.index/schema-qualified-index-nameindex (semantic.index/hnsw-index-name index))](when (and (hnsw-strategy?)- (not (semantic.u/index-exists? pgvector hnsw-index)))+ (semantic.u/index-needs-build? pgvector hnsw-index))(semantic.core/build-hnsw-index-async!)))(semantic-search.indexer/quartz-job-run! pgvector index-metadata));; Engines can activate at runtime (license applied, additional-search-engines set ondiff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/util.cljindex 37e19af8307..ec817521d2a 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj@@ -64,39 +64,50 @@{:builder-fn jdbc.rs/as-unqualified-lower-maps})(:table_exists false))))+(defn index-state+ "Return the catalog state of `index-name`: `:ready`, `:building`, `:invalid`, or `nil` when absent.+ A schema-qualified name matches only within its schema; an unqualified name matches any schema."+ [pgvector index-name]+ (let [[schema index] (qualified-table-parts index-name)+ columns (str "x.indisready AS is_ready, x.indisvalid AS is_valid, "+ "EXISTS (SELECT 1 FROM pg_stat_progress_create_index p "+ " WHERE p.index_relid = i.oid) AS is_building")]+ (when-some [{:keys [is_ready is_valid is_building]}+ (jdbc/execute-one! pgvector+ (if schema+ [(str "SELECT " columns " "+ "FROM pg_class i "+ "JOIN pg_namespace n ON n.oid = i.relnamespace "+ "JOIN pg_index x ON x.indexrelid = i.oid "+ "WHERE n.nspname = ? AND i.relname = ?")+ schema index]+ [(str "SELECT " columns " "+ "FROM pg_class i "+ "JOIN pg_index x ON x.indexrelid = i.oid "+ "WHERE i.relname = "+ "? "+ "ORDER BY CASE "+ "WHEN x.indisready AND x.indisvalid THEN 0 "+ "WHEN EXISTS (SELECT 1 FROM pg_stat_progress_create_index p "+ " WHERE p.index_relid = i.oid) THEN 1 "+ "ELSE 2 END "+ "LIMIT 1")+ index])+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})]+ (cond+ (and is_ready is_valid) :ready+ is_building :building+ :else :invalid))))+(defn index-exists?- "Does an index named `index-name` exist in the pgvector DB's pg_indexes?- A schema-qualified (dotted) name matches only within its schema; an unqualified name matches any schema."+ "Whether `index-name` is ready and valid. See [[index-state]]."[pgvector index-name]- (let [[schema index] (qualified-table-parts index-name)]- (-> (jdbc/execute-one! pgvector- (if schema- ["SELECT exists (select 1 FROM pg_indexes WHERE schemaname = ? AND indexname = ?) index_exists"- schema index]- ["SELECT exists (select 1 FROM pg_indexes WHERE indexname = ?) index_exists"- index])- {:builder-fn jdbc.rs/as-unqualified-lower-maps})- (:index_exists false))))+ (= :ready (index-state pgvector index-name)))-(defn index-ready?- "Whether `index-name` is present and marked ready and valid by PostgreSQL. An in-progress or abandoned- concurrent build is not ready. A schema-qualified name matches only within its schema."+(defn index-needs-build?+ "Whether `index-name` is absent or invalid with no concurrent build in progress."[pgvector index-name]- (let [[schema index] (qualified-table-parts index-name)]- (-> (jdbc/execute-one! pgvector- (if schema- [(str "SELECT exists (SELECT 1 FROM pg_class i "- "JOIN pg_namespace n ON n.oid = i.relnamespace "- "JOIN pg_index x ON x.indexrelid = i.oid "- "WHERE n.nspname = ? AND i.relname = ? "- "AND x.indisready AND x.indisvalid) AS index_ready")- schema index]- [(str "SELECT exists (SELECT 1 FROM pg_class i "- "JOIN pg_index x ON x.indexrelid = i.oid "- "WHERE i.relname = ? AND x.indisready AND x.indisvalid) AS index_ready")- index])- {:builder-fn jdbc.rs/as-unqualified-lower-maps})- (:index_ready false))))+ (contains? #{nil :invalid} (index-state pgvector index-name)))(defn semantic-search-configured?"Whether to schedule the semantic-search Quartz jobs at startup.diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.cljindex b189edbe209..8cf3bc71a0e 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj@@ -16,7 +16,9 @@[metabase.search.in-place.legacy :as in-place.legacy][metabase.search.in-place.scoring :as in-place.scoring][metabase.search.settings :as search.settings]- [metabase.test :as mt]))+ [metabase.test :as mt])+ (:import+ (java.util.concurrent CountDownLatch TimeUnit)))(set! *warn-on-reflection* true)@@ -34,6 +36,35 @@(constantly semantic.tu/mock-embedding-model)](is (true? (semantic.core/supported?))))))))+(deftest ^:sequential build-hnsw-index-async-deduplicates-local-builds-test+ (let [started (CountDownLatch. 1)+ release (CountDownLatch. 1)+ reset-complete (CountDownLatch. 1)+ build-running? (atom false)+ calls (atom 0)]+ (add-watch build-running? ::reset-complete+ (fn [_ _ previous current]+ (when (and previous (not current))+ (.countDown reset-complete))))+ (with-redefs-fn {#'semantic.core/hnsw-index-build-running? build-running?}+ #(mt/with-dynamic-fn-redefs+ [semantic.util/semantic-search-active? (constantly true)+ semantic.env/get-pgvector-datasource! (constantly ::pgvector)+ semantic.env/get-index-metadata (constantly ::index-metadata)+ semantic.pgvector-api/ensure-active-hnsw-index!+ (fn [& _]+ (swap! calls inc)+ (.countDown started)+ (.await release))]+ (try+ (semantic.core/build-hnsw-index-async!)+ (is (.await started 5 TimeUnit/SECONDS) "the first build started")+ (semantic.core/build-hnsw-index-async!)+ (is (= 1 @calls) "the overlapping request did not start another future")+ (finally+ (.countDown release)))+ (is (.await reset-complete 5 TimeUnit/SECONDS) "the build future reset its local gate")))))+(deftest ^:sequential repair-snapshot-precedes-canonical-document-read-test(let [events (atom [])active-state (atom [nil {:metadata-row {:id 17}}])diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.cljindex 6eb212a8851..8b0d0b8bb6a 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.clj@@ -127,7 +127,7 @@[semantic.index-metadata/get-active-index-state (constantly active-state)semantic.health/active-index-queryable? (constantly true)semantic.settings/semantic-search-vector-strategy (constantly :hnsw)- semantic.util/index-ready? (constantly false)+ semantic.util/index-exists? (constantly false)semantic.embedding/embedder-circuit-untrusted? (constantly false)embedding-health/embedding-service-reachable? (constantly {:reachable? true :error nil})](is (=? {:health 0 :message #".*required HNSW index is missing.*"}diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.cljindex 01d93331561..dc9431f801a 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj@@ -13,6 +13,7 @@[metabase-enterprise.semantic-search.settings :as semantic.settings][metabase-enterprise.semantic-search.spec-trace-test-util :as spec-trace][metabase-enterprise.semantic-search.test-util :as semantic.tu]+ [metabase-enterprise.semantic-search.util :as semantic.util][metabase.analytics-interface.core :as analytics][metabase.api.common :as api][metabase.collections.models.collection :as collection]@@ -270,17 +271,32 @@"relfilenode is unchanged, so the index was not dropped, rebuilt, or reindexed"))))))(deftest query-index-hnsw-without-index-throws-test- (testing "a query under any HNSW-index-backed strategy fails fast when no HNSW index exists, rather than silently scanning"- (mt/with-premium-features #{:semantic-search}- (with-open [index-ref (semantic.tu/open-temp-index! :hnsw? false)]+ (mt/with-premium-features #{:semantic-search}+ (with-open [index-ref (semantic.tu/open-temp-index! :hnsw? false)]+ (testing "a query under any HNSW-index-backed strategy fails fast when no usable HNSW index exists"(is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref))))(doseq [strategy [:hnsw :hnsw-iterative-relaxed :hnsw-iterative-strict]](testing strategy(is (thrown-with-msg?- clojure.lang.ExceptionInfo #"no HNSW index exists"+ clojure.lang.ExceptionInfo #"no usable HNSW index exists"(semantic.index/query-index (semantic.env/get-pgvector-datasource!)@index-ref- {:search-string "puppy" :vector-search-strategy strategy})))))))))+ {:search-string "puppy" :vector-search-strategy strategy}))))))+ (testing "an active concurrent build temporarily uses the exact-scan path"+ (let [query (atom nil)]+ (mt/with-dynamic-fn-redefs+ [semantic.util/index-state (constantly :building)+ semantic.index/reducible-search-query (fn [_ q]+ (reset! query q)+ [])]+ (is (map? (semantic.index/query-index (semantic.env/get-pgvector-datasource!)+ @index-ref+ {:search-string "puppy" :vector-search-strategy :hnsw}))))+ (let [query-sql (first (sql/format @query :quoted true))]+ (is (str/includes? query-sql "AS MATERIALIZED")+ "a building HNSW index must use the materialized exact-scan query")+ (is (not (re-find #"ORDER BY embedding <=>[^)]*LIMIT" query-sql))+ "a building HNSW index must not use the approximate HNSW scan")))))))(deftest drop-index-table!-test(mt/with-premium-features #{:semantic-search}@@ -415,13 +431,19 @@(testing "ensure upsert! and delete! don't realize the full reducible at once"(semantic.tu/check-index-has-no-mock-docs)(testing "upsert-index!"- (with-redefs [semantic.index/upsert-index-pooled! (only-first-call realized @#'semantic.index/upsert-index-pooled!)]- (is (= {"card" 2} (semantic.tu/upsert-index! mock-docs))))+ (let [original-upsert-index-pooled! (mt/original-fn #'semantic.index/upsert-index-pooled!)]+ (mt/with-dynamic-fn-redefs+ [semantic.index/upsert-index-pooled!+ (only-first-call realized original-upsert-index-pooled!)]+ (is (= {"card" 2} (semantic.tu/upsert-index! mock-docs)))))(semantic.tu/check-index-has-mock-card))(reset! realized 0)(testing "delete-from-index!"- (with-redefs [semantic.index/delete-from-index-batch-sql (only-first-call realized @#'semantic.index/delete-from-index-batch-sql)]- (is (= {"card" 2} (semantic.tu/delete-from-index! "card" (eduction (map :id) mock-docs)))))+ (let [original-delete-from-index-batch-sql (mt/original-fn #'semantic.index/delete-from-index-batch-sql)]+ (mt/with-dynamic-fn-redefs+ [semantic.index/delete-from-index-batch-sql+ (only-first-call realized original-delete-from-index-batch-sql)]+ (is (= {"card" 2} (semantic.tu/delete-from-index! "card" (eduction (map :id) mock-docs))))))(semantic.tu/check-index-has-no-mock-docs))))))))(defn- track-concurrency@@ -444,6 +466,8 @@update-fn @#'semantic.index/upsert-index-batch!docs (take 100 (map-indexed (fn [i doc] (assoc doc :id (str i)))(cycle (semantic.tu/mock-documents))))]+ ;; This function is invoked by multiple index worker threads; dynamic redefs would not propagate to them.+ #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}(with-redefs [semantic.index/upsert-index-batch! (track-concurrencymax-concurrent(fn [& args] (apply update-fn args)))]@@ -526,6 +550,8 @@(binding [semantic.index/*batch-size* batch-size](let [{:keys [calls proxy]} (semantic.tu/spy semantic.embedding/process-embeddings-streaming)inter-batch-cache-hit? (atom false)]+ ;; These functions are invoked by the embedding worker; dynamic redefs would not propagate to it.+ #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}(with-redefs [semantic.embedding/process-embeddings-streaming proxysemantic.index/partition-existing-embeddings(let [orig @#'semantic.index/partition-existing-embeddings]diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.cljindex 7edfded2bd1..dbc4e623eb5 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj@@ -10,6 +10,7 @@[metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api][metabase-enterprise.semantic-search.settings :as semantic.settings][metabase-enterprise.semantic-search.test-util :as semantic.tu]+ [metabase-enterprise.semantic-search.util :as semantic.util][metabase.search.ingestion :as search.ingestion][metabase.test :as mt][metabase.util :as u]@@ -112,6 +113,57 @@(testing "is a noop when there is no active index"(is (nil? (semantic.pgvector-api/ensure-active-hnsw-index! pgvector index-metadata)))))))+(deftest ensure-hnsw-index-under-lock!-test+ (let [index {:table-name "index_1"}+ calls (atom [])]+ (mt/with-dynamic-fn-redefs+ [semantic.index/drop-index-concurrently-if-exists! (fn [& args] (swap! calls conj [:drop args]))+ semantic.index/create-hnsw-index-if-not-exists! (fn [& args] (swap! calls conj [:create args]))]+ (doseq [[state expected-operations] [[:ready []]+ [:building []]+ [nil [:create]]+ [:invalid [:drop :create]]]]+ (reset! calls [])+ (mt/with-dynamic-fn-redefs [semantic.util/index-state (constantly state)]+ (#'semantic.pgvector-api/ensure-hnsw-index-under-lock! ::connection index))+ (is (= expected-operations (mapv first @calls)) (str state " maintenance operations"))))))++(deftest with-hnsw-index-lock-serializes-callers-test+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ index-name (str "test-hnsw-lock-" (random-uuid))+ lock-name (str "metabase-semantic-hnsw:" index-name)+ entered (promise)+ release (promise)+ worker (future+ (#'semantic.pgvector-api/with-hnsw-index-lock+ pgvector+ index-name+ (fn [_]+ (deliver entered true)+ @release)))]+ (try+ (is (true? (deref entered 30000 false)) "the first caller acquired the lock")+ (with-open [probe (jdbc/get-connection pgvector)]+ (.setAutoCommit probe true)+ (is (false? (:locked (jdbc/execute-one!+ probe+ ["SELECT pg_try_advisory_lock(hashtextextended(?, 0)) AS locked" lock-name]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})))+ "a second caller cannot acquire the held lock"))+ (deliver release true)+ (is (not= ::timeout (deref worker 30000 ::timeout)) "the first caller released the lock")+ (with-open [probe (jdbc/get-connection pgvector)]+ (.setAutoCommit probe true)+ (is (true? (:locked (jdbc/execute-one!+ probe+ ["SELECT pg_try_advisory_lock(hashtextextended(?, 0)) AS locked" lock-name]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})))+ "the lock is available after the first caller releases it")+ (jdbc/execute-one! probe ["SELECT pg_advisory_unlock(hashtextextended(?, 0))" lock-name]))+ (finally+ (deliver release true)+ (deref worker 30000 nil)))))+(defn- open-semantic-search! ^Closeable [pgvector index-metadata embedding-model](semantic.tu/closeable(semantic.pgvector-api/init-semantic-search! pgvector index-metadata embedding-model)@@ -133,9 +185,9 @@(with-open [index-ref (open-semantic-search! pgvector index-metadata model1)];; no specific behaviour, only proxies the active index to the index search(testing "is only a proxy for the active index call"- (let [{:keys [proxy calls]} (semantic.tu/spy semantic.index/upsert-index!)+ (let [{:keys [proxy calls]} (semantic.tu/spy (mt/original-fn #'semantic.index/upsert-index!))documents (semantic.tu/mock-documents)]- (with-redefs [semantic.index/upsert-index! proxy]+ (mt/with-dynamic-fn-redefs [semantic.index/upsert-index! proxy](testing "check proxies correct args and ret is untouched"(let [ret (sut pgvector index-metadata documents)](is (= [{:args [pgvector @index-ref documents]@@ -164,8 +216,8 @@;; no specific behaviour, only proxies the active index to the index search(testing "is only a proxy for the active index call"(semantic.pgvector-api/index-documents! pgvector index-metadata documents)- (let [{:keys [calls proxy]} (semantic.tu/spy semantic.index/delete-from-index!)]- (with-redefs [semantic.index/delete-from-index! proxy]+ (let [{:keys [calls proxy]} (semantic.tu/spy (mt/original-fn #'semantic.index/delete-from-index!))]+ (mt/with-dynamic-fn-redefs [semantic.index/delete-from-index! proxy](testing "check proxies correct args and ret is untouched"(let [ret1 (sut pgvector index-metadata dash dash-ids)ret2 (sut pgvector index-metadata card card-ids)diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.cljindex b9aa772128a..c80f0209892 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj@@ -2,6 +2,9 @@(:require[clojure.test :refer :all][metabase-enterprise.semantic-search.core :as semantic.core]+ [metabase-enterprise.semantic-search.env :as semantic.env]+ [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]+ [metabase-enterprise.semantic-search.indexer :as semantic.indexer][metabase-enterprise.semantic-search.settings :as semantic.settings][metabase-enterprise.semantic-search.task.indexer :as sut][metabase-enterprise.semantic-search.util :as semantic.u]@@ -29,3 +32,22 @@(mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly strategy)](task/init! ::sut/SemanticSearchIndexer))(is (= 1 @builds))))))))++(deftest indexer-builds-only-absent-or-abandoned-hnsw-indexes-test+ (let [builds (atom 0)+ runs (atom 0)+ job (sut/->SemanticSearchIndexer)]+ (mt/with-dynamic-fn-redefs+ [semantic.u/semantic-search-active? (constantly true)+ semantic.env/get-pgvector-datasource! (constantly ::pgvector)+ semantic.env/get-index-metadata (constantly ::index-metadata)+ semantic.index-metadata/get-active-index-state (constantly {:index {:table-name "index_1"}})+ semantic.settings/semantic-search-vector-strategy (constantly :hnsw)+ semantic.core/build-hnsw-index-async! #(swap! builds inc)+ semantic.indexer/quartz-job-run! (fn [& _] (swap! runs inc))]+ (doseq [[state expected-builds] [[nil 1] [:invalid 1] [:building 0] [:ready 0]]]+ (reset! builds 0)+ (mt/with-dynamic-fn-redefs [semantic.u/index-state (constantly state)]+ (.execute ^org.quartz.Job job nil))+ (is (= expected-builds @builds) (str state " build count")))+ (is (= 4 @runs) "index maintenance continues in every catalog state"))))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.cljindex 733d2722240..343a004c095 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.clj@@ -33,21 +33,41 @@(is (= :index_table_1.model(semantic.util/conflict-target-column "semantic_search.index_table_1" "model")))))-(deftest index-ready?-catalog-query-test+(deftest index-state-test+ (testing "classifies catalog and concurrent-build state"+ (doseq [[row expected] [[nil nil]+ [{:is_ready true, :is_valid true, :is_building false} :ready]+ [{:is_ready false, :is_valid false, :is_building true} :building]+ [{:is_ready true, :is_valid false, :is_building false} :invalid]]]+ (mt/with-dynamic-fn-redefs [jdbc/execute-one! (constantly row)]+ (is (= expected (semantic.util/index-state ::pgvector "index_1"))))))+ (testing "only ready indexes exist; absent and abandoned invalid indexes need a build"+ (doseq [[state exists? needs-build?] [[nil false true]+ [:ready true false]+ [:building false false]+ [:invalid false true]]]+ (mt/with-dynamic-fn-redefs [semantic.util/index-state (constantly state)]+ (is (= exists? (semantic.util/index-exists? ::pgvector "index_1")))+ (is (= needs-build? (semantic.util/index-needs-build? ::pgvector "index_1")))))))++(deftest index-state-catalog-query-test(let [queries (atom [])](mt/with-dynamic-fn-redefs[jdbc/execute-one! (fn [_ query _](swap! queries conj query)- {:index_ready true})]- (is (true? (semantic.util/index-ready? ::pgvector "semantic_search.index_1")))- (is (true? (semantic.util/index-ready? ::pgvector "index_2"))))- (testing "both catalog queries require PostgreSQL to mark the index ready and valid"+ {:is_ready true, :is_valid true, :is_building false})]+ (is (= :ready (semantic.util/index-state ::pgvector "semantic_search.index_1")))+ (is (= :ready (semantic.util/index-state ::pgvector "index_2"))))+ (testing "both catalog queries distinguish active builds from abandoned invalid indexes"(doseq [[statement] @queries](is (re-find #"x\.indisready" statement))- (is (re-find #"x\.indisvalid" statement))))+ (is (re-find #"x\.indisvalid" statement))+ (is (re-find #"pg_stat_progress_create_index" statement))))(testing "qualified names retain their schema constraint"(is (= ["semantic_search" "index_1"] (rest (first @queries))))- (is (= ["index_2"] (rest (second @queries)))))))+ (is (= ["index_2"] (rest (second @queries))))+ (is (re-find #"ORDER BY CASE" (first (second @queries)))+ "unqualified names choose the best state deterministically"))))(deftest catalog-lookups-schema-scoping-test(testing "qualified names scope table/index existence checks to their schema; a same-named object in