Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Recover invalid semantic search HNSW indexes (#78428)

Chris Truter0d2b72368769
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.edn
index d1e05ec5329..7024ebb56cd 100644
--- a/.clj-kondo/ratchets.edn
+++ b/.clj-kondo/ratchets.edn
@@ -22,7 +22,7 @@
:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests 14
:metabase/modules 8
:metabase/namespace-name 4
- :metabase/prefer-with-dynamic-fn-redefs 25
+ :metabase/prefer-with-dynamic-fn-redefs 27
:metabase/test-helpers-use-non-thread-safe-functions 17
:metabase/validate-defendpoint-has-response-schema 441
:metabase/validate-defendpoint-query-params-use-kebab-case 40
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
index 5b41441e212..3e16252b003 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
@@ -53,19 +53,24 @@
(and (semantic.util/semantic-search-available?)
(semantic.embedding/embedding-supported? (semantic.embedding/get-configured-model))))
+(defonce ^:private hnsw-index-build-running? (atom false))
+
(defn build-hnsw-index-async!
"Build the HNSW index on the active semantic search index in the background, returning promptly.
- No-ops when semantic search isn't active on this instance. Backs the just-in-time HNSW build, which
- runs only when an instance is configured to the `:hnsw` vector-search strategy."
+ No-ops when semantic search isn't active or this process already has a build running. Database-level
+ coordination in [[semantic.pgvector-api/ensure-active-hnsw-index!]] serializes builds across instances."
[]
- (when (semantic.util/semantic-search-active?)
+ (when (and (semantic.util/semantic-search-active?)
+ (compare-and-set! hnsw-index-build-running? false true))
(future
(try
(semantic.pgvector-api/ensure-active-hnsw-index! (semantic.env/get-pgvector-datasource!)
(semantic.env/get-index-metadata))
(catch Throwable t
- (log/error t "Failed to build HNSW index for semantic search")))))
+ (log/error t "Failed to build HNSW index for semantic search"))
+ (finally
+ (reset! hnsw-index-build-running? false)))))
nil)
(defn- with-zero-semantic-distance-score
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/health.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/health.clj
index 04ac4c6d72b..5796b7bd2a5 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/health.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/health.clj
@@ -45,7 +45,7 @@
(try
(or (not (contains? search.config/hnsw-index-backed-strategies
(semantic.settings/semantic-search-vector-strategy)))
- (semantic.util/index-ready?
+ (semantic.util/index-exists?
pgvector
(semantic.index/schema-qualified-index-name index (semantic.index/hnsw-index-name index))))
(catch InterruptedException e
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
index 0bf4d0ec53d..3c45443f7f4 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
@@ -525,6 +525,12 @@
concurrently? (str/replace-first "CREATE INDEX " "CREATE INDEX CONCURRENTLY "))]
(jdbc/execute! connectable (into [sql] params))))
+(defn drop-index-concurrently-if-exists!
+ "Drop `index-name` without blocking writes. Must run outside a transaction."
+ [connectable index-name]
+ (jdbc/execute! connectable
+ [(str "DROP INDEX CONCURRENTLY IF EXISTS " (semantic.util/quote-table index-name))]))
+
(defn create-index-table-if-not-exists!
"Ensure that the index table exists and is ready to be populated. If
force-reset? is true, drops and recreates the table if it exists.
@@ -1179,16 +1185,23 @@
search-string (:search-string search-context)]
(if (str/blank? search-string)
{:results [] :raw-count 0}
- (do
+ (let [index-name (schema-qualified-index-name index (hnsw-index-name index))
+ index-state (when (contains? search.config/hnsw-index-backed-strategies
+ (vector-search-strategy search-context))
+ (semantic.util/index-state db index-name))
+ search-context (cond-> search-context
+ (= :building index-state) (assoc :vector-search-strategy :brute-force))]
;; `:vector-search-allow-missing-index?` is a deliberate opt-out for callers that want the inner
;; query to run without the HNSW index (e.g. the strategy matrix test probing the exact seq-scan
- ;; path); production traffic leaves it unset and gets the fail-fast.
+ ;; path). A concurrent build also uses that exact path until PostgreSQL marks the index ready; an
+ ;; absent or abandoned invalid index fails fast.
(when (and (contains? search.config/hnsw-index-backed-strategies (vector-search-strategy search-context))
(not (:vector-search-allow-missing-index? search-context))
- (not (semantic.util/index-exists? db (schema-qualified-index-name index (hnsw-index-name index)))))
- (throw (ex-info (str "HNSW-index-backed vector-search strategy requested but no HNSW index exists. "
- "Set the semantic-search-vector-strategy setting to an index-backed strategy "
- "(:hnsw or :hnsw-iterative-*) to build it.")
+ (contains? #{nil :invalid} index-state))
+ (throw (ex-info (str "HNSW-index-backed vector-search strategy requested but no usable HNSW index exists. "
+ "The index is absent or was abandoned invalid. It will be rebuilt by the next "
+ "maintenance pass; retry shortly, or pass :vector-search-allow-missing-index? true "
+ "to bypass this check.")
{:table-name (:table-name index)
:strategy (vector-search-strategy search-context)})))
(let [timer (u/start-timer)
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.clj
index 5939f5d55ca..3abebe30f75 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/pgvector_api.clj
@@ -16,9 +16,13 @@
[metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
[metabase-enterprise.semantic-search.repair :as semantic.repair]
[metabase-enterprise.semantic-search.settings :as semantic.settings]
+ [metabase-enterprise.semantic-search.util :as semantic.util]
[metabase.util :as u]
- [metabase.util.log :as log])
- (:import (java.time Instant)))
+ [metabase.util.log :as log]
+ [next.jdbc :as jdbc])
+ (:import
+ (java.sql Connection)
+ (java.time Instant)))
(set! *warn-on-reflection* true)
@@ -60,16 +64,51 @@
(semantic.index-metadata/activate-index! tx index-metadata index-id))
index))
+(defn- with-hnsw-index-lock
+ [pgvector index-name f]
+ (with-open [^Connection conn (jdbc/get-connection pgvector)]
+ (let [autocommit (.getAutoCommit conn)
+ lock-name (str "metabase-semantic-hnsw:" index-name)]
+ (.setAutoCommit conn true)
+ (try
+ (jdbc/execute-one! conn ["SELECT pg_advisory_lock(hashtextextended(?, 0))" lock-name])
+ (try
+ (f conn)
+ (finally
+ (jdbc/execute-one! conn ["SELECT pg_advisory_unlock(hashtextextended(?, 0))" lock-name])))
+ (finally
+ (.setAutoCommit conn autocommit))))))
+
+(defn- ensure-hnsw-index-under-lock!
+ [conn index]
+ (let [index-name (semantic.index/schema-qualified-index-name index (semantic.index/hnsw-index-name index))]
+ (case (semantic.util/index-state conn index-name)
+ :ready
+ (log/debug "HNSW index is already ready" index-name)
+ :building
+ (log/info "HNSW index build is already in progress" index-name)
+ :invalid
+ (do
+ (log/warn "Replacing invalid HNSW index" index-name)
+ (semantic.index/drop-index-concurrently-if-exists! conn index-name)
+ (semantic.index/create-hnsw-index-if-not-exists! conn index {:concurrently? true}))
+ nil
+ (do
+ (log/info "Building HNSW index" index-name)
+ (semantic.index/create-hnsw-index-if-not-exists! conn index {:concurrently? true})))))
+
(defn ensure-active-hnsw-index!
- "Build the HNSW index on the active index table if it does not already exist.
+ "Ensure that the active index has a usable HNSW index.
Called when an instance is (re)configured to the `:hnsw` vector-search strategy. No-ops when there is no
- active index. Builds the index with `CREATE INDEX CONCURRENTLY` since the table is typically populated."
+ active index. Serializes maintenance across instances, leaves active concurrent builds alone, and replaces
+ abandoned invalid indexes. Builds with concurrent DDL because the table is typically populated."
[pgvector index-metadata]
(if-let [{:keys [index]} (semantic.index-metadata/get-active-index-state pgvector index-metadata)]
(do
- (log/info "Building HNSW index for active semantic search index" (:table-name index))
- (semantic.index/create-hnsw-index-if-not-exists! pgvector index {:concurrently? true}))
+ (let [index-name (semantic.index/schema-qualified-index-name index (semantic.index/hnsw-index-name index))]
+ (with-hnsw-index-lock pgvector index-name #(ensure-hnsw-index-under-lock! % index)))
+ nil)
(log/info "No active semantic search index; skipping HNSW index build")))
(defn init-semantic-search!
@@ -177,8 +216,7 @@
(def index (:index index-state))
;; warning: deletes everything
- (require 'next.jdbc)
- (next.jdbc/execute! pgvector ((requiring-resolve 'honey.sql/format) {:delete-from (keyword (:table-name index))} :quoted true))
+ (jdbc/execute! pgvector ((requiring-resolve 'honey.sql/format) {:delete-from (keyword (:table-name index))} :quoted true))
(def searchable-documents (vec ((requiring-resolve 'metabase.search.ingestion/searchable-documents))))
(index-documents! pgvector index-metadata searchable-documents)
(mt/as-admin (query pgvector index-metadata {:search-string "sharp objects"})))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj
index 2a331c5eaba..af3b246bceb 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj
@@ -49,12 +49,12 @@
(if-let [{:keys [index]} (semantic.index-metadata/get-active-index-state pgvector index-metadata)]
(do
;; The strategy setter's build event no-ops while semantic is inactive, so an index-backed
- ;; strategy configured before (re)activation arrives here with no HNSW index. CONCURRENTLY
- ;; registers the index in pg_indexes as soon as the build starts, so this fires only once.
+ ;; strategy configured before (re)activation arrives here with no HNSW index. Retry absent
+ ;; or abandoned invalid indexes; an active concurrent build is left alone.
(let [hnsw-index (semantic.index/schema-qualified-index-name
index (semantic.index/hnsw-index-name index))]
(when (and (hnsw-strategy?)
- (not (semantic.u/index-exists? pgvector hnsw-index)))
+ (semantic.u/index-needs-build? pgvector hnsw-index))
(semantic.core/build-hnsw-index-async!)))
(semantic-search.indexer/quartz-job-run! pgvector index-metadata))
;; Engines can activate at runtime (license applied, additional-search-engines set on
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj
index 37e19af8307..ec817521d2a 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj
@@ -64,39 +64,50 @@
{:builder-fn jdbc.rs/as-unqualified-lower-maps})
(:table_exists false))))
+(defn index-state
+ "Return the catalog state of `index-name`: `:ready`, `:building`, `:invalid`, or `nil` when absent.
+ A schema-qualified name matches only within its schema; an unqualified name matches any schema."
+ [pgvector index-name]
+ (let [[schema index] (qualified-table-parts index-name)
+ columns (str "x.indisready AS is_ready, x.indisvalid AS is_valid, "
+ "EXISTS (SELECT 1 FROM pg_stat_progress_create_index p "
+ " WHERE p.index_relid = i.oid) AS is_building")]
+ (when-some [{:keys [is_ready is_valid is_building]}
+ (jdbc/execute-one! pgvector
+ (if schema
+ [(str "SELECT " columns " "
+ "FROM pg_class i "
+ "JOIN pg_namespace n ON n.oid = i.relnamespace "
+ "JOIN pg_index x ON x.indexrelid = i.oid "
+ "WHERE n.nspname = ? AND i.relname = ?")
+ schema index]
+ [(str "SELECT " columns " "
+ "FROM pg_class i "
+ "JOIN pg_index x ON x.indexrelid = i.oid "
+ "WHERE i.relname = "
+ "? "
+ "ORDER BY CASE "
+ "WHEN x.indisready AND x.indisvalid THEN 0 "
+ "WHEN EXISTS (SELECT 1 FROM pg_stat_progress_create_index p "
+ " WHERE p.index_relid = i.oid) THEN 1 "
+ "ELSE 2 END "
+ "LIMIT 1")
+ index])
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})]
+ (cond
+ (and is_ready is_valid) :ready
+ is_building :building
+ :else :invalid))))
+
(defn index-exists?
- "Does an index named `index-name` exist in the pgvector DB's pg_indexes?
- A schema-qualified (dotted) name matches only within its schema; an unqualified name matches any schema."
+ "Whether `index-name` is ready and valid. See [[index-state]]."
[pgvector index-name]
- (let [[schema index] (qualified-table-parts index-name)]
- (-> (jdbc/execute-one! pgvector
- (if schema
- ["SELECT exists (select 1 FROM pg_indexes WHERE schemaname = ? AND indexname = ?) index_exists"
- schema index]
- ["SELECT exists (select 1 FROM pg_indexes WHERE indexname = ?) index_exists"
- index])
- {:builder-fn jdbc.rs/as-unqualified-lower-maps})
- (:index_exists false))))
+ (= :ready (index-state pgvector index-name)))
-(defn index-ready?
- "Whether `index-name` is present and marked ready and valid by PostgreSQL. An in-progress or abandoned
- concurrent build is not ready. A schema-qualified name matches only within its schema."
+(defn index-needs-build?
+ "Whether `index-name` is absent or invalid with no concurrent build in progress."
[pgvector index-name]
- (let [[schema index] (qualified-table-parts index-name)]
- (-> (jdbc/execute-one! pgvector
- (if schema
- [(str "SELECT exists (SELECT 1 FROM pg_class i "
- "JOIN pg_namespace n ON n.oid = i.relnamespace "
- "JOIN pg_index x ON x.indexrelid = i.oid "
- "WHERE n.nspname = ? AND i.relname = ? "
- "AND x.indisready AND x.indisvalid) AS index_ready")
- schema index]
- [(str "SELECT exists (SELECT 1 FROM pg_class i "
- "JOIN pg_index x ON x.indexrelid = i.oid "
- "WHERE i.relname = ? AND x.indisready AND x.indisvalid) AS index_ready")
- index])
- {:builder-fn jdbc.rs/as-unqualified-lower-maps})
- (:index_ready false))))
+ (contains? #{nil :invalid} (index-state pgvector index-name)))
(defn semantic-search-configured?
"Whether to schedule the semantic-search Quartz jobs at startup.
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj
index b189edbe209..8cf3bc71a0e 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj
@@ -16,7 +16,9 @@
[metabase.search.in-place.legacy :as in-place.legacy]
[metabase.search.in-place.scoring :as in-place.scoring]
[metabase.search.settings :as search.settings]
- [metabase.test :as mt]))
+ [metabase.test :as mt])
+ (:import
+ (java.util.concurrent CountDownLatch TimeUnit)))
(set! *warn-on-reflection* true)
@@ -34,6 +36,35 @@
(constantly semantic.tu/mock-embedding-model)]
(is (true? (semantic.core/supported?))))))))
+(deftest ^:sequential build-hnsw-index-async-deduplicates-local-builds-test
+ (let [started (CountDownLatch. 1)
+ release (CountDownLatch. 1)
+ reset-complete (CountDownLatch. 1)
+ build-running? (atom false)
+ calls (atom 0)]
+ (add-watch build-running? ::reset-complete
+ (fn [_ _ previous current]
+ (when (and previous (not current))
+ (.countDown reset-complete))))
+ (with-redefs-fn {#'semantic.core/hnsw-index-build-running? build-running?}
+ #(mt/with-dynamic-fn-redefs
+ [semantic.util/semantic-search-active? (constantly true)
+ semantic.env/get-pgvector-datasource! (constantly ::pgvector)
+ semantic.env/get-index-metadata (constantly ::index-metadata)
+ semantic.pgvector-api/ensure-active-hnsw-index!
+ (fn [& _]
+ (swap! calls inc)
+ (.countDown started)
+ (.await release))]
+ (try
+ (semantic.core/build-hnsw-index-async!)
+ (is (.await started 5 TimeUnit/SECONDS) "the first build started")
+ (semantic.core/build-hnsw-index-async!)
+ (is (= 1 @calls) "the overlapping request did not start another future")
+ (finally
+ (.countDown release)))
+ (is (.await reset-complete 5 TimeUnit/SECONDS) "the build future reset its local gate")))))
+
(deftest ^:sequential repair-snapshot-precedes-canonical-document-read-test
(let [events (atom [])
active-state (atom [nil {:metadata-row {:id 17}}])
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.clj
index 6eb212a8851..8b0d0b8bb6a 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.clj
@@ -127,7 +127,7 @@
[semantic.index-metadata/get-active-index-state (constantly active-state)
semantic.health/active-index-queryable? (constantly true)
semantic.settings/semantic-search-vector-strategy (constantly :hnsw)
- semantic.util/index-ready? (constantly false)
+ semantic.util/index-exists? (constantly false)
semantic.embedding/embedder-circuit-untrusted? (constantly false)
embedding-health/embedding-service-reachable? (constantly {:reachable? true :error nil})]
(is (=? {:health 0 :message #".*required HNSW index is missing.*"}
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
index 01d93331561..dc9431f801a 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
@@ -13,6 +13,7 @@
[metabase-enterprise.semantic-search.settings :as semantic.settings]
[metabase-enterprise.semantic-search.spec-trace-test-util :as spec-trace]
[metabase-enterprise.semantic-search.test-util :as semantic.tu]
+ [metabase-enterprise.semantic-search.util :as semantic.util]
[metabase.analytics-interface.core :as analytics]
[metabase.api.common :as api]
[metabase.collections.models.collection :as collection]
@@ -270,17 +271,32 @@
"relfilenode is unchanged, so the index was not dropped, rebuilt, or reindexed"))))))
(deftest query-index-hnsw-without-index-throws-test
- (testing "a query under any HNSW-index-backed strategy fails fast when no HNSW index exists, rather than silently scanning"
- (mt/with-premium-features #{:semantic-search}
- (with-open [index-ref (semantic.tu/open-temp-index! :hnsw? false)]
+ (mt/with-premium-features #{:semantic-search}
+ (with-open [index-ref (semantic.tu/open-temp-index! :hnsw? false)]
+ (testing "a query under any HNSW-index-backed strategy fails fast when no usable HNSW index exists"
(is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref))))
(doseq [strategy [:hnsw :hnsw-iterative-relaxed :hnsw-iterative-strict]]
(testing strategy
(is (thrown-with-msg?
- clojure.lang.ExceptionInfo #"no HNSW index exists"
+ clojure.lang.ExceptionInfo #"no usable HNSW index exists"
(semantic.index/query-index (semantic.env/get-pgvector-datasource!)
@index-ref
- {:search-string "puppy" :vector-search-strategy strategy})))))))))
+ {:search-string "puppy" :vector-search-strategy strategy}))))))
+ (testing "an active concurrent build temporarily uses the exact-scan path"
+ (let [query (atom nil)]
+ (mt/with-dynamic-fn-redefs
+ [semantic.util/index-state (constantly :building)
+ semantic.index/reducible-search-query (fn [_ q]
+ (reset! query q)
+ [])]
+ (is (map? (semantic.index/query-index (semantic.env/get-pgvector-datasource!)
+ @index-ref
+ {:search-string "puppy" :vector-search-strategy :hnsw}))))
+ (let [query-sql (first (sql/format @query :quoted true))]
+ (is (str/includes? query-sql "AS MATERIALIZED")
+ "a building HNSW index must use the materialized exact-scan query")
+ (is (not (re-find #"ORDER BY embedding <=>[^)]*LIMIT" query-sql))
+ "a building HNSW index must not use the approximate HNSW scan")))))))
(deftest drop-index-table!-test
(mt/with-premium-features #{:semantic-search}
@@ -415,13 +431,19 @@
(testing "ensure upsert! and delete! don't realize the full reducible at once"
(semantic.tu/check-index-has-no-mock-docs)
(testing "upsert-index!"
- (with-redefs [semantic.index/upsert-index-pooled! (only-first-call realized @#'semantic.index/upsert-index-pooled!)]
- (is (= {"card" 2} (semantic.tu/upsert-index! mock-docs))))
+ (let [original-upsert-index-pooled! (mt/original-fn #'semantic.index/upsert-index-pooled!)]
+ (mt/with-dynamic-fn-redefs
+ [semantic.index/upsert-index-pooled!
+ (only-first-call realized original-upsert-index-pooled!)]
+ (is (= {"card" 2} (semantic.tu/upsert-index! mock-docs)))))
(semantic.tu/check-index-has-mock-card))
(reset! realized 0)
(testing "delete-from-index!"
- (with-redefs [semantic.index/delete-from-index-batch-sql (only-first-call realized @#'semantic.index/delete-from-index-batch-sql)]
- (is (= {"card" 2} (semantic.tu/delete-from-index! "card" (eduction (map :id) mock-docs)))))
+ (let [original-delete-from-index-batch-sql (mt/original-fn #'semantic.index/delete-from-index-batch-sql)]
+ (mt/with-dynamic-fn-redefs
+ [semantic.index/delete-from-index-batch-sql
+ (only-first-call realized original-delete-from-index-batch-sql)]
+ (is (= {"card" 2} (semantic.tu/delete-from-index! "card" (eduction (map :id) mock-docs))))))
(semantic.tu/check-index-has-no-mock-docs))))))))
(defn- track-concurrency
@@ -444,6 +466,8 @@
update-fn @#'semantic.index/upsert-index-batch!
docs (take 100 (map-indexed (fn [i doc] (assoc doc :id (str i)))
(cycle (semantic.tu/mock-documents))))]
+ ;; This function is invoked by multiple index worker threads; dynamic redefs would not propagate to them.
+ #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}
(with-redefs [semantic.index/upsert-index-batch! (track-concurrency
max-concurrent
(fn [& args] (apply update-fn args)))]
@@ -526,6 +550,8 @@
(binding [semantic.index/*batch-size* batch-size]
(let [{:keys [calls proxy]} (semantic.tu/spy semantic.embedding/process-embeddings-streaming)
inter-batch-cache-hit? (atom false)]
+ ;; These functions are invoked by the embedding worker; dynamic redefs would not propagate to it.
+ #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}
(with-redefs [semantic.embedding/process-embeddings-streaming proxy
semantic.index/partition-existing-embeddings
(let [orig @#'semantic.index/partition-existing-embeddings]
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj
index 7edfded2bd1..dbc4e623eb5 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/pgvector_api_test.clj
@@ -10,6 +10,7 @@
[metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
[metabase-enterprise.semantic-search.settings :as semantic.settings]
[metabase-enterprise.semantic-search.test-util :as semantic.tu]
+ [metabase-enterprise.semantic-search.util :as semantic.util]
[metabase.search.ingestion :as search.ingestion]
[metabase.test :as mt]
[metabase.util :as u]
@@ -112,6 +113,57 @@
(testing "is a noop when there is no active index"
(is (nil? (semantic.pgvector-api/ensure-active-hnsw-index! pgvector index-metadata)))))))
+(deftest ensure-hnsw-index-under-lock!-test
+ (let [index {:table-name "index_1"}
+ calls (atom [])]
+ (mt/with-dynamic-fn-redefs
+ [semantic.index/drop-index-concurrently-if-exists! (fn [& args] (swap! calls conj [:drop args]))
+ semantic.index/create-hnsw-index-if-not-exists! (fn [& args] (swap! calls conj [:create args]))]
+ (doseq [[state expected-operations] [[:ready []]
+ [:building []]
+ [nil [:create]]
+ [:invalid [:drop :create]]]]
+ (reset! calls [])
+ (mt/with-dynamic-fn-redefs [semantic.util/index-state (constantly state)]
+ (#'semantic.pgvector-api/ensure-hnsw-index-under-lock! ::connection index))
+ (is (= expected-operations (mapv first @calls)) (str state " maintenance operations"))))))
+
+(deftest with-hnsw-index-lock-serializes-callers-test
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ index-name (str "test-hnsw-lock-" (random-uuid))
+ lock-name (str "metabase-semantic-hnsw:" index-name)
+ entered (promise)
+ release (promise)
+ worker (future
+ (#'semantic.pgvector-api/with-hnsw-index-lock
+ pgvector
+ index-name
+ (fn [_]
+ (deliver entered true)
+ @release)))]
+ (try
+ (is (true? (deref entered 30000 false)) "the first caller acquired the lock")
+ (with-open [probe (jdbc/get-connection pgvector)]
+ (.setAutoCommit probe true)
+ (is (false? (:locked (jdbc/execute-one!
+ probe
+ ["SELECT pg_try_advisory_lock(hashtextextended(?, 0)) AS locked" lock-name]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})))
+ "a second caller cannot acquire the held lock"))
+ (deliver release true)
+ (is (not= ::timeout (deref worker 30000 ::timeout)) "the first caller released the lock")
+ (with-open [probe (jdbc/get-connection pgvector)]
+ (.setAutoCommit probe true)
+ (is (true? (:locked (jdbc/execute-one!
+ probe
+ ["SELECT pg_try_advisory_lock(hashtextextended(?, 0)) AS locked" lock-name]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})))
+ "the lock is available after the first caller releases it")
+ (jdbc/execute-one! probe ["SELECT pg_advisory_unlock(hashtextextended(?, 0))" lock-name]))
+ (finally
+ (deliver release true)
+ (deref worker 30000 nil)))))
+
(defn- open-semantic-search! ^Closeable [pgvector index-metadata embedding-model]
(semantic.tu/closeable
(semantic.pgvector-api/init-semantic-search! pgvector index-metadata embedding-model)
@@ -133,9 +185,9 @@
(with-open [index-ref (open-semantic-search! pgvector index-metadata model1)]
;; no specific behaviour, only proxies the active index to the index search
(testing "is only a proxy for the active index call"
- (let [{:keys [proxy calls]} (semantic.tu/spy semantic.index/upsert-index!)
+ (let [{:keys [proxy calls]} (semantic.tu/spy (mt/original-fn #'semantic.index/upsert-index!))
documents (semantic.tu/mock-documents)]
- (with-redefs [semantic.index/upsert-index! proxy]
+ (mt/with-dynamic-fn-redefs [semantic.index/upsert-index! proxy]
(testing "check proxies correct args and ret is untouched"
(let [ret (sut pgvector index-metadata documents)]
(is (= [{:args [pgvector @index-ref documents]
@@ -164,8 +216,8 @@
;; no specific behaviour, only proxies the active index to the index search
(testing "is only a proxy for the active index call"
(semantic.pgvector-api/index-documents! pgvector index-metadata documents)
- (let [{:keys [calls proxy]} (semantic.tu/spy semantic.index/delete-from-index!)]
- (with-redefs [semantic.index/delete-from-index! proxy]
+ (let [{:keys [calls proxy]} (semantic.tu/spy (mt/original-fn #'semantic.index/delete-from-index!))]
+ (mt/with-dynamic-fn-redefs [semantic.index/delete-from-index! proxy]
(testing "check proxies correct args and ret is untouched"
(let [ret1 (sut pgvector index-metadata dash dash-ids)
ret2 (sut pgvector index-metadata card card-ids)
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj
index b9aa772128a..c80f0209892 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj
@@ -2,6 +2,9 @@
(:require
[clojure.test :refer :all]
[metabase-enterprise.semantic-search.core :as semantic.core]
+ [metabase-enterprise.semantic-search.env :as semantic.env]
+ [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
+ [metabase-enterprise.semantic-search.indexer :as semantic.indexer]
[metabase-enterprise.semantic-search.settings :as semantic.settings]
[metabase-enterprise.semantic-search.task.indexer :as sut]
[metabase-enterprise.semantic-search.util :as semantic.u]
@@ -29,3 +32,22 @@
(mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly strategy)]
(task/init! ::sut/SemanticSearchIndexer))
(is (= 1 @builds))))))))
+
+(deftest indexer-builds-only-absent-or-abandoned-hnsw-indexes-test
+ (let [builds (atom 0)
+ runs (atom 0)
+ job (sut/->SemanticSearchIndexer)]
+ (mt/with-dynamic-fn-redefs
+ [semantic.u/semantic-search-active? (constantly true)
+ semantic.env/get-pgvector-datasource! (constantly ::pgvector)
+ semantic.env/get-index-metadata (constantly ::index-metadata)
+ semantic.index-metadata/get-active-index-state (constantly {:index {:table-name "index_1"}})
+ semantic.settings/semantic-search-vector-strategy (constantly :hnsw)
+ semantic.core/build-hnsw-index-async! #(swap! builds inc)
+ semantic.indexer/quartz-job-run! (fn [& _] (swap! runs inc))]
+ (doseq [[state expected-builds] [[nil 1] [:invalid 1] [:building 0] [:ready 0]]]
+ (reset! builds 0)
+ (mt/with-dynamic-fn-redefs [semantic.u/index-state (constantly state)]
+ (.execute ^org.quartz.Job job nil))
+ (is (= expected-builds @builds) (str state " build count")))
+ (is (= 4 @runs) "index maintenance continues in every catalog state"))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.clj
index 733d2722240..343a004c095 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.clj
@@ -33,21 +33,41 @@
(is (= :index_table_1.model
(semantic.util/conflict-target-column "semantic_search.index_table_1" "model")))))
-(deftest index-ready?-catalog-query-test
+(deftest index-state-test
+ (testing "classifies catalog and concurrent-build state"
+ (doseq [[row expected] [[nil nil]
+ [{:is_ready true, :is_valid true, :is_building false} :ready]
+ [{:is_ready false, :is_valid false, :is_building true} :building]
+ [{:is_ready true, :is_valid false, :is_building false} :invalid]]]
+ (mt/with-dynamic-fn-redefs [jdbc/execute-one! (constantly row)]
+ (is (= expected (semantic.util/index-state ::pgvector "index_1"))))))
+ (testing "only ready indexes exist; absent and abandoned invalid indexes need a build"
+ (doseq [[state exists? needs-build?] [[nil false true]
+ [:ready true false]
+ [:building false false]
+ [:invalid false true]]]
+ (mt/with-dynamic-fn-redefs [semantic.util/index-state (constantly state)]
+ (is (= exists? (semantic.util/index-exists? ::pgvector "index_1")))
+ (is (= needs-build? (semantic.util/index-needs-build? ::pgvector "index_1")))))))
+
+(deftest index-state-catalog-query-test
(let [queries (atom [])]
(mt/with-dynamic-fn-redefs
[jdbc/execute-one! (fn [_ query _]
(swap! queries conj query)
- {:index_ready true})]
- (is (true? (semantic.util/index-ready? ::pgvector "semantic_search.index_1")))
- (is (true? (semantic.util/index-ready? ::pgvector "index_2"))))
- (testing "both catalog queries require PostgreSQL to mark the index ready and valid"
+ {:is_ready true, :is_valid true, :is_building false})]
+ (is (= :ready (semantic.util/index-state ::pgvector "semantic_search.index_1")))
+ (is (= :ready (semantic.util/index-state ::pgvector "index_2"))))
+ (testing "both catalog queries distinguish active builds from abandoned invalid indexes"
(doseq [[statement] @queries]
(is (re-find #"x\.indisready" statement))
- (is (re-find #"x\.indisvalid" statement))))
+ (is (re-find #"x\.indisvalid" statement))
+ (is (re-find #"pg_stat_progress_create_index" statement))))
(testing "qualified names retain their schema constraint"
(is (= ["semantic_search" "index_1"] (rest (first @queries))))
- (is (= ["index_2"] (rest (second @queries)))))))
+ (is (= ["index_2"] (rest (second @queries))))
+ (is (re-find #"ORDER BY CASE" (first (second @queries)))
+ "unqualified names choose the best state deterministically"))))
(deftest catalog-lookups-schema-scoping-test
(testing "qualified names scope table/index existence checks to their schema; a same-named object in