Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Support HNSW iterative-scan + cross-test each strategy (#75722)

Chris Truter746a65d593e4
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
index b7718e4a250..8b6388d8537 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
@@ -470,9 +470,10 @@
(defn create-hnsw-index-if-not-exists!
"Create the HNSW index on the `embedding` column of `index`'s table, if it does not already exist.
- HNSW indexes are expensive to build and maintain, so we only create one when an instance is configured to
- use the `:hnsw` vector-search strategy (see [[create-index-table-if-not-exists!]] and the async build
- triggered by [[metabase-enterprise.semantic-search.settings/semantic-search-vector-strategy]]).
+ HNSW indexes are expensive to build and maintain, so we only create one when an instance is configured for
+ an HNSW-index-backed vector-search strategy (`:hnsw` or `:hnsw-iterative-*`; see
+ [[metabase.search.config/hnsw-index-backed-strategies]], [[create-index-table-if-not-exists!]], and the
+ async build triggered by [[metabase-enterprise.semantic-search.settings/semantic-search-vector-strategy]]).
Pass `concurrently? true` to build with `CREATE INDEX CONCURRENTLY` (for a populated table whose writes
shouldn't be locked out); it must run outside a transaction."
@@ -492,8 +493,9 @@
"Ensure that the index table exists and is ready to be populated. If
force-reset? is true, drops and recreates the table if it exists.
- The HNSW index is only created when the instance is configured for the `:hnsw` vector-search strategy;
- otherwise it is built just-in-time when the strategy is configured (see [[create-hnsw-index-if-not-exists!]])."
+ The HNSW index is only created when the instance is configured for an HNSW-index-backed vector-search
+ strategy (`:hnsw` or `:hnsw-iterative-*`); under `:brute-force` it is skipped, and built just-in-time when
+ an index-backed strategy is later configured (see [[create-hnsw-index-if-not-exists!]])."
[connectable index & {:keys [force-reset?] :or {force-reset? false}}]
(try
(let [{:keys [embedding-model table-name]} index
@@ -508,7 +510,8 @@
(-> (sql.helpers/create-table (keyword table-name) :if-not-exists)
(sql.helpers/with-columns (index-table-schema vector-dimensions))
sql-format-quoted))
- (when (= :hnsw (semantic-settings/semantic-search-vector-strategy))
+ (when (contains? search.config/hnsw-index-backed-strategies
+ (semantic-settings/semantic-search-vector-strategy))
(create-hnsw-index-if-not-exists! connectable index))
(jdbc/execute!
connectable
@@ -711,24 +714,108 @@
:order-by [[:semantic_rank :asc]]
:limit (semantic-settings/semantic-search-results-limit)}))
+(defn- hnsw-iterative-search-query
+ "Build the semantic vector subquery as an index-backed iterative scan with `filters` applied inline."
+ [index embedding-literal filters]
+ ;; The filters live inside the ordered/limited candidate scan (unlike `hnsw-search-query`, which
+ ;; post-filters), so the planner can pick the HNSW index and pgvector's iterative scan keeps pulling
+ ;; neighbours until the limit is met or `hnsw.max_scan_tuples` is hit.
+ ;; The recall/latency trade-off is governed by the iterative-scan GUCs (see `vector-session-settings`),
+ ;; not the SQL shape.
+ ;; The cutoff stays in the outer query (like `hnsw-search-query`) so the inner scan fills up to the limit
+ ;; before trimming.
+ (let [inner (cond-> {:select (into common-search-columns
+ [[[:raw (str "embedding <=> " embedding-literal)] :distance]])
+ :from [(keyword (:table-name index))]
+ :order-by [[[:raw (str "embedding <=> " embedding-literal)] :asc]]
+ :limit (semantic-settings/semantic-search-results-limit)}
+ filters (assoc :where filters))]
+ {:with [[:vector_candidates inner]]
+ :select (into common-search-columns
+ [[[:raw "row_number() OVER (ORDER BY distance ASC)"] :semantic_rank]
+ [:distance :semantic_distance]])
+ :from [:vector_candidates]
+ :where [:<= :distance max-cosine-distance]
+ :order-by [[:semantic_rank :asc]]}))
+
(defn- vector-search-strategy
"Resolve the vector-search strategy for `search-context`, falling back to the configured default setting."
[search-context]
(or (:vector-search-strategy search-context)
(semantic-settings/semantic-search-vector-strategy)))
+(def ^:private iterative-strategy->guc
+ "Iterative vector-search strategies mapped to pgvector's `hnsw.iterative_scan` GUC value.
+ The strategy keyword encodes the ordering, so there is no separate order knob."
+ {:hnsw-iterative-relaxed "relaxed_order"
+ :hnsw-iterative-strict "strict_order"})
+
(defn- semantic-search-query
"Build the semantic vector subquery, dispatching on the resolved [[vector-search-strategy]].
- `:brute-force` is exact and filter-first; `:hnsw` (the default) is approximate and index-backed."
+ `:brute-force` is exact and filter-first; the `:hnsw-iterative-*` strategies are index-backed with inline
+ filters and an iterative scan; `:hnsw` (the default) is approximate, index-backed and post-filters."
[index embedding search-context]
(let [filters (search-filters search-context)
embedding-literal (format-embedding embedding)
strategy (vector-search-strategy search-context)]
- (case strategy
- :brute-force (brute-force-search-query index embedding-literal filters)
- :hnsw (hnsw-search-query index embedding-literal filters)
- (do (log/warnf "Unknown vector-search strategy %s; falling back to :hnsw" (pr-str strategy))
- (hnsw-search-query index embedding-literal filters)))))
+ (cond
+ (= :brute-force strategy) (brute-force-search-query index embedding-literal filters)
+ (iterative-strategy->guc strategy) (hnsw-iterative-search-query index embedding-literal filters)
+ (= :hnsw strategy) (hnsw-search-query index embedding-literal filters)
+ :else (do (log/warnf "Unknown vector-search strategy %s; falling back to :brute-force" (pr-str strategy))
+ (brute-force-search-query index embedding-literal filters)))))
+
+(defn- explain?
+ "Whether to run the gated EXPLAIN (ANALYZE) instrumentation for `search-context`, falling back to the
+ configured default setting."
+ [search-context]
+ (if (contains? search-context :vector-search-explain?)
+ (boolean (:vector-search-explain? search-context))
+ (semantic-settings/semantic-search-explain)))
+
+(defn- vector-session-settings
+ "`SET LOCAL` statements for the pgvector session GUCs implied by `search-context`, as a (possibly empty)
+ vector of single-element `[sql]` statement vectors.
+ Only the `:hnsw-iterative-*` strategies set the iterative-scan GUCs; any strategy may request
+ `force-index?`.
+ Each knob falls back to its EE setting when unset on the context."
+ [search-context]
+ ;; Values come from a fixed map or validated positive integers, so they are safe to interpolate (GUC
+ ;; names and values can't be passed as bound parameters).
+ ;; They are clamped to pgvector's GUC ranges (ef_search 1..1000, max_scan_tuples 32-bit) because an
+ ;; out-of-range SET LOCAL raises mid-transaction and would fail the whole search; the API schema enforces
+ ;; the same ranges, the clamp covers the settings path.
+ (let [iterative-guc (iterative-strategy->guc (vector-search-strategy search-context))
+ clamp (fn [v lo hi] (-> v (max lo) (min hi)))]
+ (cond-> []
+ iterative-guc
+ (conj [(str "SET LOCAL hnsw.iterative_scan = " iterative-guc)]
+ [(format "SET LOCAL hnsw.ef_search = %d"
+ (clamp (or (:vector-search-ef-search search-context)
+ (semantic-settings/semantic-search-ef-search))
+ 1 1000))]
+ [(format "SET LOCAL hnsw.max_scan_tuples = %d"
+ (clamp (or (:vector-search-max-scan-tuples search-context)
+ (semantic-settings/semantic-search-max-scan-tuples))
+ 1 Integer/MAX_VALUE))])
+
+ (:vector-search-force-index? search-context)
+ (conj ["SET LOCAL enable_seqscan = off"]))))
+
+(defn- run-in-vector-session!
+ "Apply the [[vector-session-settings]] for `search-context` on a transaction over `db`, then call
+ `(f conn)` with that transaction.
+ `SET LOCAL` resets at COMMIT, so the pooled connection is left clean.
+ When no session settings are needed and instrumentation is off, `f` is called directly on `db` (no
+ transaction) to keep the default strategy's hot path unchanged."
+ [db search-context f]
+ (let [stmts (vector-session-settings search-context)]
+ (if (and (empty? stmts) (not (explain? search-context)))
+ (f db)
+ (jdbc/with-transaction [tx db]
+ (doseq [stmt stmts]
+ (jdbc/execute! tx stmt))
+ (f tx)))))
(defn- flatten-ctes
"Flatten nested :with clauses into a single top-level :with.
@@ -923,6 +1010,103 @@
(u/ignore-exceptions
(with-open [_conn (.getConnection ^PooledDataSource db)]))))))
+;; ----------------------------------------------------------------------------------------------------------
+;; Gated vector-search instrumentation
+;;
+;; When `vector-search-explain?` is set (per-request) or `semantic-search-explain` (setting) is on, we
+;; re-run the inner vector subquery under EXPLAIN (ANALYZE) to measure how the chosen strategy actually
+;; behaves: which plan the index-table scan got, how many tuples it visited, how long the inner query took,
+;; and how that compares to a filter-first scan's candidate pool.
+;; It is off by default because the EXPLAIN re-executes the inner query.
+;; ----------------------------------------------------------------------------------------------------------
+
+(defn- explain->element
+ "Coerce the single `EXPLAIN (… FORMAT JSON)` result value into its one plan element
+ (`{\"Plan\" {…} \"Execution Time\" …}`)."
+ [explain-value]
+ (let [decoded (cond
+ (instance? PGobject explain-value) (json/decode (unwrap-pgobject explain-value))
+ (string? explain-value) (json/decode explain-value)
+ :else explain-value)]
+ (first decoded)))
+
+(defn- find-scan-node
+ "Depth-first search of an EXPLAIN plan tree for the scan node over `table-name`."
+ [plan table-name]
+ (when (map? plan)
+ (if (and (#{"Seq Scan" "Index Scan" "Index Only Scan" "Bitmap Heap Scan"} (get plan "Node Type"))
+ (= table-name (get plan "Relation Name")))
+ plan
+ (some #(find-scan-node % table-name) (get plan "Plans")))))
+
+(defn- scan-node-metrics
+ "Pull the index-table scan node out of an EXPLAIN plan tree and summarise it: the plan node chosen, the
+ HNSW index used (nil for a seq scan), the tuples actually visited (returned + filtered out), and the
+ node's total execution time."
+ [plan table-name]
+ (when-let [node (find-scan-node plan table-name)]
+ ;; EXPLAIN reports per-loop averages, and for a parallel scan (the likely plan for the brute-force
+ ;; baseline) loops = workers: row totals need the multiply, but per-loop "Actual Total Time" already
+ ;; approximates wall clock because the workers run concurrently, so the time must NOT be multiplied
+ (let [loops (get node "Actual Loops" 1)
+ actual-rows (* (get node "Actual Rows" 0) loops)
+ removed (* (get node "Rows Removed by Filter" 0) loops)]
+ {:node-type (get node "Node Type")
+ :index-name (get node "Index Name")
+ :actual-rows actual-rows
+ :rows-removed-by-filter removed
+ :tuples-scanned (+ actual-rows removed)
+ :inner-ms (get node "Actual Total Time" 0)
+ :shared-hit (get node "Shared Hit Blocks")
+ :shared-read (get node "Shared Read Blocks")})))
+
+(defn- explain-vector-subquery
+ "Run the inner vector subquery under EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) on `conn` and return its
+ [[scan-node-metrics]] plus the subquery's total execution time.
+ Re-executes the inner query."
+ [conn index embedding search-context]
+ (let [[sql & params] (sql-format-quoted (semantic-search-query index embedding search-context))
+ explain-sql (str "EXPLAIN (ANALYZE true, BUFFERS true, FORMAT JSON) " sql)
+ element (-> (jdbc/execute-one! conn (into [explain-sql] params)) vals first explain->element)]
+ (assoc (scan-node-metrics (get element "Plan") (:table-name index))
+ :execution-ms (get element "Execution Time")
+ :planning-ms (get element "Planning Time"))))
+
+(defn- prefilter-pool-size
+ "Count of index rows the non-vector filters alone select -- the candidate pool a filter-first
+ (brute-force) scan would compute distances over.
+ The vector scan's `tuples-scanned` relative to this is the overfetch."
+ [conn index search-context]
+ (let [filters (search-filters search-context)
+ q (cond-> {:select [[[:raw "count(*)"] :n]] :from [(keyword (:table-name index))]}
+ filters (assoc :where filters))]
+ (:n (jdbc/execute-one! conn (sql-format-quoted q) {:builder-fn jdbc.rs/as-unqualified-lower-maps}))))
+
+(defn- record-vector-instrumentation!
+ "Measure and report how the resolved strategy executed the inner vector subquery: EXPLAIN ANALYZE the
+ scan and count the prefilter pool, then log a structured line and bump the analytics counters.
+ Never throws -- instrumentation failures must not break search."
+ [conn index embedding search-context raw-count]
+ (try
+ (let [strategy (name (vector-search-strategy search-context))
+ scan (explain-vector-subquery conn index embedding search-context)
+ pool (prefilter-pool-size conn index search-context)]
+ (log/info "Semantic vector-search instrumentation"
+ (merge {:strategy strategy
+ :prefilter-pool-size pool
+ :raw-count raw-count
+ :used-index? (some? (:index-name scan))
+ :overfetch-ratio (when (and pool (pos? pool) (:tuples-scanned scan))
+ (double (/ (:tuples-scanned scan) pool)))}
+ scan))
+ (analytics/inc! :metabase-search/semantic-vector-inner-ms {:strategy strategy} (or (:inner-ms scan) 0))
+ (analytics/inc! :metabase-search/semantic-vector-tuples-scanned {:strategy strategy} (or (:tuples-scanned scan) 0))
+ (analytics/inc! :metabase-search/semantic-prefilter-pool-size {:strategy strategy} (or pool 0))
+ (analytics/inc! :metabase-search/semantic-vector-scan-used-index
+ {:strategy strategy :plan-node (or (:node-type scan) "unknown")} 1))
+ (catch Exception e
+ (log/warn e "Failed to record vector-search instrumentation"))))
+
(defn query-index
"Query the index for documents similar to the search string.
Returns a map with :results and :raw-count."
@@ -932,11 +1116,17 @@
(if (str/blank? search-string)
{:results [] :raw-count 0}
(do
- (when (and (= :hnsw (vector-search-strategy search-context))
+ ;; `:vector-search-allow-missing-index?` is a deliberate opt-out for callers that want the inner
+ ;; query to run without the HNSW index (e.g. the strategy matrix test probing the exact seq-scan
+ ;; path); production traffic leaves it unset and gets the fail-fast.
+ (when (and (contains? search.config/hnsw-index-backed-strategies (vector-search-strategy search-context))
+ (not (:vector-search-allow-missing-index? search-context))
(not (semantic.util/index-exists? db (hnsw-index-name index))))
- (throw (ex-info (str "HNSW vector-search strategy requested but no HNSW index exists. "
- "Set the semantic-search-vector-strategy setting to :hnsw to build it.")
- {:table-name (:table-name index)})))
+ (throw (ex-info (str "HNSW-index-backed vector-search strategy requested but no HNSW index exists. "
+ "Set the semantic-search-vector-strategy setting to an index-backed strategy "
+ "(:hnsw or :hnsw-iterative-*) to build it.")
+ {:table-name (:table-name index)
+ :strategy (vector-search-strategy search-context)})))
(let [timer (u/start-timer)
;; Warm the pool concurrently with the embedding round-trip: the pool holds zero idle
@@ -958,11 +1148,24 @@
query (scored-search-query index embedding search-context scorers)
xform (comp (map decode-legacy-input)
(map (partial legacy-input-with-score weights (keys scorers))))
- reducible (reducible-search-query db query)
- raw-results (tracing/with-span :search "search.semantic.db-query"
- {:search/query-length (count search-string)}
- (into [] xform reducible))
- db-query-time-ms (u/since-ms db-timer)
+ ;; Run the query (and any gated instrumentation) inside a vector session so the iterative-scan
+ ;; GUCs from `vector-session-settings` apply on the same connection. Falls back to the plain
+ ;; datasource path for the default strategy with instrumentation off. The query duration is
+ ;; captured before the instrumentation runs, so the EXPLAIN re-execution and prefilter count
+ ;; don't inflate the latency signal they exist to analyze.
+ db-query-ms* (volatile! 0.0)
+ raw-results (run-in-vector-session!
+ db
+ search-context
+ (fn [conn]
+ (let [results (tracing/with-span :search "search.semantic.db-query"
+ {:search/query-length (count search-string)}
+ (into [] xform (reducible-search-query conn query)))]
+ (vreset! db-query-ms* (u/since-ms db-timer))
+ (when (explain? search-context)
+ (record-vector-instrumentation! conn index embedding search-context (count results)))
+ results)))
+ db-query-time-ms @db-query-ms*
filter-timer (u/start-timer)
filtered-results (tracing/with-span :search "search.semantic.permission-filter"
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj
index a74a52e09cf..ea2690b3bf0 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj
@@ -123,10 +123,12 @@
(defsetting semantic-search-vector-strategy
(deferred-tru
- (str "Default vector-search strategy for semantic search: `hnsw` (approximate, HNSW-index-backed) or "
+ (str "Default vector-search strategy for semantic search: `hnsw` (approximate, HNSW-index-backed), "
"`brute-force` (exact, applies non-vector filters first then computes cosine distance over the "
- "survivors). Defaults to `brute-force`, which needs no index; selecting `hnsw` builds the HNSW index "
- "just-in-time. Individual requests may override this via the `vector_search_strategy` API parameter."))
+ "survivors), or `hnsw-iterative-relaxed`/`hnsw-iterative-strict` (HNSW-index-backed iterative scans "
+ "with inline filters). Defaults to `brute-force`, which needs no index; selecting `hnsw` builds the "
+ "HNSW index just-in-time. Individual requests may override this via the `vector_search_strategy` API "
+ "parameter."))
:type :keyword
:default :brute-force
:encryption :no
@@ -142,9 +144,54 @@
{:invalid-value new-value
:valid-values valid-vector-search-strategies})))
(setting/set-value-of-type! :keyword :semantic-search-vector-strategy kw)
- ;; Gated on the transition (not every set) so re-setting :hnsw doesn't rebuild the index.
- (when (and (= kw :hnsw) (not= old :hnsw))
- (events/publish-event! :event/semantic-search-hnsw-enabled {})))))
+ ;; Every HNSW-index-backed strategy needs the index, so build it when transitioning into one
+ ;; from a non-index-backed strategy. Gated on the transition (not every set) so switching
+ ;; between index-backed strategies -- e.g. :hnsw -> :hnsw-iterative-strict -- doesn't rebuild.
+ (let [index-backed? search.config/hnsw-index-backed-strategies]
+ (when (and (index-backed? kw) (not (index-backed? old)))
+ (events/publish-event! :event/semantic-search-hnsw-enabled {}))))))
+
+(defsetting semantic-search-ef-search
+ (deferred-tru
+ (str "Default pgvector `hnsw.ef_search` (HNSW candidate-list size) for the `hnsw-iterative-*` strategies. "
+ "Larger values improve recall at the cost of latency. Individual requests may override this via the "
+ "`vector_search_ef_search` API parameter."))
+ :type :positive-integer
+ :default 40
+ :encryption :no
+ :export? false
+ :visibility :internal
+ :doc false)
+
+(defsetting semantic-search-max-scan-tuples
+ (deferred-tru
+ (str "Default pgvector `hnsw.max_scan_tuples` (soft cap on tuples an iterative scan visits) for the "
+ "`hnsw-iterative-*` strategies. Larger values improve recall under selective filters at the cost of "
+ "latency. Individual requests may override this via the `vector_search_max_scan_tuples` API parameter."))
+ :type :positive-integer
+ :default 20000
+ :encryption :no
+ :export? false
+ :visibility :internal
+ :doc false)
+
+;; The `vector_search_explain` API parameter only covers requests you author yourself; this setting exists
+;; to instrument organic traffic. The frontend issues the real search requests and cannot pass the
+;; parameter, so populating the vector-scan Prometheus counters and waterfall logs over production traffic
+;; means flipping instrumentation on instance-wide (MB_SEMANTIC_SEARCH_EXPLAIN on hosted, no deploy).
+;; EXPLAIN ANALYZE re-runs the inner vector subquery, so the intended lifecycle is on-for-an-investigation,
+;; then off.
+(defsetting semantic-search-explain
+ (deferred-tru
+ (str "Run EXPLAIN (ANALYZE) instrumentation of the inner vector subquery for every semantic search? "
+ "Expensive (re-executes the inner query); intended for ad-hoc analysis. Individual requests may "
+ "override this via the `vector_search_explain` API parameter."))
+ :type :boolean
+ :default false
+ :encryption :no
+ :export? false
+ :visibility :internal
+ :doc false)
(defsetting semantic-search-min-results-threshold
(deferred-tru "Minimum number of semantic search results required before falling back to other engines.")
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj
index bec65c03f0c..0045a21b5f5 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj
@@ -8,6 +8,7 @@
[metabase-enterprise.semantic-search.indexer :as semantic-search.indexer]
[metabase-enterprise.semantic-search.settings :as semantic.settings]
[metabase-enterprise.semantic-search.util :as semantic.u]
+ [metabase.search.config :as search.config]
[metabase.search.engine :as search.engine]
[metabase.task.core :as task]
[metabase.util.log :as log])
@@ -69,7 +70,9 @@
(simple/with-interval-in-milliseconds (.toMillis run-frequency))
(simple/repeat-forever))))]
(task/schedule-task! job trigger))
- ;; Safety net: an instance that booted already configured for :hnsw (strategy set via env var, or set
- ;; before an active index existed) never saw the setter's transition event, so build the index now.
- (when (= :hnsw (semantic.settings/semantic-search-vector-strategy))
+ ;; Safety net: an instance that booted already configured for an HNSW-index-backed strategy (set via env
+ ;; var, or set before an active index existed) never saw the setter's transition event, so build the
+ ;; index now. Covers :hnsw and the :hnsw-iterative-* strategies, which all query through the index.
+ (when (contains? search.config/hnsw-index-backed-strategies
+ (semantic.settings/semantic-search-vector-strategy))
(semantic.core/build-hnsw-index-async!))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
index 699e3629012..1f57e677627 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
@@ -51,8 +51,17 @@
(let [sql (vector-search-sql :hnsw)]
(is (not (str/includes? sql "MATERIALIZED")))
(is (re-find #"ORDER BY embedding <=>" sql))
- ;; the filter is applied in the outer query, after the candidate set is chosen
- (is (str/includes? sql "\"archived\" = FALSE"))))
+ ;; the filter is applied in the outer query, after the candidate set is chosen, so it appears *after*
+ ;; the inner ORDER BY embedding <=> ... LIMIT in the generated SQL
+ (is (str/includes? sql "\"archived\" = FALSE"))
+ (is (not (re-find #"(?s)\"archived\" = FALSE.*ORDER BY embedding <=>" sql)))))
+ (testing ":hnsw-iterative-* filters inline with the ordered/limited index scan (iterative scan)"
+ (doseq [strategy [:hnsw-iterative-relaxed :hnsw-iterative-strict]]
+ (let [sql (vector-search-sql strategy)]
+ (is (not (str/includes? sql "MATERIALIZED")))
+ ;; the filter lives *inside* the ordered+limited candidate scan (before the ORDER BY embedding <=>),
+ ;; so the HNSW index sees it and the iterative scan can extend until the limit is met
+ (is (re-find #"(?s)\"archived\" = FALSE.*ORDER BY embedding <=>" sql)))))
(testing "no explicit strategy falls back to the configured default setting"
(is (= (vector-search-sql (semantic.settings/semantic-search-vector-strategy))
(vector-search-sql nil)))))
@@ -80,10 +89,10 @@
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Invalid vector-search strategy"
(semantic.settings/semantic-search-vector-strategy! :nonsense))))
(testing "valid strategies round-trip"
- (doseq [strategy [:hnsw :brute-force]]
+ (doseq [strategy [:hnsw :brute-force :hnsw-iterative-relaxed :hnsw-iterative-strict]]
(mt/with-temporary-setting-values [semantic.settings/semantic-search-vector-strategy strategy]
(is (= strategy (semantic.settings/semantic-search-vector-strategy))))))
- (testing "the setter kicks off the background build job only on the transition into :hnsw"
+ (testing "the setter kicks off the background build job on the transition into any HNSW-index-backed strategy"
;; This asserts *when* the build is triggered (the transition gating), not what the build does -- the
;; build itself is covered by pgvector-api-test/ensure-active-hnsw-index!-test. The setter publishes
;; :event/semantic-search-hnsw-enabled, whose handler (semantic-search.events) calls the async build; we
@@ -96,32 +105,106 @@
(is (= 1 @triggers) "transitioning :brute-force -> :hnsw kicks off a build")
(semantic.settings/semantic-search-vector-strategy! :hnsw)
(is (= 1 @triggers) "setting :hnsw again when already :hnsw does not re-trigger")
+ ;; switching between index-backed strategies reuses the existing index, so no rebuild
+ (semantic.settings/semantic-search-vector-strategy! :hnsw-iterative-strict)
+ (is (= 1 @triggers) "switching :hnsw -> :hnsw-iterative-strict does not re-trigger")
(semantic.settings/semantic-search-vector-strategy! :brute-force)
- (is (= 1 @triggers) "switching away from :hnsw does not trigger")
- (semantic.settings/semantic-search-vector-strategy! :hnsw)
- (is (= 2 @triggers) "transitioning back into :hnsw triggers again")))))))
+ (is (= 1 @triggers) "switching away to :brute-force does not trigger")
+ (semantic.settings/semantic-search-vector-strategy! :hnsw-iterative-relaxed)
+ (is (= 2 @triggers) "transitioning :brute-force -> :hnsw-iterative-relaxed kicks off a build")))))))
+
+(deftest vector-session-settings-test
+ (testing ":hnsw and :brute-force need no session GUCs"
+ (is (empty? (#'semantic.index/vector-session-settings {:vector-search-strategy :hnsw})))
+ (is (empty? (#'semantic.index/vector-session-settings {:vector-search-strategy :brute-force}))))
+ (testing ":hnsw-iterative-* emits iterative_scan/ef_search/max_scan_tuples SET LOCALs; the strategy sets the order"
+ (is (= ["SET LOCAL hnsw.iterative_scan = strict_order"
+ "SET LOCAL hnsw.ef_search = 100"
+ "SET LOCAL hnsw.max_scan_tuples = 50000"]
+ (map first (#'semantic.index/vector-session-settings
+ {:vector-search-strategy :hnsw-iterative-strict
+ :vector-search-ef-search 100
+ :vector-search-max-scan-tuples 50000}))))
+ (is (= "SET LOCAL hnsw.iterative_scan = relaxed_order"
+ (first (first (#'semantic.index/vector-session-settings
+ {:vector-search-strategy :hnsw-iterative-relaxed}))))))
+ (testing "force-index? appends enable_seqscan = off for any strategy"
+ (is (= ["SET LOCAL enable_seqscan = off"]
+ (map first (#'semantic.index/vector-session-settings
+ {:vector-search-strategy :hnsw :vector-search-force-index? true}))))))
+
+(deftest ^:synchronized semantic-search-instrumentation-test
+ (mt/with-premium-features #{:semantic-search}
+ (with-open [_ (semantic.tu/open-temp-index!)]
+ (semantic.tu/upsert-index! (semantic.tu/mock-documents))
+ (testing "vector-search-explain? runs EXPLAIN ANALYZE and emits the vector-scan instrumentation metrics"
+ (let [analytics-calls (atom [])]
+ (mt/with-dynamic-fn-redefs [analytics/inc! (fn [metric & args]
+ (swap! analytics-calls conj [metric args]))]
+ (mt/with-test-user :crowberto
+ (semantic.index/query-index (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index
+ {:search-string "dog training" :vector-search-explain? true}))
+ (let [by-metric (into {} (map (juxt first (comp vec second))) @analytics-calls)]
+ (testing "all four instrumentation metrics fire with numeric values"
+ (doseq [metric [:metabase-search/semantic-vector-inner-ms
+ :metabase-search/semantic-vector-tuples-scanned
+ :metabase-search/semantic-prefilter-pool-size]]
+ (is (contains? by-metric metric))
+ ;; args are [labels amount]; the amount must be a non-negative number
+ (is (number? (second (by-metric metric))) (str metric " amount should be numeric"))
+ (is (<= 0 (second (by-metric metric))) (str metric " amount should be non-negative"))))
+ (testing "the scan plan node is reported as a label"
+ (is (contains? by-metric :metabase-search/semantic-vector-scan-used-index))
+ (is (string? (get-in (by-metric :metabase-search/semantic-vector-scan-used-index)
+ [0 :plan-node]))))))))
+ (testing "with instrumentation off (the default) no instrumentation metrics are emitted"
+ (let [analytics-calls (atom [])]
+ (mt/with-dynamic-fn-redefs [analytics/inc! (fn [metric & args]
+ (swap! analytics-calls conj [metric args]))]
+ (mt/with-test-user :crowberto
+ (semantic.index/query-index (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index
+ {:search-string "dog training"}))
+ (is (not (contains? (set (map first @analytics-calls))
+ :metabase-search/semantic-vector-inner-ms)))))))))
+
+(defn- expected-index-defs
+ "The index-DDL contract of [[semantic.index/create-index-table-if-not-exists!]] for `index`.
+ Maps the name of each index it must create to a pattern its pg_indexes indexdef must match."
+ [index]
+ ;; the pkey and (model, model_id) unique-constraint indexes come from the table schema, not CREATE INDEX
+ ;; statements, so they are deliberately absent here; =? tolerates them as extra keys in the actual
+ {(semantic.index/hnsw-index-name index) #"CREATE INDEX .* USING hnsw \(embedding vector_cosine_ops\)"
+ (semantic.index/fts-index-name index) #"CREATE INDEX .* USING gin \(text_search_vector\)"
+ (semantic.index/fts-native-index-name index) #"CREATE INDEX .* USING gin \(text_search_with_native_query_vector\)"
+ (#'semantic.index/content-index-name index) #"CREATE INDEX .* USING btree \(content\)"})
(deftest create-index-table!-test
(mt/with-premium-features #{:semantic-search}
(with-open [index-ref (semantic.tu/open-temp-index! :hnsw? false)]
- ;; open-temp-index! creates the temp table, so drop it in order to test create!.
- (semantic.index/drop-index-table! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index)
- (testing "index table is not present before create!"
- (is (not (semantic.tu/table-exists-in-db? (:table-name @index-ref))))
- (is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref))))
- (is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-index-name @index-ref))))
- (is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-native-index-name @index-ref)))))
- (testing "under the default :brute-force strategy, create! builds the table and FTS indexes but not the HNSW index"
- (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly :brute-force)]
- (semantic.index/create-index-table-if-not-exists! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index {:force-reset? true}))
- (is (semantic.tu/table-exists-in-db? (:table-name @index-ref)))
- (is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref))))
- (is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-index-name @index-ref)))
- (is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-native-index-name @index-ref))))
- (testing "when configured for the :hnsw strategy, create! also builds the HNSW index"
- (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly :hnsw)]
- (semantic.index/create-index-table-if-not-exists! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index {:force-reset? true}))
- (is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref)))))))
+ (let [index @index-ref
+ table-name (:table-name index)]
+ ;; open-temp-index! creates the temp table, so drop it in order to test create!.
+ (semantic.index/drop-index-table! (semantic.env/get-pgvector-datasource!) index)
+ (testing "neither the table nor any of its indexes is present before create!"
+ (is (not (semantic.tu/table-exists-in-db? table-name)))
+ (is (= {} (semantic.tu/table-indexes table-name))))
+ (testing "under the default :brute-force strategy, create! builds the table and FTS indexes but not the HNSW index"
+ (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly :brute-force)]
+ (semantic.index/create-index-table-if-not-exists!
+ (semantic.env/get-pgvector-datasource!) index {:force-reset? true}))
+ (is (semantic.tu/table-exists-in-db? table-name))
+ (is (=? (dissoc (expected-index-defs index) (semantic.index/hnsw-index-name index))
+ (semantic.tu/table-indexes table-name)))
+ (is (not (semantic.tu/table-has-index? table-name (semantic.index/hnsw-index-name index)))))
+ (testing "under the :hnsw strategy, force-reset? rebuilds the full contract (incl. HNSW) and wipes data"
+ ;; seed rows first; rows that survive the reset would mean force-reset? was silently ignored
+ (semantic.tu/upsert-index! (semantic.tu/mock-documents))
+ (is (pos? (semantic.tu/index-count index)))
+ (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly :hnsw)]
+ (semantic.index/create-index-table-if-not-exists!
+ (semantic.env/get-pgvector-datasource!) index {:force-reset? true}))
+ (is (zero? (semantic.tu/index-count index)))
+ (is (=? (expected-index-defs index) (semantic.tu/table-indexes table-name))))))))
(deftest create-hnsw-index-if-not-exists!-test
(mt/with-premium-features #{:semantic-search}
@@ -138,15 +221,17 @@
"relfilenode is unchanged, so the index was not dropped, rebuilt, or reindexed"))))))
(deftest query-index-hnsw-without-index-throws-test
- (testing "a query using the :hnsw strategy fails fast when no HNSW index exists, rather than silently scanning"
+ (testing "a query under any HNSW-index-backed strategy fails fast when no HNSW index exists, rather than silently scanning"
(mt/with-premium-features #{:semantic-search}
(with-open [index-ref (semantic.tu/open-temp-index! :hnsw? false)]
(is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref))))
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo #"no HNSW index exists"
- (semantic.index/query-index (semantic.env/get-pgvector-datasource!)
- @index-ref
- {:search-string "puppy" :vector-search-strategy :hnsw})))))))
+ (doseq [strategy [:hnsw :hnsw-iterative-relaxed :hnsw-iterative-strict]]
+ (testing strategy
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo #"no HNSW index exists"
+ (semantic.index/query-index (semantic.env/get-pgvector-datasource!)
+ @index-ref
+ {:search-string "puppy" :vector-search-strategy strategy})))))))))
(deftest drop-index-table!-test
(mt/with-premium-features #{:semantic-search}
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj
index 52362c63951..62b623b8d0c 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj
@@ -11,10 +11,10 @@
(set! *warn-on-reflection* true)
(deftest startup-hnsw-safety-net-test
- (testing "the indexer task's startup builds the HNSW index only when configured for :hnsw"
- ;; Covers instances that boot already configured for :hnsw (e.g. strategy set via env var), where the
- ;; setter's transition event never fired -- see the safety net in sut/init!. Redefs avoid scheduling a
- ;; real quartz job or spawning a real build future.
+ (testing "the indexer task's startup builds the HNSW index when configured for any index-backed strategy"
+ ;; Covers instances that boot already configured for an HNSW-index-backed strategy (e.g. strategy set via
+ ;; env var), where the setter's transition event never fired -- see the safety net in sut/init!. Redefs
+ ;; avoid scheduling a real quartz job or spawning a real build future.
(let [builds (atom 0)]
(mt/with-dynamic-fn-redefs [semantic.u/semantic-search-available? (constantly true)
task/schedule-task! (fn [& _] nil)
@@ -23,7 +23,9 @@
(mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly :brute-force)]
(task/init! ::sut/SemanticSearchIndexer))
(is (zero? @builds)))
- (testing ":hnsw triggers a build"
- (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly :hnsw)]
- (task/init! ::sut/SemanticSearchIndexer))
- (is (= 1 @builds)))))))
+ (doseq [strategy [:hnsw :hnsw-iterative-relaxed :hnsw-iterative-strict]]
+ (testing (str strategy " triggers a build")
+ (reset! builds 0)
+ (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly strategy)]
+ (task/init! ::sut/SemanticSearchIndexer))
+ (is (= 1 @builds))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
index bfdd4c2b31a..2655afe530b 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
@@ -514,6 +514,16 @@
(-> result first vals first))
(catch Exception _ false))))
+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
+(defn table-indexes
+ "Map of index name -> pg_indexes indexdef for `table-name`; empty when the table does not exist."
+ [table-name]
+ (into {}
+ (map (juxt :indexname :indexdef))
+ (jdbc/execute! (semantic.env/get-pgvector-datasource!)
+ ["SELECT indexname, indexdef FROM pg_indexes WHERE tablename = ?" (name table-name)]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})))
+
#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
(defn index-relfilenode
"Return the on-disk relfilenode for the index named `index-name`, or nil if it doesn't exist. A stable
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/vector_strategy_matrix_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/vector_strategy_matrix_test.clj
new file mode 100644
index 00000000000..8024c821699
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/vector_strategy_matrix_test.clj
@@ -0,0 +1,548 @@
+(ns metabase-enterprise.semantic-search.vector-strategy-matrix-test
+ "Quality matrix over the vector-search strategies on a small deterministic dataset.
+
+ Headline:
+
+ - naive :hnsw -- too risky around under-fetching.
+ An anti-correlated :verified filter empties its pool outright (final recall 0.0 where :brute-force
+ scores 1.0), and even a half-selective filter halves it before the permission drop halves it again
+ ({:raw-count 10 :returned 6} against 20/16 for every other variant).
+ - :brute-force -- reliable, but poor asymptotic performance.
+ Exact in every cell of the matrix, at the cost of computing distances over the whole filtered table.
+ And because that cost tracks the filtered pool size, latency varies query-to-query with the filters --
+ intermittent slowness that can frustrate more than a uniformly slower engine.
+ - :hnsw-iterative-* -- promising, but can still underperform.
+ It recovers everything the naive scan loses to filters (recall 1.0 vs 0.0) at index-backed cost.
+ Yet approximation shows up at real dimensionality (recall@20 observed at 0.15-0.6 on the 32-d packed
+ dataset with ef_search 16), a binding hnsw.max_scan_tuples empties a filtered scan that :brute-force
+ answers exactly, and the post-retrieval re-ranking and permission stages lose just as much through it
+ (0.4 recall under limit truncation, a 20 -> 12 pool drop) because they run after retrieval.
+
+ Caveat: every number above comes from a very, very small index (300 docs; 2000 for the packed dataset).
+ The mechanisms are the same at scale, but their effects can be far more drastic.
+
+ Covers four retrieval variants -- no HNSW index (exact seq scan), naive :hnsw (post-filter), and the two
+ iterative-scan strategies -- with :brute-force as the exact SQL reference, and measures three things:
+
+ 1. distance recall / NDCG of candidate retrieval against a ground truth computed in pure Clojure,
+ 2. recall of the final re-ranked top results against an exhaustive run of the same scoring pipeline,
+ 3. under-fetch caused by post-retrieval permission filtering.
+
+ The trade-offs demonstrated: even exact retrieval under-delivers end-to-end (the candidate pool is
+ truncated at `semantic-search-results-limit` before re-ranking, and permission checks drop candidates
+ after), naive :hnsw misses everything under an anti-correlated filter, and the iterative strategies fix
+ filter-driven misses but not limit truncation or permission under-fetch.
+ Under a half-selective filter naive :hnsw also under-fetches at the SQL stage before the permission drop
+ cuts again -- the two losses compound for the post-filtering strategy only.
+
+ The matrix cells assert exact values and orderings, not thresholds.
+ Embeddings sit at exact distinct distances from the probe (no ties), the heap is populated in a fixed
+ order before the index exists, table stats are pinned with ANALYZE, and every indexed query forces the
+ index path (a 300-row table would otherwise always seq-scan).
+ The HNSW graph itself is NOT reproducible (pgvector draws node levels from the per-backend global PRNG,
+ which `setseed` does not touch), so exact cells are only those whose outcome is invariant across graphs;
+ each was validated against 30-60 independently built graphs.
+ On the 4-d matrix dataset the graph is exact regardless of its random shape -- even 3000 densely packed
+ filler docs do not change that -- so the matrix pins the structural differences between the strategies'
+ SQL shapes and scan termination rules.
+ Graph approximation error needs dimensionality, not tree size: [[graph-approximation-test]] covers that
+ axis on a 32-d densely packed dataset with banded assertions (band edges sit several misses away from
+ everything observed across dozens of graphs)."
+ (:require
+ [clojure.core.memoize :as memoize]
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [metabase-enterprise.semantic-search.env :as semantic.env]
+ [metabase-enterprise.semantic-search.index :as semantic.index]
+ [metabase-enterprise.semantic-search.scoring :as semantic.scoring]
+ [metabase-enterprise.semantic-search.test-util :as semantic.tu]
+ [metabase.permissions.models.permissions :as perms]
+ [metabase.permissions.models.permissions-group :as perms-group]
+ [metabase.test :as mt]
+ [next.jdbc :as jdbc])
+ (:import
+ (java.util Random)))
+
+(set! *warn-on-reflection* true)
+
+(use-fixtures :once #'semantic.tu/once-fixture)
+
+;;;; quality metrics
+
+(def ^:private cutoff
+ "The production cosine-distance cutoff; the dataset's bands are designed around it."
+ @#'semantic.index/max-cosine-distance)
+
+(defn- recall-at
+ "Fraction of the ideal top-`k` ids present in the returned top-`k`."
+ [k ideal-ids returned-ids]
+ (let [ideal (set (take k ideal-ids))]
+ (double (/ (count (filter ideal (take k returned-ids)))
+ (count ideal)))))
+
+(defn- sim
+ "Graded relevance of a cosine distance `d` in [0,2]: 1 (identical) .. 0 (opposite)."
+ [d]
+ (max 0.0 (- 1.0 (/ (double d) 2.0))))
+
+(defn- dcg
+ "Discounted cumulative gain of a sequence of per-position relevances."
+ [rels]
+ (transduce (map-indexed (fn [i r] (/ (double r) (/ (Math/log (+ i 2.0)) (Math/log 2.0))))) + 0.0 rels))
+
+(defn- ndcg-at
+ "NDCG@`k` of `returned-ids` against the exact `ideal-ids` ranking.
+ Relevance is graded by true distance via `id->distance`."
+ [k id->distance ideal-ids returned-ids]
+ (/ (dcg (map (comp sim id->distance) (take k returned-ids)))
+ (dcg (map (comp sim id->distance) (take k ideal-ids)))))
+
+;;;; dataset
+
+(def ^:private probe-text "zebra quark")
+(def ^:private probe-vector [1.0 0.0 0.0 0.0])
+
+(def ^:private bands
+ "Band layout for the 300-doc dataset, in insertion (and ascending-distance) order.
+
+ - :restricted sits nearest the probe but in a permission-restricted collection, so it occupies top-k slots
+ that the permission filter then drops (under-fetch).
+ - :core/:mid are bulk filler so retrieval is nowhere near saturated.
+ - :pinned is inside the cutoff but far outside any candidate pool -- the re-ranking lever for layer 2.
+ - :rare holds the only verified docs and is anti-correlated with the probe: a :verified filter on it is
+ adversarial for post-filtering strategies. The adversarial filter must be a column with no supporting
+ btree: a :models filter would be served exactly by the (model, model_id) unique constraint at this table
+ size, bypassing the HNSW scan the matrix is exercising.
+ - :decoy lies beyond the cutoff and must never be returned."
+ [{:band :restricted :n 8 :model "card" :d0 0.020 :dstep 0.005}
+ {:band :core :n 180 :model "card" :d0 0.100 :dstep 0.0015}
+ {:band :mid :n 74 :model "card" :d0 0.400 :dstep 0.0008}
+ {:band :pinned :n 6 :model "card" :d0 0.465 :dstep 0.002 :pinned true}
+ {:band :rare :n 12 :model "card" :d0 0.550 :dstep 0.008 :verified true}
+ {:band :decoy :n 20 :model "card" :d0 0.750 :dstep 0.007}])
+
+(defn- vector-at-distance
+ "A unit `dims`-d vector at exact cosine distance `d` (pgvector `<=>`) from the probe `[1 0 0 ...]`."
+ [^Random rng dims d]
+ ;; the probe component is fixed by `d` and the remainder is a seeded random unit direction orthogonal to
+ ;; the probe axis, so the cosine against the probe is exactly 1 - d
+ (let [u (loop []
+ (let [g (repeatedly (dec dims) #(.nextGaussian rng))
+ n (Math/sqrt (double (reduce + (map * g g))))]
+ (if (< n 1e-6) (recur) (mapv #(/ % n) g))))
+ a (- 1.0 d)
+ b (Math/sqrt (- 1.0 (* a a)))]
+ (into [a] (map #(* b %)) u)))
+
+(defn- make-dataset
+ "Build the deterministic banded dataset.
+ Returns {:docs [...], :embeddings {text -> vec}, :id->distance {id -> d}, :id->band {id -> band}}.
+ The creator alternates with id parity (odd -> crowberto, even -> rasta), so a `created-by` filter is
+ half-selective in every band.
+ Doc names share no stem with [[probe-text]], so the keyword side of the hybrid query never matches and the
+ RRF rank reduces to the pure semantic (distance) rank."
+ [{:keys [seed restricted-collection-id]}]
+ (let [rng (Random. (long seed))
+ rows (for [{:keys [band n model d0 dstep pinned verified]} bands
+ i (range n)]
+ {:band band :model model :pinned pinned :verified verified :d (+ d0 (* i dstep))})
+ rows (map-indexed (fn [idx row]
+ (let [{:keys [band model]} row]
+ (assoc row
+ :id (inc idx)
+ :name (format "%s %s %03d" (name band) model (inc idx))
+ :vector (vector-at-distance rng 4 (:d row)))))
+ rows)]
+ {:docs (vec (for [{:keys [band model pinned verified d id name]} rows]
+ (let [collection-id (when (= band :restricted) restricted-collection-id)]
+ {:model model
+ :id id
+ :name name
+ :searchable_text name
+ :embeddable_text name
+ :created_at #t "2025-01-01T12:00:00Z"
+ :creator_id (mt/user->id (if (odd? id) :crowberto :rasta))
+ :archived false
+ :pinned (boolean pinned)
+ :verified (boolean verified)
+ :collection_id collection-id
+ ;; the permission filter runs on the decoded legacy_input, so like the real
+ ;; ingestion documents it must carry collection_id itself
+ :legacy_input {:id id :model model :name name :collection_id collection-id}
+ :metadata {:title name :distance d}})))
+ :embeddings (into {probe-text probe-vector} (map (juxt :name :vector)) rows)
+ :id->distance (into {} (map (juxt :id :d)) rows)
+ :id->band (into {} (map (juxt :id :band)) rows)}))
+
+(defn- exact-ids
+ "Ids of docs satisfying `pred`, within the distance cutoff, in exact ascending-distance order.
+ This is the pure-Clojure ground truth the SQL retrieval is measured against."
+ [{:keys [docs id->distance]} pred]
+ (->> docs
+ (filter pred)
+ (filter #(<= (id->distance (:id %)) cutoff))
+ (sort-by #(id->distance (:id %)))
+ (mapv :id)))
+
+(defn- band-ids
+ [{:keys [id->band]} band]
+ (set (keep (fn [[id b]] (when (= b band) id)) id->band)))
+
+(deftest dataset-sanity-test
+ (let [ds (make-dataset {:seed 42 :restricted-collection-id -1})]
+ (testing "300 docs, distinct strictly-positive distances, unit vectors"
+ (is (= 300 (count (:docs ds))))
+ (is (= 300 (count (distinct (vals (:id->distance ds))))))
+ (is (every? pos? (vals (:id->distance ds))))
+ (is (every? #(< (abs (- 1.0 (reduce + (map * % %)))) 1e-9)
+ (vals (dissoc (:embeddings ds) probe-text)))))
+ (testing "geometry the assertions rely on"
+ (let [top-20 (set (take 20 (exact-ids ds (constantly true))))]
+ (testing "exact top-20 = the whole restricted band + nearest core docs, nothing verified"
+ (is (every? (some-fn (band-ids ds :restricted) (band-ids ds :core)) top-20))
+ (is (every? top-20 (band-ids ds :restricted))))
+ (testing "rare verified docs are all inside the cutoff, decoys all outside"
+ (is (= (band-ids ds :rare) (set (exact-ids ds :verified))))
+ (is (empty? (filter (band-ids ds :decoy) (exact-ids ds (constantly true))))))
+ (testing "at least a full pool of accessible docs lies within the cutoff"
+ (is (<= 20 (count (remove (band-ids ds :restricted) (exact-ids ds (constantly true)))))))
+ (testing "creator parity makes a created-by filter half-selective: 10 odd-id docs in the top-20, and
+ the nearest 20 odd-id docs are 4 restricted + 16 core"
+ (is (= 10 (count (filter odd? top-20))))
+ (let [odd-top-20 (take 20 (exact-ids ds (comp odd? :id)))]
+ (is (= 20 (count odd-top-20)))
+ (is (= 4 (count (filter (band-ids ds :restricted) odd-top-20))))
+ (is (= 16 (count (filter (band-ids ds :core) odd-top-20))))))))))
+
+;;;; fixture
+
+(def ^:private results-limit
+ "The candidate-pool size every variant retrieves, well below the 300-doc dataset so nothing saturates."
+ 20)
+
+(def ^:private iterative-ef-search
+ "Below [[results-limit]], so the iterative strategies must run more than one scan iteration."
+ 16)
+
+(defn- assert-pgvector-prereqs!
+ "Fail fast unless pgvector supports iterative scans and the server ef_search default is what the naive
+ :hnsw cells assume (there is deliberately no per-context ef knob for the naive strategy)."
+ []
+ (jdbc/with-transaction [tx (semantic.env/get-pgvector-datasource!)]
+ ;; a vector op loads the extension library, which registers the hnsw.* GUCs for SHOW
+ (jdbc/execute! tx ["SELECT '[1,2,3]'::vector <=> '[1,2,3]'::vector"])
+ (let [version (-> (jdbc/execute-one! tx ["SELECT extversion FROM pg_extension WHERE extname = 'vector'"])
+ vals first str)
+ ef-search (-> (jdbc/execute-one! tx ["SHOW hnsw.ef_search"]) vals first)]
+ (when-not (<= 0 (compare (mapv parse-long (take 2 (str/split version #"\.")))
+ [0 8]))
+ (throw (ex-info "vector-strategy matrix needs pgvector >= 0.8.0 (iterative scans)"
+ {:extversion version})))
+ (when-not (= "40" ef-search)
+ (throw (ex-info "vector-strategy matrix assumes the stock hnsw.ef_search default"
+ {:hnsw.ef_search ef-search}))))))
+
+(def ^:private ^:dynamic *index*
+ "The semantic index the helpers below operate on; [[graph-approximation-test]] rebinds it."
+ semantic.tu/mock-index)
+
+(defn- drop-hnsw-index!
+ []
+ (jdbc/execute! (semantic.env/get-pgvector-datasource!)
+ [(str "DROP INDEX IF EXISTS " (semantic.index/hnsw-index-name *index*))]))
+
+(defn- build-hnsw-index!
+ "Build the HNSW index on [[*index*]]'s table."
+ []
+ ;; the graph is NOT reproducible: pgvector draws node levels from the per-backend global PRNG, which
+ ;; `setseed` does not touch, so every assertion against the index must hold for any random graph
+ (jdbc/with-transaction [tx (semantic.env/get-pgvector-datasource!)]
+ ;; serial build keeps insertion order fixed and removes worker-count variation
+ (jdbc/execute! tx ["SET LOCAL max_parallel_maintenance_workers = 0"])
+ ;; m/ef_construction are the current pgvector defaults, pinned against future default changes
+ (jdbc/execute! tx [(format (str "CREATE INDEX %s ON %s USING hnsw (embedding vector_cosine_ops) "
+ "WITH (m = 16, ef_construction = 64)")
+ (semantic.index/hnsw-index-name *index*)
+ (:table-name *index*))])))
+
+(defn- do-with-matrix-fixture!
+ "Run `(f dataset)` against a temp pgvector DB whose index table is populated with the matrix dataset but has
+ NO HNSW index yet. `f` runs its exact cells first, then calls [[build-hnsw-index!]] for the indexed ones."
+ [f]
+ (mt/with-premium-features #{:semantic-search}
+ (mt/with-temporary-setting-values [semantic-search-results-limit results-limit]
+ (mt/with-temp [:model/Collection {restricted-id :id} {:name "Matrix Restricted Collection"}]
+ (perms/revoke-collection-permissions! (perms-group/all-users) restricted-id)
+ (let [ds (make-dataset {:seed 42 :restricted-collection-id restricted-id})]
+ (semantic.tu/with-mock-embeddings (:embeddings ds)
+ (semantic.tu/with-test-db! {:mode :mock-initialized}
+ (assert-pgvector-prereqs!)
+ (drop-hnsw-index!)
+ (semantic.tu/upsert-index! (:docs ds) :serial? true)
+ ;; pin the stats the planner sees, so plan choice cannot vary with autoanalyze timing
+ (jdbc/execute! (semantic.env/get-pgvector-datasource!)
+ [(str "ANALYZE " (:table-name *index*))])
+ (f ds))))))))
+
+(def ^:private base-ctx
+ {:search-string probe-text
+ :search-engine "semantic"
+ :archived? false})
+
+(defn- variant-ctx
+ "Per-variant search-context overrides."
+ [variant]
+ (case variant
+ ;; the :hnsw query shape run before build-hnsw-index!, to measure exact pre-index retrieval. Opt out of
+ ;; the missing-index fail-fast, which production traffic gets but this baseline deliberately bypasses.
+ :no-index {:vector-search-strategy :hnsw
+ :vector-search-allow-missing-index? true}
+ :brute-force {:vector-search-strategy :brute-force}
+ ;; force-index because the planner would otherwise always seq-scan a 300-row table, making every
+ ;; variant silently exact
+ :hnsw {:vector-search-strategy :hnsw
+ :vector-search-force-index? true}
+ (:hnsw-iterative-relaxed :hnsw-iterative-strict)
+ ;; both GUC knobs pinned so nothing depends on settings; max_scan_tuples exceeds the table so it never
+ ;; binds except where a cell overrides it
+ {:vector-search-strategy variant
+ :vector-search-force-index? true
+ :vector-search-ef-search iterative-ef-search
+ :vector-search-max-scan-tuples 10000}))
+
+(defn- query-results!
+ "Run the full [[semantic.index/query-index]] pipeline for `variant`.
+ Returns query-index's {:results ... :raw-count ...}."
+ [variant extra-ctx]
+ (memoize/memo-clear! #'semantic.scoring/view-count-percentiles)
+ (semantic.index/query-index (semantic.env/get-pgvector-datasource!)
+ *index*
+ (merge base-ctx
+ {:current-user-id (mt/user->id :rasta)}
+ (variant-ctx variant)
+ extra-ctx)))
+
+(defn- query-ids!
+ "Ordered result ids for `variant`, through the full pipeline."
+ [variant & {:as extra-ctx}]
+ (mapv :id (:results (query-results! variant extra-ctx))))
+
+;;;; if one of these tests flakes
+;;
+;; The HNSW graph is rebuilt randomly on every run (see [[build-hnsw-index!]] -- it cannot be seeded), so a
+;; failure here is one of three things:
+;;
+;; 1. An EXACT matrix cell failing intermittently. These cells assert outcomes that are invariant across
+;; random graphs (each was validated against 30-60 independently built graphs on pgvector 0.8.1), so an
+;; intermittent failure means the invariance itself broke -- almost certainly a pgvector behavior change
+;; (graph build defaults, scan-phase sizing, max_scan_tuples semantics). Do not loosen the assertion
+;; blindly: re-validate with the loop below, work out which scan behavior changed, and either restore the
+;; invariant by pinning the changed parameter or convert the cell to a band derived from fresh
+;; measurements.
+;; 2. A BANDED cell in [[graph-approximation-test]] failing. The edges sit several misses beyond the range
+;; observed across dozens of graphs (the observed range is noted per cell), but a rare excursion past an
+;; edge is possible. Re-derive the edges rather than nudging by one miss: run the loop below, take the
+;; min/max per cell, and set edges ~0.1-0.15 of recall beyond them. Keep upper edges strictly below 1.0;
+;; a band touching 1.0 no longer demonstrates approximation, and the fix is a harder dataset (higher
+;; dims, denser packing), not a wider band.
+;; 3. A deterministic failure after an environment change (new pgvector version, changed server defaults).
+;; [[assert-pgvector-prereqs!]] catches the known dependencies on server state; extend it when a new one
+;; is discovered.
+;;
+;; Each test run builds a fresh graph, so re-measuring a cell's distribution is just repetition:
+;;
+;; (frequencies (map (fn [_] (dissoc (clojure.test/run-test <the-deftest>) :type)) (range 30)))
+;;
+;; For per-cell values, evaluate the cell's recall expression inside the fixture across runs (bind an atom,
+;; collect, take min/max). Beware deriving bands from raw SQL probes instead: in-pipeline recall has fatter
+;; tails (an ef8-vs-ef64 margin assertion derived from SQL probes flaked within 10 in-pipeline runs).
+
+;;;; layer 1: distance-level retrieval vs exact ground truth
+
+(deftest distance-recall-matrix-test
+ (do-with-matrix-fixture!
+ (fn [ds]
+ ;; isolate retrieval: weights reduce ranking to the distance rank, permission filtering is bypassed
+ (semantic.tu/with-only-semantic-weights
+ (mt/with-dynamic-fn-redefs [semantic.index/filter-read-permitted identity]
+ (let [exact-all (exact-ids ds (constantly true))
+ exact-rare (exact-ids ds :verified)]
+ (testing "without an HNSW index"
+ (testing "retrieval is exact: the pool is the true top-20 in distance order"
+ (is (= (take 20 exact-all) (query-ids! :no-index)))
+ (is (= (take 20 exact-all) (query-ids! :brute-force))))
+ (testing "but post-filtering still finds NOTHING for an anti-correlated :verified filter"
+ ;; the exact global top-20 contains nothing verified, so filtering it afterwards yields zero;
+ ;; only filter-first (:brute-force) retrieval is immune
+ (is (= [] (query-ids! :no-index :verified true)))
+ (is (= exact-rare (query-ids! :brute-force :verified true)))))
+ (build-hnsw-index!)
+ (testing "with the HNSW index (forced)"
+ (testing "unfiltered retrieval is exact: graph approximation error does not manifest at this scale"
+ ;; a 300-doc 4-d graph returns the true top-20 even at ef_search 2, so these cells pin
+ ;; exactness rather than a recall band; the approximation-error axis is a scale phenomenon
+ ;; and is measured by the at-scale benchmarking harness, not this matrix
+ (doseq [variant [:hnsw :hnsw-iterative-relaxed :hnsw-iterative-strict]]
+ (let [ids (query-ids! variant)]
+ (testing variant
+ (is (= (take 20 exact-all) ids))
+ (is (= 1.0 (ndcg-at 20 (:id->distance ds) exact-all ids)))))))
+ (testing "naive :hnsw post-filters the same global pool, so the adversarial filter finds nothing"
+ ;; structural, not graph luck: the scan's top-20 would need fewer than 20 of the 262 docs
+ ;; nearer than the rare band for anything verified to enter the pool
+ (is (= [] (query-ids! :hnsw :verified true))))
+ (testing "iterative scans pull neighbours past the filter and fully recover, in exact order"
+ ;; the inner limit (20) exceeds the 12 matches, so the scan runs until the graph is exhausted
+ (doseq [variant [:hnsw-iterative-relaxed :hnsw-iterative-strict]]
+ (testing variant
+ (is (= exact-rare (query-ids! variant :verified true))))))
+ (testing "hnsw.max_scan_tuples truncates an iterative scan before it reaches far-away matches"
+ ;; the rare band sits at distance ranks 269-280, far past a 64-tuple visit budget; the cap
+ ;; has scan-phase granularity, but no phase overshoots by the ~200 tuples that would need
+ (is (= [] (query-ids! :hnsw-iterative-relaxed
+ :verified true
+ :vector-search-max-scan-tuples 64)))))))))))
+
+;;;; layer 2: final top results vs an exhaustive run of the same scoring pipeline
+
+(deftest full-pipeline-recall-matrix-test
+ (do-with-matrix-fixture!
+ (fn [ds]
+ (mt/with-dynamic-fn-redefs [semantic.index/filter-read-permitted identity]
+ (testing "re-ranking: a deterministic :pinned boost promotes docs the candidate pool never contains"
+ (semantic.tu/with-weights {:rrf 1 :pinned 10}
+ (let [ideal (mt/with-temporary-setting-values [semantic-search-results-limit 1000]
+ (query-ids! :brute-force))
+ ideal-top (take 10 ideal)
+ exact-all (exact-ids ds (constantly true))]
+ (testing "the ideal top-10 is the pinned band plus the nearest docs"
+ (is (= (into (band-ids ds :pinned) (take 4 exact-all))
+ (set ideal-top))))
+ (testing "exact retrieval scores only its truncated pool: pinned docs sit at distance ranks 263-268"
+ (is (= 0.4 (recall-at 10 ideal-top (query-ids! :no-index)))))
+ (build-hnsw-index!)
+ (testing "indexed retrieval is stuck at the same recall: iterating does not fix limit truncation"
+ ;; the filters are non-selective here, so every variant's pool is the same exact top-20
+ (doseq [variant [:hnsw :hnsw-iterative-relaxed :hnsw-iterative-strict]]
+ (testing variant
+ (is (= 0.4 (recall-at 10 ideal-top (query-ids! variant))))))))))
+ (testing "under a selective filter the indexed iterative variants are strictly better end-to-end"
+ (semantic.tu/with-only-semantic-weights
+ (let [ideal-rare (mt/with-temporary-setting-values [semantic-search-results-limit 1000]
+ (query-ids! :brute-force :verified true))]
+ (is (= 0.0 (recall-at 10 ideal-rare (query-ids! :no-index :verified true))))
+ (is (= 0.0 (recall-at 10 ideal-rare (query-ids! :hnsw :verified true))))
+ (doseq [variant [:hnsw-iterative-relaxed :hnsw-iterative-strict]]
+ (testing variant
+ (is (= 1.0 (recall-at 10 ideal-rare (query-ids! variant :verified true)))))))))))))
+
+;;;; layer 3: under-fetch from post-retrieval permission filtering
+
+(deftest permission-under-fetch-matrix-test
+ (do-with-matrix-fixture!
+ (fn [_ds]
+ (semantic.tu/with-only-semantic-weights
+ (let [run! (fn [variant user & {:as extra-ctx}]
+ (mt/with-test-user user
+ (let [{:keys [results raw-count]} (query-results! variant
+ (merge {:current-user-id (mt/user->id user)}
+ extra-ctx))]
+ {:raw-count raw-count :returned (count results)})))]
+ (testing "exact variants: the restricted band fills 8 of the 20 pool slots, then gets dropped"
+ (doseq [variant [:no-index :brute-force]]
+ (testing variant
+ (is (= {:raw-count 20 :returned 12} (run! variant :rasta))))))
+ (build-hnsw-index!)
+ (testing "indexed variants under-fetch identically; iterating cannot help (the check runs in Clojure)"
+ (doseq [variant [:hnsw :hnsw-iterative-relaxed :hnsw-iterative-strict]]
+ (testing variant
+ (is (= {:raw-count 20 :returned 12} (run! variant :rasta))))))
+ (testing "an admin sees the full pool, pinning the loss on permissions rather than retrieval"
+ (is (= {:raw-count 20 :returned 20} (run! :hnsw :crowberto))))
+ ;; The half-selective created-by filter (odd ids -> crowberto) keeps 10 of the exact top-20
+ ;; (4 restricted + 6 core). Naive :hnsw post-filters that fixed pool, so it arrives at the
+ ;; permission check already halved; filter-first and iterative scans refill to a full pool of 20
+ ;; (4 restricted + 16 core). Structural, not graph luck: the forced top-20 is exact at this scale.
+ ;; (At scale, graph approximation can shift this either way -- missing nearby restricted docs
+ ;; lessens the permission drop, missing nearby permitted docs worsens it -- so only the SQL-stage
+ ;; effect is pinned here.)
+ (testing "a half-selective filter compounds with permissions for the post-filtering strategy only"
+ (let [created-by [(mt/user->id :crowberto)]]
+ (testing "naive :hnsw under-fetches at the SQL stage, then again at the permission stage"
+ (is (= {:raw-count 10 :returned 6} (run! :hnsw :rasta :created-by created-by))))
+ (testing "filter-first and iterative variants under-fetch only at the permission stage"
+ (doseq [variant [:brute-force :hnsw-iterative-relaxed :hnsw-iterative-strict]]
+ (testing variant
+ (is (= {:raw-count 20 :returned 16} (run! variant :rasta :created-by created-by)))))))))))))
+
+;;;; graph approximation: a 32-d densely packed dataset where HNSW genuinely misses
+
+;; a separate 32-d index because graph approximation error needs dimensionality, not tree size: a 4-d graph
+;; stays exact even with thousands of packed fillers
+(def ^:private packed-index
+ "The index for [[graph-approximation-test]]'s 32-d packed dataset."
+ (semantic.index/default-index {:provider "mock" :model-name "packed" :vector-dimensions 32}
+ :table-name "index_table_packed_test"))
+
+(defn- make-packed-dataset
+ "`n` filler cards packed at near-identical distances from the probe, so the true top-20 differ from any
+ approximate top-20 by margins the graph cannot resolve.
+ Doc ids are assigned in ascending-distance order, so the exact top-k is simply ids 1..k."
+ [{:keys [seed n]}]
+ (let [rng (Random. (long seed))
+ rows (mapv (fn [i]
+ {:id (inc i)
+ :name (format "packed card %04d" (inc i))
+ :vector (vector-at-distance rng 32 (+ 0.10 (* 2e-5 i)))})
+ (range n))]
+ {:docs (vec (for [{:keys [id name]} rows]
+ {:model "card"
+ :id id
+ :name name
+ :searchable_text name
+ :embeddable_text name
+ :created_at #t "2025-01-01T12:00:00Z"
+ :creator_id (mt/user->id :rasta)
+ :archived false
+ :legacy_input {:id id :model "card" :name name}
+ :metadata {:title name}}))
+ :embeddings (into {probe-text (into [1.0] (repeat 31 0.0))} (map (juxt :name :vector)) rows)
+ :exact-ids (mapv :id rows)}))
+
+(deftest graph-approximation-test
+ (mt/with-premium-features #{:semantic-search}
+ (mt/with-temporary-setting-values [semantic-search-results-limit results-limit]
+ (let [ds (make-packed-dataset {:seed 42 :n 2000})]
+ (semantic.tu/with-mock-embeddings (:embeddings ds)
+ (semantic.tu/with-test-db! {}
+ (binding [*index* packed-index]
+ (semantic.index/create-index-table-if-not-exists! (semantic.env/get-pgvector-datasource!)
+ packed-index)
+ (drop-hnsw-index!)
+ (semantic.tu/upsert-index! (:docs ds) :index packed-index :serial? true)
+ (jdbc/execute! (semantic.env/get-pgvector-datasource!)
+ [(str "ANALYZE " (:table-name packed-index))])
+ (semantic.tu/with-only-semantic-weights
+ (mt/with-dynamic-fn-redefs [semantic.index/filter-read-permitted identity]
+ (let [exact-top (take 20 (:exact-ids ds))
+ recall20 (fn [variant & {:as extra-ctx}]
+ (recall-at 20 exact-top (query-ids! variant extra-ctx)))]
+ (testing "without an index, retrieval stays exact even at 32 dims"
+ (is (= exact-top (query-ids! :no-index))))
+ (build-hnsw-index!)
+ ;; the graph differs every build, so these cells assert bands; each edge sits several
+ ;; misses beyond everything observed across dozens of independently built graphs
+ ;; (the observed range is noted per cell) -- a value at 1.0 or near 0 means the cell broke
+ (testing "naive :hnsw genuinely misses: recall@20 lands in a band strictly below 1"
+ ;; observed 0.5-0.85
+ (is (<= 0.3 (recall20 :hnsw) 0.95)))
+ (testing "iterative scans with ef_search 16 (below the limit of 20) miss more"
+ ;; observed 0.35-0.6
+ (is (<= 0.15 (recall20 :hnsw-iterative-relaxed) 0.85))
+ ;; observed 0.15-0.55, below relaxed on every graph measured: strict_order discards
+ ;; more per visit in exchange for emission order, which the outer sort makes worthless
+ ;; in this pipeline (the gap can shrink to one miss, so no cross-assertion)
+ (is (<= 0.05 (recall20 :hnsw-iterative-strict) 0.8)))
+ (testing "a saturating ef_search dials approximation away again, on the same graph"
+ (is (= exact-top (query-ids! :hnsw-iterative-relaxed
+ :vector-search-ef-search 200))))))))))))))
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index aa2784f4ead..0ceb2b8dd11 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -400,6 +400,19 @@
(prometheus/counter :metabase-search/semantic-db-query-ms
{:description "Total number of ms spent querying the search index"
:labels [:embedding-model]})
+ ;; the four semantic-vector-* diagnostics are emitted only while the semantic-search-explain setting is on
+ (prometheus/counter :metabase-search/semantic-vector-inner-ms
+ {:description "Total ms spent in the inner vector subquery (diagnostic, off by default)"
+ :labels [:strategy]})
+ (prometheus/counter :metabase-search/semantic-vector-tuples-scanned
+ {:description "Total tuples the vector scan visited, returned plus filter-removed (diagnostic, off by default)"
+ :labels [:strategy]})
+ (prometheus/counter :metabase-search/semantic-prefilter-pool-size
+ {:description "Total rows a filter-first (brute-force) scan would have computed distances over (diagnostic, off by default)"
+ :labels [:strategy]})
+ (prometheus/counter :metabase-search/semantic-vector-scan-used-index
+ {:description "Count of vector searches by strategy and the plan node chosen for the index-table scan (diagnostic, off by default)"
+ :labels [:strategy :plan-node]})
(prometheus/counter :metabase-search/semantic-appdb-scores-ms
{:description "Total number of ms spent adding appdb-based scores"})
(prometheus/counter :metabase-search/semantic-fallback-triggered
diff --git a/src/metabase/search/api.clj b/src/metabase/search/api.clj
index 7ad0d6fa6d5..cae6faa6021 100644
--- a/src/metabase/search/api.clj
+++ b/src/metabase/search/api.clj
@@ -179,6 +179,11 @@
[:model_ancestors {:default false} [:maybe :boolean]]
[:search_engine {:optional true} [:maybe string?]]
[:vector_search_strategy {:optional true} [:maybe (into [:enum] (map name) search.config/vector-search-strategies)]]
+ ;; bounded to pgvector's GUC ranges so out-of-range values get a 400 instead of erroring the
+ ;; SET LOCAL mid-transaction (a 500)
+ [:vector_search_ef_search {:optional true} [:maybe [:int {:min 1 :max 1000}]]]
+ [:vector_search_max_scan_tuples {:optional true} [:maybe [:int {:min 1 :max 2147483647}]]]
+ [:vector_search_explain {:optional true} [:maybe :boolean]]
[:search_native_query {:optional true} [:maybe :boolean]]
[:verified {:optional true} [:maybe true?]]
[:ids {:optional true} [:maybe (ms/QueryVectorOf ms/PositiveInt)]]
@@ -209,6 +214,9 @@
model-ancestors :model_ancestors
search-engine :search_engine
vector-search-strategy :vector_search_strategy
+ vector-search-ef-search :vector_search_ef_search
+ vector-search-max-scan-tuples :vector_search_max_scan_tuples
+ vector-search-explain :vector_search_explain
search-native-query :search_native_query
table-db-id :table_db_id
include-metadata :include_metadata
@@ -234,6 +242,9 @@
:offset (request/offset)
:search-engine search-engine
:vector-search-strategy vector-search-strategy
+ :vector-search-ef-search vector-search-ef-search
+ :vector-search-max-scan-tuples vector-search-max-scan-tuples
+ :vector-search-explain? vector-search-explain
:search-native-query search-native-query
:search-string (some-> q str/trim not-empty)
:table-db-id table-db-id
@@ -265,7 +276,12 @@
- `last_edited_at`: search for items last edited at a specific timestamp
- `last_edited_by`: search for items last edited by a specific user
- `search_native_query`: set to true to search the content of native queries
- - `vector_search_strategy`: for the semantic engine, `hnsw` (approximate index search, default) or `brute-force` (exact filter-first search); ignored by other engines
+ - `vector_search_strategy`: for the semantic engine: `hnsw` (approximate index search, default),
+ `brute-force` (exact filter-first search), or `hnsw-iterative-relaxed` / `hnsw-iterative-strict`
+ (index-backed iterative scans with inline filters); ignored by other engines
+ - `vector_search_ef_search`: override pgvector's `hnsw.ef_search` (1-1000) for the iterative strategies; (admin only)
+ - `vector_search_max_scan_tuples`: override pgvector's `hnsw.max_scan_tuples` for the iterative strategies; (admin only)
+ - `vector_search_explain`: set to true to record vector-scan instrumentation for this search (expensive); (admin only)
- `verified`: set to true to search for verified items only (requires Content Management or Official Collections premium feature)
- `ids`: search for items with those ids, works iff single value passed to `models`
- `display_type`: search for cards/models with specific display types
@@ -282,6 +298,12 @@
[_route-params
query-params :- search-request-schema]
(api/check-valid-page-params (request/limit) (request/offset))
+ ;; tuning/diagnostic knobs are admin-only: explain re-executes the vector scan and counts the whole index
+ ;; table per request, and the scan-budget knobs let a request inflate its own cost
+ (when (or (:vector_search_ef_search query-params)
+ (:vector_search_max_scan_tuples query-params)
+ (some? (:vector_search_explain query-params)))
+ (api/check-superuser))
(try
(u/prog1 (search/search (params->search-context query-params))
(analytics/inc! :metabase-search/response-ok)
diff --git a/src/metabase/search/config.clj b/src/metabase/search/config.clj
index 8af1d7d5766..f0c5f1f90b9 100644
--- a/src/metabase/search/config.clj
+++ b/src/metabase/search/config.clj
@@ -317,8 +317,24 @@
"Valid semantic-search vector-search strategies, as keywords. Mastered here (rather than in the EE module)
so the OSS search API param and the EE semantic-search setting validation share one definition.
Note: the per-strategy dispatch in [[metabase-enterprise.semantic-search.index/semantic-search-query]] is a
- separate `case` (with an `:hnsw` default) and must be updated by hand when adding a strategy."
- [:hnsw :brute-force])
+ separate dispatch (unknown strategies fall back to `:brute-force`) and must be updated by hand when
+ adding a strategy.
+
+ - `:hnsw` approximate, index-backed, post-filters the candidate set
+ - `:brute-force` exact, filter-first full scan (skips the index)
+ - `:hnsw-iterative-relaxed` index-backed iterative scan, filters inline, results in approximate distance
+ order (pgvector `hnsw.iterative_scan = relaxed_order`)
+ - `:hnsw-iterative-strict` as above but exact distance order (`hnsw.iterative_scan = strict_order`)
+
+ The iterative scan keeps pulling neighbours until the limit is met or `hnsw.max_scan_tuples` is hit; the
+ `ef-search` and `max-scan-tuples` knobs tune it further."
+ [:hnsw :brute-force :hnsw-iterative-relaxed :hnsw-iterative-strict])
+
+(def hnsw-index-backed-strategies
+ "The subset of [[vector-search-strategies]] that read the HNSW index. Selecting one of these must build the
+ index just-in-time, and a query under one must fail fast when the index is missing. `:brute-force` is the
+ only strategy that needs no index."
+ #{:hnsw :hnsw-iterative-relaxed :hnsw-iterative-strict})
(def ^:private ui-contexts
"Search `context` values issued by the frontend, one per UI surface.
@@ -411,9 +427,20 @@
[:models [:set SearchableModel]]
;; TODO this is optional only for tests, clean those up!
[:search-engine {:optional true} keyword?]
- ;; Semantic-engine vector-search strategy (:hnsw or :brute-force). When absent, the engine uses its
- ;; configured default setting.
- [:vector-search-strategy {:optional true} [:maybe keyword?]]
+ ;; Semantic-engine vector-search strategy (see [[search.config/vector-search-strategies]]). When absent,
+ ;; the engine uses its configured default setting. The remaining vector-search-* knobs are experimental
+ ;; tuning parameters; they only affect the `:hnsw-iterative-*` strategies (except `explain?`, which works
+ ;; for any strategy). Each falls back to its EE setting default when absent.
+ [:vector-search-strategy {:optional true} [:maybe keyword?]]
+ ;; pgvector `hnsw.ef_search` -- HNSW candidate-list size
+ [:vector-search-ef-search {:optional true} [:maybe pos-int?]]
+ ;; pgvector `hnsw.max_scan_tuples` -- soft cap on tuples an iterative scan visits
+ [:vector-search-max-scan-tuples {:optional true} [:maybe pos-int?]]
+ ;; true to run gated EXPLAIN (ANALYZE) instrumentation of the inner vector subquery (expensive)
+ [:vector-search-explain? {:optional true} [:maybe :boolean]]
+ ;; true to `SET LOCAL enable_seqscan = off` to force the planner onto the HNSW index (experiment knob;
+ ;; deliberately not exposed over HTTP)
+ [:vector-search-force-index? {:optional true} [:maybe :boolean]]
[:search-string {:optional true} [:maybe ms/NonBlankString]]
[:weights {:optional true} [:maybe [:map-of :keyword number?]]]
;;
diff --git a/src/metabase/search/impl.clj b/src/metabase/search/impl.clj
index 812be93eb3d..35b2bc21516 100644
--- a/src/metabase/search/impl.clj
+++ b/src/metabase/search/impl.clj
@@ -273,6 +273,10 @@
[:table-db-id {:optional true} [:maybe ms/PositiveInt]]
[:search-engine {:optional true} [:maybe string?]]
[:vector-search-strategy {:optional true} [:maybe string?]]
+ [:vector-search-ef-search {:optional true} [:maybe ms/PositiveInt]]
+ [:vector-search-max-scan-tuples {:optional true} [:maybe ms/PositiveInt]]
+ [:vector-search-explain? {:optional true} [:maybe boolean?]]
+ [:vector-search-force-index? {:optional true} [:maybe boolean?]]
[:search-native-query {:optional true} [:maybe boolean?]]
[:model-ancestors? {:optional true} [:maybe boolean?]]
[:verified {:optional true} [:maybe true?]]
@@ -313,6 +317,10 @@
offset
search-engine
vector-search-strategy
+ vector-search-ef-search
+ vector-search-max-scan-tuples
+ vector-search-explain?
+ vector-search-force-index?
search-native-query
search-string
table-db-id
@@ -358,6 +366,10 @@
(some? limit) (assoc :limit-int limit)
(some? offset) (assoc :offset-int offset)
(not (str/blank? vector-search-strategy)) (assoc :vector-search-strategy (keyword vector-search-strategy))
+ (some? vector-search-ef-search) (assoc :vector-search-ef-search vector-search-ef-search)
+ (some? vector-search-max-scan-tuples) (assoc :vector-search-max-scan-tuples vector-search-max-scan-tuples)
+ (some? vector-search-explain?) (assoc :vector-search-explain? vector-search-explain?)
+ (some? vector-search-force-index?) (assoc :vector-search-force-index? vector-search-force-index?)
(some? search-native-query) (assoc :search-native-query search-native-query)
(some? verified) (assoc :verified verified)
(some? curated) (assoc :curated? curated)
diff --git a/test/metabase/search/api_test.clj b/test/metabase/search/api_test.clj
index 93d31ace555..42b18aa4554 100644
--- a/test/metabase/search/api_test.clj
+++ b/test/metabase/search/api_test.clj
@@ -334,6 +334,38 @@
(is (=? {:engine string?}
(mt/user-http-request :crowberto :get 200 "search" :q "x" :context "search-app"))))))
+(deftest vector-search-knobs-test
+ (testing "the vector-search tuning/diagnostic knobs are admin-only"
+ (doseq [[param value] {:vector_search_ef_search 100
+ :vector_search_max_scan_tuples 50000
+ :vector_search_explain true}]
+ (testing param
+ (is (= "You don't have permissions to do that."
+ (mt/user-http-request :rasta :get 403 "search" :q "x" param value))))))
+ (testing "an admin's knobs thread through into the search context"
+ (let [captured (atom nil)]
+ (mt/with-dynamic-fn-redefs [search/search (fn [search-ctx]
+ (reset! captured search-ctx)
+ {:data [] :total 0 :engine "test"})]
+ (mt/user-http-request :crowberto :get 200 "search" :q "x"
+ :vector_search_strategy "hnsw-iterative-strict"
+ :vector_search_ef_search 100
+ :vector_search_max_scan_tuples 50000
+ :vector_search_explain true)
+ (is (=? {:vector-search-strategy :hnsw-iterative-strict
+ :vector-search-ef-search 100
+ :vector-search-max-scan-tuples 50000
+ :vector-search-explain? true}
+ @captured))
+ (testing "absent knobs stay absent, so the semantic engine falls back to its setting defaults"
+ (mt/user-http-request :crowberto :get 200 "search" :q "x")
+ (is (not-any? @captured [:vector-search-strategy :vector-search-ef-search
+ :vector-search-max-scan-tuples :vector-search-explain?
+ :vector-search-force-index?]))))))
+ (testing "values outside pgvector's GUC ranges are rejected up front"
+ (is (=? {:errors {:vector_search_ef_search some?}}
+ (mt/user-http-request :crowberto :get 400 "search" :q "x" :vector_search_ef_search 2000)))))
+
(deftest basic-test
(testing "Basic search, should find 1 of each entity type, all items in the root collection"
(with-search-items-in-root-collection "test"
diff --git a/test/metabase/search/impl_test.clj b/test/metabase/search/impl_test.clj
index 7f0929959d9..e19b758c8af 100644
--- a/test/metabase/search/impl_test.clj
+++ b/test/metabase/search/impl_test.clj
@@ -36,6 +36,21 @@
(testing "Subclasses"
(is (= :search.engine/hybrid (#'search.impl/parse-engine "hybrid"))))))
+(deftest vector-search-knobs-absent-test
+ ;; threading and gating of the knobs is covered end-to-end in [[metabase.search.api-test]]; this pins the
+ ;; contract that search-context must never DEFAULT them -- absence is what lets the semantic engine fall
+ ;; back to its setting-backed defaults, and the schema's default-value transformer makes that easy to
+ ;; break by adding an innocent-looking :default
+ (let [ctx (search.impl/search-context {:current-user-id 1
+ :current-user-perms #{"/"}
+ :is-superuser? true
+ :is-impersonated-user? false
+ :is-sandboxed-user? false
+ :models nil
+ :search-string "x"})]
+ (is (not-any? ctx [:vector-search-strategy :vector-search-ef-search :vector-search-max-scan-tuples
+ :vector-search-explain? :vector-search-force-index?]))))
+
(deftest ^:parallel order-clause-test
(testing "it includes all columns and normalizes the query"
(is (= [[:case