The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Support HNSW iterative-scan + cross-test each strategy (#75722)
Chris Truter746a65d593e4
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.cljindex b7718e4a250..8b6388d8537 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj@@ -470,9 +470,10 @@(defn create-hnsw-index-if-not-exists!"Create the HNSW index on the `embedding` column of `index`'s table, if it does not already exist.- HNSW indexes are expensive to build and maintain, so we only create one when an instance is configured to- use the `:hnsw` vector-search strategy (see [[create-index-table-if-not-exists!]] and the async build- triggered by [[metabase-enterprise.semantic-search.settings/semantic-search-vector-strategy]]).+ HNSW indexes are expensive to build and maintain, so we only create one when an instance is configured for+ an HNSW-index-backed vector-search strategy (`:hnsw` or `:hnsw-iterative-*`; see+ [[metabase.search.config/hnsw-index-backed-strategies]], [[create-index-table-if-not-exists!]], and the+ async build triggered by [[metabase-enterprise.semantic-search.settings/semantic-search-vector-strategy]]).Pass `concurrently? true` to build with `CREATE INDEX CONCURRENTLY` (for a populated table whose writesshouldn't be locked out); it must run outside a transaction."@@ -492,8 +493,9 @@"Ensure that the index table exists and is ready to be populated. Ifforce-reset? is true, drops and recreates the table if it exists.- The HNSW index is only created when the instance is configured for the `:hnsw` vector-search strategy;- otherwise it is built just-in-time when the strategy is configured (see [[create-hnsw-index-if-not-exists!]])."+ The HNSW index is only created when the instance is configured for an HNSW-index-backed vector-search+ strategy (`:hnsw` or `:hnsw-iterative-*`); under `:brute-force` it is skipped, and built just-in-time when+ an index-backed strategy is later configured (see [[create-hnsw-index-if-not-exists!]])."[connectable index & {:keys [force-reset?] :or {force-reset? false}}](try(let [{:keys [embedding-model table-name]} index@@ -508,7 +510,8 @@(-> (sql.helpers/create-table (keyword table-name) :if-not-exists)(sql.helpers/with-columns (index-table-schema vector-dimensions))sql-format-quoted))- (when (= :hnsw (semantic-settings/semantic-search-vector-strategy))+ (when (contains? search.config/hnsw-index-backed-strategies+ (semantic-settings/semantic-search-vector-strategy))(create-hnsw-index-if-not-exists! connectable index))(jdbc/execute!connectable@@ -711,24 +714,108 @@:order-by [[:semantic_rank :asc]]:limit (semantic-settings/semantic-search-results-limit)}))+(defn- hnsw-iterative-search-query+ "Build the semantic vector subquery as an index-backed iterative scan with `filters` applied inline."+ [index embedding-literal filters]+ ;; The filters live inside the ordered/limited candidate scan (unlike `hnsw-search-query`, which+ ;; post-filters), so the planner can pick the HNSW index and pgvector's iterative scan keeps pulling+ ;; neighbours until the limit is met or `hnsw.max_scan_tuples` is hit.+ ;; The recall/latency trade-off is governed by the iterative-scan GUCs (see `vector-session-settings`),+ ;; not the SQL shape.+ ;; The cutoff stays in the outer query (like `hnsw-search-query`) so the inner scan fills up to the limit+ ;; before trimming.+ (let [inner (cond-> {:select (into common-search-columns+ [[[:raw (str "embedding <=> " embedding-literal)] :distance]])+ :from [(keyword (:table-name index))]+ :order-by [[[:raw (str "embedding <=> " embedding-literal)] :asc]]+ :limit (semantic-settings/semantic-search-results-limit)}+ filters (assoc :where filters))]+ {:with [[:vector_candidates inner]]+ :select (into common-search-columns+ [[[:raw "row_number() OVER (ORDER BY distance ASC)"] :semantic_rank]+ [:distance :semantic_distance]])+ :from [:vector_candidates]+ :where [:<= :distance max-cosine-distance]+ :order-by [[:semantic_rank :asc]]}))+(defn- vector-search-strategy"Resolve the vector-search strategy for `search-context`, falling back to the configured default setting."[search-context](or (:vector-search-strategy search-context)(semantic-settings/semantic-search-vector-strategy)))+(def ^:private iterative-strategy->guc+ "Iterative vector-search strategies mapped to pgvector's `hnsw.iterative_scan` GUC value.+ The strategy keyword encodes the ordering, so there is no separate order knob."+ {:hnsw-iterative-relaxed "relaxed_order"+ :hnsw-iterative-strict "strict_order"})+(defn- semantic-search-query"Build the semantic vector subquery, dispatching on the resolved [[vector-search-strategy]].- `:brute-force` is exact and filter-first; `:hnsw` (the default) is approximate and index-backed."+ `:brute-force` is exact and filter-first; the `:hnsw-iterative-*` strategies are index-backed with inline+ filters and an iterative scan; `:hnsw` (the default) is approximate, index-backed and post-filters."[index embedding search-context](let [filters (search-filters search-context)embedding-literal (format-embedding embedding)strategy (vector-search-strategy search-context)]- (case strategy- :brute-force (brute-force-search-query index embedding-literal filters)- :hnsw (hnsw-search-query index embedding-literal filters)- (do (log/warnf "Unknown vector-search strategy %s; falling back to :hnsw" (pr-str strategy))- (hnsw-search-query index embedding-literal filters)))))+ (cond+ (= :brute-force strategy) (brute-force-search-query index embedding-literal filters)+ (iterative-strategy->guc strategy) (hnsw-iterative-search-query index embedding-literal filters)+ (= :hnsw strategy) (hnsw-search-query index embedding-literal filters)+ :else (do (log/warnf "Unknown vector-search strategy %s; falling back to :brute-force" (pr-str strategy))+ (brute-force-search-query index embedding-literal filters)))))++(defn- explain?+ "Whether to run the gated EXPLAIN (ANALYZE) instrumentation for `search-context`, falling back to the+ configured default setting."+ [search-context]+ (if (contains? search-context :vector-search-explain?)+ (boolean (:vector-search-explain? search-context))+ (semantic-settings/semantic-search-explain)))++(defn- vector-session-settings+ "`SET LOCAL` statements for the pgvector session GUCs implied by `search-context`, as a (possibly empty)+ vector of single-element `[sql]` statement vectors.+ Only the `:hnsw-iterative-*` strategies set the iterative-scan GUCs; any strategy may request+ `force-index?`.+ Each knob falls back to its EE setting when unset on the context."+ [search-context]+ ;; Values come from a fixed map or validated positive integers, so they are safe to interpolate (GUC+ ;; names and values can't be passed as bound parameters).+ ;; They are clamped to pgvector's GUC ranges (ef_search 1..1000, max_scan_tuples 32-bit) because an+ ;; out-of-range SET LOCAL raises mid-transaction and would fail the whole search; the API schema enforces+ ;; the same ranges, the clamp covers the settings path.+ (let [iterative-guc (iterative-strategy->guc (vector-search-strategy search-context))+ clamp (fn [v lo hi] (-> v (max lo) (min hi)))]+ (cond-> []+ iterative-guc+ (conj [(str "SET LOCAL hnsw.iterative_scan = " iterative-guc)]+ [(format "SET LOCAL hnsw.ef_search = %d"+ (clamp (or (:vector-search-ef-search search-context)+ (semantic-settings/semantic-search-ef-search))+ 1 1000))]+ [(format "SET LOCAL hnsw.max_scan_tuples = %d"+ (clamp (or (:vector-search-max-scan-tuples search-context)+ (semantic-settings/semantic-search-max-scan-tuples))+ 1 Integer/MAX_VALUE))])++ (:vector-search-force-index? search-context)+ (conj ["SET LOCAL enable_seqscan = off"]))))++(defn- run-in-vector-session!+ "Apply the [[vector-session-settings]] for `search-context` on a transaction over `db`, then call+ `(f conn)` with that transaction.+ `SET LOCAL` resets at COMMIT, so the pooled connection is left clean.+ When no session settings are needed and instrumentation is off, `f` is called directly on `db` (no+ transaction) to keep the default strategy's hot path unchanged."+ [db search-context f]+ (let [stmts (vector-session-settings search-context)]+ (if (and (empty? stmts) (not (explain? search-context)))+ (f db)+ (jdbc/with-transaction [tx db]+ (doseq [stmt stmts]+ (jdbc/execute! tx stmt))+ (f tx)))))(defn- flatten-ctes"Flatten nested :with clauses into a single top-level :with.@@ -923,6 +1010,103 @@(u/ignore-exceptions(with-open [_conn (.getConnection ^PooledDataSource db)]))))))+;; ----------------------------------------------------------------------------------------------------------+;; Gated vector-search instrumentation+;;+;; When `vector-search-explain?` is set (per-request) or `semantic-search-explain` (setting) is on, we+;; re-run the inner vector subquery under EXPLAIN (ANALYZE) to measure how the chosen strategy actually+;; behaves: which plan the index-table scan got, how many tuples it visited, how long the inner query took,+;; and how that compares to a filter-first scan's candidate pool.+;; It is off by default because the EXPLAIN re-executes the inner query.+;; ----------------------------------------------------------------------------------------------------------++(defn- explain->element+ "Coerce the single `EXPLAIN (… FORMAT JSON)` result value into its one plan element+ (`{\"Plan\" {…} \"Execution Time\" …}`)."+ [explain-value]+ (let [decoded (cond+ (instance? PGobject explain-value) (json/decode (unwrap-pgobject explain-value))+ (string? explain-value) (json/decode explain-value)+ :else explain-value)]+ (first decoded)))++(defn- find-scan-node+ "Depth-first search of an EXPLAIN plan tree for the scan node over `table-name`."+ [plan table-name]+ (when (map? plan)+ (if (and (#{"Seq Scan" "Index Scan" "Index Only Scan" "Bitmap Heap Scan"} (get plan "Node Type"))+ (= table-name (get plan "Relation Name")))+ plan+ (some #(find-scan-node % table-name) (get plan "Plans")))))++(defn- scan-node-metrics+ "Pull the index-table scan node out of an EXPLAIN plan tree and summarise it: the plan node chosen, the+ HNSW index used (nil for a seq scan), the tuples actually visited (returned + filtered out), and the+ node's total execution time."+ [plan table-name]+ (when-let [node (find-scan-node plan table-name)]+ ;; EXPLAIN reports per-loop averages, and for a parallel scan (the likely plan for the brute-force+ ;; baseline) loops = workers: row totals need the multiply, but per-loop "Actual Total Time" already+ ;; approximates wall clock because the workers run concurrently, so the time must NOT be multiplied+ (let [loops (get node "Actual Loops" 1)+ actual-rows (* (get node "Actual Rows" 0) loops)+ removed (* (get node "Rows Removed by Filter" 0) loops)]+ {:node-type (get node "Node Type")+ :index-name (get node "Index Name")+ :actual-rows actual-rows+ :rows-removed-by-filter removed+ :tuples-scanned (+ actual-rows removed)+ :inner-ms (get node "Actual Total Time" 0)+ :shared-hit (get node "Shared Hit Blocks")+ :shared-read (get node "Shared Read Blocks")})))++(defn- explain-vector-subquery+ "Run the inner vector subquery under EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) on `conn` and return its+ [[scan-node-metrics]] plus the subquery's total execution time.+ Re-executes the inner query."+ [conn index embedding search-context]+ (let [[sql & params] (sql-format-quoted (semantic-search-query index embedding search-context))+ explain-sql (str "EXPLAIN (ANALYZE true, BUFFERS true, FORMAT JSON) " sql)+ element (-> (jdbc/execute-one! conn (into [explain-sql] params)) vals first explain->element)]+ (assoc (scan-node-metrics (get element "Plan") (:table-name index))+ :execution-ms (get element "Execution Time")+ :planning-ms (get element "Planning Time"))))++(defn- prefilter-pool-size+ "Count of index rows the non-vector filters alone select -- the candidate pool a filter-first+ (brute-force) scan would compute distances over.+ The vector scan's `tuples-scanned` relative to this is the overfetch."+ [conn index search-context]+ (let [filters (search-filters search-context)+ q (cond-> {:select [[[:raw "count(*)"] :n]] :from [(keyword (:table-name index))]}+ filters (assoc :where filters))]+ (:n (jdbc/execute-one! conn (sql-format-quoted q) {:builder-fn jdbc.rs/as-unqualified-lower-maps}))))++(defn- record-vector-instrumentation!+ "Measure and report how the resolved strategy executed the inner vector subquery: EXPLAIN ANALYZE the+ scan and count the prefilter pool, then log a structured line and bump the analytics counters.+ Never throws -- instrumentation failures must not break search."+ [conn index embedding search-context raw-count]+ (try+ (let [strategy (name (vector-search-strategy search-context))+ scan (explain-vector-subquery conn index embedding search-context)+ pool (prefilter-pool-size conn index search-context)]+ (log/info "Semantic vector-search instrumentation"+ (merge {:strategy strategy+ :prefilter-pool-size pool+ :raw-count raw-count+ :used-index? (some? (:index-name scan))+ :overfetch-ratio (when (and pool (pos? pool) (:tuples-scanned scan))+ (double (/ (:tuples-scanned scan) pool)))}+ scan))+ (analytics/inc! :metabase-search/semantic-vector-inner-ms {:strategy strategy} (or (:inner-ms scan) 0))+ (analytics/inc! :metabase-search/semantic-vector-tuples-scanned {:strategy strategy} (or (:tuples-scanned scan) 0))+ (analytics/inc! :metabase-search/semantic-prefilter-pool-size {:strategy strategy} (or pool 0))+ (analytics/inc! :metabase-search/semantic-vector-scan-used-index+ {:strategy strategy :plan-node (or (:node-type scan) "unknown")} 1))+ (catch Exception e+ (log/warn e "Failed to record vector-search instrumentation"))))+(defn query-index"Query the index for documents similar to the search string.Returns a map with :results and :raw-count."@@ -932,11 +1116,17 @@(if (str/blank? search-string){:results [] :raw-count 0}(do- (when (and (= :hnsw (vector-search-strategy search-context))+ ;; `:vector-search-allow-missing-index?` is a deliberate opt-out for callers that want the inner+ ;; query to run without the HNSW index (e.g. the strategy matrix test probing the exact seq-scan+ ;; path); production traffic leaves it unset and gets the fail-fast.+ (when (and (contains? search.config/hnsw-index-backed-strategies (vector-search-strategy search-context))+ (not (:vector-search-allow-missing-index? search-context))(not (semantic.util/index-exists? db (hnsw-index-name index))))- (throw (ex-info (str "HNSW vector-search strategy requested but no HNSW index exists. "- "Set the semantic-search-vector-strategy setting to :hnsw to build it.")- {:table-name (:table-name index)})))+ (throw (ex-info (str "HNSW-index-backed vector-search strategy requested but no HNSW index exists. "+ "Set the semantic-search-vector-strategy setting to an index-backed strategy "+ "(:hnsw or :hnsw-iterative-*) to build it.")+ {:table-name (:table-name index)+ :strategy (vector-search-strategy search-context)})))(let [timer (u/start-timer);; Warm the pool concurrently with the embedding round-trip: the pool holds zero idle@@ -958,11 +1148,24 @@query (scored-search-query index embedding search-context scorers)xform (comp (map decode-legacy-input)(map (partial legacy-input-with-score weights (keys scorers))))- reducible (reducible-search-query db query)- raw-results (tracing/with-span :search "search.semantic.db-query"- {:search/query-length (count search-string)}- (into [] xform reducible))- db-query-time-ms (u/since-ms db-timer)+ ;; Run the query (and any gated instrumentation) inside a vector session so the iterative-scan+ ;; GUCs from `vector-session-settings` apply on the same connection. Falls back to the plain+ ;; datasource path for the default strategy with instrumentation off. The query duration is+ ;; captured before the instrumentation runs, so the EXPLAIN re-execution and prefilter count+ ;; don't inflate the latency signal they exist to analyze.+ db-query-ms* (volatile! 0.0)+ raw-results (run-in-vector-session!+ db+ search-context+ (fn [conn]+ (let [results (tracing/with-span :search "search.semantic.db-query"+ {:search/query-length (count search-string)}+ (into [] xform (reducible-search-query conn query)))]+ (vreset! db-query-ms* (u/since-ms db-timer))+ (when (explain? search-context)+ (record-vector-instrumentation! conn index embedding search-context (count results)))+ results)))+ db-query-time-ms @db-query-ms*filter-timer (u/start-timer)filtered-results (tracing/with-span :search "search.semantic.permission-filter"diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.cljindex a74a52e09cf..ea2690b3bf0 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj@@ -123,10 +123,12 @@(defsetting semantic-search-vector-strategy(deferred-tru- (str "Default vector-search strategy for semantic search: `hnsw` (approximate, HNSW-index-backed) or "+ (str "Default vector-search strategy for semantic search: `hnsw` (approximate, HNSW-index-backed), ""`brute-force` (exact, applies non-vector filters first then computes cosine distance over the "- "survivors). Defaults to `brute-force`, which needs no index; selecting `hnsw` builds the HNSW index "- "just-in-time. Individual requests may override this via the `vector_search_strategy` API parameter."))+ "survivors), or `hnsw-iterative-relaxed`/`hnsw-iterative-strict` (HNSW-index-backed iterative scans "+ "with inline filters). Defaults to `brute-force`, which needs no index; selecting `hnsw` builds the "+ "HNSW index just-in-time. Individual requests may override this via the `vector_search_strategy` API "+ "parameter.")):type :keyword:default :brute-force:encryption :no@@ -142,9 +144,54 @@{:invalid-value new-value:valid-values valid-vector-search-strategies})))(setting/set-value-of-type! :keyword :semantic-search-vector-strategy kw)- ;; Gated on the transition (not every set) so re-setting :hnsw doesn't rebuild the index.- (when (and (= kw :hnsw) (not= old :hnsw))- (events/publish-event! :event/semantic-search-hnsw-enabled {})))))+ ;; Every HNSW-index-backed strategy needs the index, so build it when transitioning into one+ ;; from a non-index-backed strategy. Gated on the transition (not every set) so switching+ ;; between index-backed strategies -- e.g. :hnsw -> :hnsw-iterative-strict -- doesn't rebuild.+ (let [index-backed? search.config/hnsw-index-backed-strategies]+ (when (and (index-backed? kw) (not (index-backed? old)))+ (events/publish-event! :event/semantic-search-hnsw-enabled {}))))))++(defsetting semantic-search-ef-search+ (deferred-tru+ (str "Default pgvector `hnsw.ef_search` (HNSW candidate-list size) for the `hnsw-iterative-*` strategies. "+ "Larger values improve recall at the cost of latency. Individual requests may override this via the "+ "`vector_search_ef_search` API parameter."))+ :type :positive-integer+ :default 40+ :encryption :no+ :export? false+ :visibility :internal+ :doc false)++(defsetting semantic-search-max-scan-tuples+ (deferred-tru+ (str "Default pgvector `hnsw.max_scan_tuples` (soft cap on tuples an iterative scan visits) for the "+ "`hnsw-iterative-*` strategies. Larger values improve recall under selective filters at the cost of "+ "latency. Individual requests may override this via the `vector_search_max_scan_tuples` API parameter."))+ :type :positive-integer+ :default 20000+ :encryption :no+ :export? false+ :visibility :internal+ :doc false)++;; The `vector_search_explain` API parameter only covers requests you author yourself; this setting exists+;; to instrument organic traffic. The frontend issues the real search requests and cannot pass the+;; parameter, so populating the vector-scan Prometheus counters and waterfall logs over production traffic+;; means flipping instrumentation on instance-wide (MB_SEMANTIC_SEARCH_EXPLAIN on hosted, no deploy).+;; EXPLAIN ANALYZE re-runs the inner vector subquery, so the intended lifecycle is on-for-an-investigation,+;; then off.+(defsetting semantic-search-explain+ (deferred-tru+ (str "Run EXPLAIN (ANALYZE) instrumentation of the inner vector subquery for every semantic search? "+ "Expensive (re-executes the inner query); intended for ad-hoc analysis. Individual requests may "+ "override this via the `vector_search_explain` API parameter."))+ :type :boolean+ :default false+ :encryption :no+ :export? false+ :visibility :internal+ :doc false)(defsetting semantic-search-min-results-threshold(deferred-tru "Minimum number of semantic search results required before falling back to other engines.")diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.cljindex bec65c03f0c..0045a21b5f5 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj@@ -8,6 +8,7 @@[metabase-enterprise.semantic-search.indexer :as semantic-search.indexer][metabase-enterprise.semantic-search.settings :as semantic.settings][metabase-enterprise.semantic-search.util :as semantic.u]+ [metabase.search.config :as search.config][metabase.search.engine :as search.engine][metabase.task.core :as task][metabase.util.log :as log])@@ -69,7 +70,9 @@(simple/with-interval-in-milliseconds (.toMillis run-frequency))(simple/repeat-forever))))](task/schedule-task! job trigger))- ;; Safety net: an instance that booted already configured for :hnsw (strategy set via env var, or set- ;; before an active index existed) never saw the setter's transition event, so build the index now.- (when (= :hnsw (semantic.settings/semantic-search-vector-strategy))+ ;; Safety net: an instance that booted already configured for an HNSW-index-backed strategy (set via env+ ;; var, or set before an active index existed) never saw the setter's transition event, so build the+ ;; index now. Covers :hnsw and the :hnsw-iterative-* strategies, which all query through the index.+ (when (contains? search.config/hnsw-index-backed-strategies+ (semantic.settings/semantic-search-vector-strategy))(semantic.core/build-hnsw-index-async!))))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.cljindex 699e3629012..1f57e677627 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj@@ -51,8 +51,17 @@(let [sql (vector-search-sql :hnsw)](is (not (str/includes? sql "MATERIALIZED")))(is (re-find #"ORDER BY embedding <=>" sql))- ;; the filter is applied in the outer query, after the candidate set is chosen- (is (str/includes? sql "\"archived\" = FALSE"))))+ ;; the filter is applied in the outer query, after the candidate set is chosen, so it appears *after*+ ;; the inner ORDER BY embedding <=> ... LIMIT in the generated SQL+ (is (str/includes? sql "\"archived\" = FALSE"))+ (is (not (re-find #"(?s)\"archived\" = FALSE.*ORDER BY embedding <=>" sql)))))+ (testing ":hnsw-iterative-* filters inline with the ordered/limited index scan (iterative scan)"+ (doseq [strategy [:hnsw-iterative-relaxed :hnsw-iterative-strict]]+ (let [sql (vector-search-sql strategy)]+ (is (not (str/includes? sql "MATERIALIZED")))+ ;; the filter lives *inside* the ordered+limited candidate scan (before the ORDER BY embedding <=>),+ ;; so the HNSW index sees it and the iterative scan can extend until the limit is met+ (is (re-find #"(?s)\"archived\" = FALSE.*ORDER BY embedding <=>" sql)))))(testing "no explicit strategy falls back to the configured default setting"(is (= (vector-search-sql (semantic.settings/semantic-search-vector-strategy))(vector-search-sql nil)))))@@ -80,10 +89,10 @@(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Invalid vector-search strategy"(semantic.settings/semantic-search-vector-strategy! :nonsense))))(testing "valid strategies round-trip"- (doseq [strategy [:hnsw :brute-force]]+ (doseq [strategy [:hnsw :brute-force :hnsw-iterative-relaxed :hnsw-iterative-strict]](mt/with-temporary-setting-values [semantic.settings/semantic-search-vector-strategy strategy](is (= strategy (semantic.settings/semantic-search-vector-strategy))))))- (testing "the setter kicks off the background build job only on the transition into :hnsw"+ (testing "the setter kicks off the background build job on the transition into any HNSW-index-backed strategy";; This asserts *when* the build is triggered (the transition gating), not what the build does -- the;; build itself is covered by pgvector-api-test/ensure-active-hnsw-index!-test. The setter publishes;; :event/semantic-search-hnsw-enabled, whose handler (semantic-search.events) calls the async build; we@@ -96,32 +105,106 @@(is (= 1 @triggers) "transitioning :brute-force -> :hnsw kicks off a build")(semantic.settings/semantic-search-vector-strategy! :hnsw)(is (= 1 @triggers) "setting :hnsw again when already :hnsw does not re-trigger")+ ;; switching between index-backed strategies reuses the existing index, so no rebuild+ (semantic.settings/semantic-search-vector-strategy! :hnsw-iterative-strict)+ (is (= 1 @triggers) "switching :hnsw -> :hnsw-iterative-strict does not re-trigger")(semantic.settings/semantic-search-vector-strategy! :brute-force)- (is (= 1 @triggers) "switching away from :hnsw does not trigger")- (semantic.settings/semantic-search-vector-strategy! :hnsw)- (is (= 2 @triggers) "transitioning back into :hnsw triggers again")))))))+ (is (= 1 @triggers) "switching away to :brute-force does not trigger")+ (semantic.settings/semantic-search-vector-strategy! :hnsw-iterative-relaxed)+ (is (= 2 @triggers) "transitioning :brute-force -> :hnsw-iterative-relaxed kicks off a build")))))))++(deftest vector-session-settings-test+ (testing ":hnsw and :brute-force need no session GUCs"+ (is (empty? (#'semantic.index/vector-session-settings {:vector-search-strategy :hnsw})))+ (is (empty? (#'semantic.index/vector-session-settings {:vector-search-strategy :brute-force}))))+ (testing ":hnsw-iterative-* emits iterative_scan/ef_search/max_scan_tuples SET LOCALs; the strategy sets the order"+ (is (= ["SET LOCAL hnsw.iterative_scan = strict_order"+ "SET LOCAL hnsw.ef_search = 100"+ "SET LOCAL hnsw.max_scan_tuples = 50000"]+ (map first (#'semantic.index/vector-session-settings+ {:vector-search-strategy :hnsw-iterative-strict+ :vector-search-ef-search 100+ :vector-search-max-scan-tuples 50000}))))+ (is (= "SET LOCAL hnsw.iterative_scan = relaxed_order"+ (first (first (#'semantic.index/vector-session-settings+ {:vector-search-strategy :hnsw-iterative-relaxed}))))))+ (testing "force-index? appends enable_seqscan = off for any strategy"+ (is (= ["SET LOCAL enable_seqscan = off"]+ (map first (#'semantic.index/vector-session-settings+ {:vector-search-strategy :hnsw :vector-search-force-index? true}))))))++(deftest ^:synchronized semantic-search-instrumentation-test+ (mt/with-premium-features #{:semantic-search}+ (with-open [_ (semantic.tu/open-temp-index!)]+ (semantic.tu/upsert-index! (semantic.tu/mock-documents))+ (testing "vector-search-explain? runs EXPLAIN ANALYZE and emits the vector-scan instrumentation metrics"+ (let [analytics-calls (atom [])]+ (mt/with-dynamic-fn-redefs [analytics/inc! (fn [metric & args]+ (swap! analytics-calls conj [metric args]))]+ (mt/with-test-user :crowberto+ (semantic.index/query-index (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index+ {:search-string "dog training" :vector-search-explain? true}))+ (let [by-metric (into {} (map (juxt first (comp vec second))) @analytics-calls)]+ (testing "all four instrumentation metrics fire with numeric values"+ (doseq [metric [:metabase-search/semantic-vector-inner-ms+ :metabase-search/semantic-vector-tuples-scanned+ :metabase-search/semantic-prefilter-pool-size]]+ (is (contains? by-metric metric))+ ;; args are [labels amount]; the amount must be a non-negative number+ (is (number? (second (by-metric metric))) (str metric " amount should be numeric"))+ (is (<= 0 (second (by-metric metric))) (str metric " amount should be non-negative"))))+ (testing "the scan plan node is reported as a label"+ (is (contains? by-metric :metabase-search/semantic-vector-scan-used-index))+ (is (string? (get-in (by-metric :metabase-search/semantic-vector-scan-used-index)+ [0 :plan-node]))))))))+ (testing "with instrumentation off (the default) no instrumentation metrics are emitted"+ (let [analytics-calls (atom [])]+ (mt/with-dynamic-fn-redefs [analytics/inc! (fn [metric & args]+ (swap! analytics-calls conj [metric args]))]+ (mt/with-test-user :crowberto+ (semantic.index/query-index (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index+ {:search-string "dog training"}))+ (is (not (contains? (set (map first @analytics-calls))+ :metabase-search/semantic-vector-inner-ms)))))))))++(defn- expected-index-defs+ "The index-DDL contract of [[semantic.index/create-index-table-if-not-exists!]] for `index`.+ Maps the name of each index it must create to a pattern its pg_indexes indexdef must match."+ [index]+ ;; the pkey and (model, model_id) unique-constraint indexes come from the table schema, not CREATE INDEX+ ;; statements, so they are deliberately absent here; =? tolerates them as extra keys in the actual+ {(semantic.index/hnsw-index-name index) #"CREATE INDEX .* USING hnsw \(embedding vector_cosine_ops\)"+ (semantic.index/fts-index-name index) #"CREATE INDEX .* USING gin \(text_search_vector\)"+ (semantic.index/fts-native-index-name index) #"CREATE INDEX .* USING gin \(text_search_with_native_query_vector\)"+ (#'semantic.index/content-index-name index) #"CREATE INDEX .* USING btree \(content\)"})(deftest create-index-table!-test(mt/with-premium-features #{:semantic-search}(with-open [index-ref (semantic.tu/open-temp-index! :hnsw? false)]- ;; open-temp-index! creates the temp table, so drop it in order to test create!.- (semantic.index/drop-index-table! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index)- (testing "index table is not present before create!"- (is (not (semantic.tu/table-exists-in-db? (:table-name @index-ref))))- (is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref))))- (is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-index-name @index-ref))))- (is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-native-index-name @index-ref)))))- (testing "under the default :brute-force strategy, create! builds the table and FTS indexes but not the HNSW index"- (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly :brute-force)]- (semantic.index/create-index-table-if-not-exists! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index {:force-reset? true}))- (is (semantic.tu/table-exists-in-db? (:table-name @index-ref)))- (is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref))))- (is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-index-name @index-ref)))- (is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/fts-native-index-name @index-ref))))- (testing "when configured for the :hnsw strategy, create! also builds the HNSW index"- (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly :hnsw)]- (semantic.index/create-index-table-if-not-exists! (semantic.env/get-pgvector-datasource!) semantic.tu/mock-index {:force-reset? true}))- (is (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref)))))))+ (let [index @index-ref+ table-name (:table-name index)]+ ;; open-temp-index! creates the temp table, so drop it in order to test create!.+ (semantic.index/drop-index-table! (semantic.env/get-pgvector-datasource!) index)+ (testing "neither the table nor any of its indexes is present before create!"+ (is (not (semantic.tu/table-exists-in-db? table-name)))+ (is (= {} (semantic.tu/table-indexes table-name))))+ (testing "under the default :brute-force strategy, create! builds the table and FTS indexes but not the HNSW index"+ (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly :brute-force)]+ (semantic.index/create-index-table-if-not-exists!+ (semantic.env/get-pgvector-datasource!) index {:force-reset? true}))+ (is (semantic.tu/table-exists-in-db? table-name))+ (is (=? (dissoc (expected-index-defs index) (semantic.index/hnsw-index-name index))+ (semantic.tu/table-indexes table-name)))+ (is (not (semantic.tu/table-has-index? table-name (semantic.index/hnsw-index-name index)))))+ (testing "under the :hnsw strategy, force-reset? rebuilds the full contract (incl. HNSW) and wipes data"+ ;; seed rows first; rows that survive the reset would mean force-reset? was silently ignored+ (semantic.tu/upsert-index! (semantic.tu/mock-documents))+ (is (pos? (semantic.tu/index-count index)))+ (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly :hnsw)]+ (semantic.index/create-index-table-if-not-exists!+ (semantic.env/get-pgvector-datasource!) index {:force-reset? true}))+ (is (zero? (semantic.tu/index-count index)))+ (is (=? (expected-index-defs index) (semantic.tu/table-indexes table-name))))))))(deftest create-hnsw-index-if-not-exists!-test(mt/with-premium-features #{:semantic-search}@@ -138,15 +221,17 @@"relfilenode is unchanged, so the index was not dropped, rebuilt, or reindexed"))))))(deftest query-index-hnsw-without-index-throws-test- (testing "a query using the :hnsw strategy fails fast when no HNSW index exists, rather than silently scanning"+ (testing "a query under any HNSW-index-backed strategy fails fast when no HNSW index exists, rather than silently scanning"(mt/with-premium-features #{:semantic-search}(with-open [index-ref (semantic.tu/open-temp-index! :hnsw? false)](is (not (semantic.tu/table-has-index? (:table-name @index-ref) (semantic.index/hnsw-index-name @index-ref))))- (is (thrown-with-msg?- clojure.lang.ExceptionInfo #"no HNSW index exists"- (semantic.index/query-index (semantic.env/get-pgvector-datasource!)- @index-ref- {:search-string "puppy" :vector-search-strategy :hnsw})))))))+ (doseq [strategy [:hnsw :hnsw-iterative-relaxed :hnsw-iterative-strict]]+ (testing strategy+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo #"no HNSW index exists"+ (semantic.index/query-index (semantic.env/get-pgvector-datasource!)+ @index-ref+ {:search-string "puppy" :vector-search-strategy strategy})))))))))(deftest drop-index-table!-test(mt/with-premium-features #{:semantic-search}diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.cljindex 52362c63951..62b623b8d0c 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj@@ -11,10 +11,10 @@(set! *warn-on-reflection* true)(deftest startup-hnsw-safety-net-test- (testing "the indexer task's startup builds the HNSW index only when configured for :hnsw"- ;; Covers instances that boot already configured for :hnsw (e.g. strategy set via env var), where the- ;; setter's transition event never fired -- see the safety net in sut/init!. Redefs avoid scheduling a- ;; real quartz job or spawning a real build future.+ (testing "the indexer task's startup builds the HNSW index when configured for any index-backed strategy"+ ;; Covers instances that boot already configured for an HNSW-index-backed strategy (e.g. strategy set via+ ;; env var), where the setter's transition event never fired -- see the safety net in sut/init!. Redefs+ ;; avoid scheduling a real quartz job or spawning a real build future.(let [builds (atom 0)](mt/with-dynamic-fn-redefs [semantic.u/semantic-search-available? (constantly true)task/schedule-task! (fn [& _] nil)@@ -23,7 +23,9 @@(mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly :brute-force)](task/init! ::sut/SemanticSearchIndexer))(is (zero? @builds)))- (testing ":hnsw triggers a build"- (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly :hnsw)]- (task/init! ::sut/SemanticSearchIndexer))- (is (= 1 @builds)))))))+ (doseq [strategy [:hnsw :hnsw-iterative-relaxed :hnsw-iterative-strict]]+ (testing (str strategy " triggers a build")+ (reset! builds 0)+ (mt/with-dynamic-fn-redefs [semantic.settings/semantic-search-vector-strategy (constantly strategy)]+ (task/init! ::sut/SemanticSearchIndexer))+ (is (= 1 @builds))))))))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.cljindex bfdd4c2b31a..2655afe530b 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj@@ -514,6 +514,16 @@(-> result first vals first))(catch Exception _ false))))+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}+(defn table-indexes+ "Map of index name -> pg_indexes indexdef for `table-name`; empty when the table does not exist."+ [table-name]+ (into {}+ (map (juxt :indexname :indexdef))+ (jdbc/execute! (semantic.env/get-pgvector-datasource!)+ ["SELECT indexname, indexdef FROM pg_indexes WHERE tablename = ?" (name table-name)]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})))+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}(defn index-relfilenode"Return the on-disk relfilenode for the index named `index-name`, or nil if it doesn't exist. A stablediff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/vector_strategy_matrix_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/vector_strategy_matrix_test.cljnew file mode 100644index 00000000000..8024c821699--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/vector_strategy_matrix_test.clj@@ -0,0 +1,548 @@+(ns metabase-enterprise.semantic-search.vector-strategy-matrix-test+ "Quality matrix over the vector-search strategies on a small deterministic dataset.++ Headline:++ - naive :hnsw -- too risky around under-fetching.+ An anti-correlated :verified filter empties its pool outright (final recall 0.0 where :brute-force+ scores 1.0), and even a half-selective filter halves it before the permission drop halves it again+ ({:raw-count 10 :returned 6} against 20/16 for every other variant).+ - :brute-force -- reliable, but poor asymptotic performance.+ Exact in every cell of the matrix, at the cost of computing distances over the whole filtered table.+ And because that cost tracks the filtered pool size, latency varies query-to-query with the filters --+ intermittent slowness that can frustrate more than a uniformly slower engine.+ - :hnsw-iterative-* -- promising, but can still underperform.+ It recovers everything the naive scan loses to filters (recall 1.0 vs 0.0) at index-backed cost.+ Yet approximation shows up at real dimensionality (recall@20 observed at 0.15-0.6 on the 32-d packed+ dataset with ef_search 16), a binding hnsw.max_scan_tuples empties a filtered scan that :brute-force+ answers exactly, and the post-retrieval re-ranking and permission stages lose just as much through it+ (0.4 recall under limit truncation, a 20 -> 12 pool drop) because they run after retrieval.++ Caveat: every number above comes from a very, very small index (300 docs; 2000 for the packed dataset).+ The mechanisms are the same at scale, but their effects can be far more drastic.++ Covers four retrieval variants -- no HNSW index (exact seq scan), naive :hnsw (post-filter), and the two+ iterative-scan strategies -- with :brute-force as the exact SQL reference, and measures three things:++ 1. distance recall / NDCG of candidate retrieval against a ground truth computed in pure Clojure,+ 2. recall of the final re-ranked top results against an exhaustive run of the same scoring pipeline,+ 3. under-fetch caused by post-retrieval permission filtering.++ The trade-offs demonstrated: even exact retrieval under-delivers end-to-end (the candidate pool is+ truncated at `semantic-search-results-limit` before re-ranking, and permission checks drop candidates+ after), naive :hnsw misses everything under an anti-correlated filter, and the iterative strategies fix+ filter-driven misses but not limit truncation or permission under-fetch.+ Under a half-selective filter naive :hnsw also under-fetches at the SQL stage before the permission drop+ cuts again -- the two losses compound for the post-filtering strategy only.++ The matrix cells assert exact values and orderings, not thresholds.+ Embeddings sit at exact distinct distances from the probe (no ties), the heap is populated in a fixed+ order before the index exists, table stats are pinned with ANALYZE, and every indexed query forces the+ index path (a 300-row table would otherwise always seq-scan).+ The HNSW graph itself is NOT reproducible (pgvector draws node levels from the per-backend global PRNG,+ which `setseed` does not touch), so exact cells are only those whose outcome is invariant across graphs;+ each was validated against 30-60 independently built graphs.+ On the 4-d matrix dataset the graph is exact regardless of its random shape -- even 3000 densely packed+ filler docs do not change that -- so the matrix pins the structural differences between the strategies'+ SQL shapes and scan termination rules.+ Graph approximation error needs dimensionality, not tree size: [[graph-approximation-test]] covers that+ axis on a 32-d densely packed dataset with banded assertions (band edges sit several misses away from+ everything observed across dozens of graphs)."+ (:require+ [clojure.core.memoize :as memoize]+ [clojure.string :as str]+ [clojure.test :refer :all]+ [metabase-enterprise.semantic-search.env :as semantic.env]+ [metabase-enterprise.semantic-search.index :as semantic.index]+ [metabase-enterprise.semantic-search.scoring :as semantic.scoring]+ [metabase-enterprise.semantic-search.test-util :as semantic.tu]+ [metabase.permissions.models.permissions :as perms]+ [metabase.permissions.models.permissions-group :as perms-group]+ [metabase.test :as mt]+ [next.jdbc :as jdbc])+ (:import+ (java.util Random)))++(set! *warn-on-reflection* true)++(use-fixtures :once #'semantic.tu/once-fixture)++;;;; quality metrics++(def ^:private cutoff+ "The production cosine-distance cutoff; the dataset's bands are designed around it."+ @#'semantic.index/max-cosine-distance)++(defn- recall-at+ "Fraction of the ideal top-`k` ids present in the returned top-`k`."+ [k ideal-ids returned-ids]+ (let [ideal (set (take k ideal-ids))]+ (double (/ (count (filter ideal (take k returned-ids)))+ (count ideal)))))++(defn- sim+ "Graded relevance of a cosine distance `d` in [0,2]: 1 (identical) .. 0 (opposite)."+ [d]+ (max 0.0 (- 1.0 (/ (double d) 2.0))))++(defn- dcg+ "Discounted cumulative gain of a sequence of per-position relevances."+ [rels]+ (transduce (map-indexed (fn [i r] (/ (double r) (/ (Math/log (+ i 2.0)) (Math/log 2.0))))) + 0.0 rels))++(defn- ndcg-at+ "NDCG@`k` of `returned-ids` against the exact `ideal-ids` ranking.+ Relevance is graded by true distance via `id->distance`."+ [k id->distance ideal-ids returned-ids]+ (/ (dcg (map (comp sim id->distance) (take k returned-ids)))+ (dcg (map (comp sim id->distance) (take k ideal-ids)))))++;;;; dataset++(def ^:private probe-text "zebra quark")+(def ^:private probe-vector [1.0 0.0 0.0 0.0])++(def ^:private bands+ "Band layout for the 300-doc dataset, in insertion (and ascending-distance) order.++ - :restricted sits nearest the probe but in a permission-restricted collection, so it occupies top-k slots+ that the permission filter then drops (under-fetch).+ - :core/:mid are bulk filler so retrieval is nowhere near saturated.+ - :pinned is inside the cutoff but far outside any candidate pool -- the re-ranking lever for layer 2.+ - :rare holds the only verified docs and is anti-correlated with the probe: a :verified filter on it is+ adversarial for post-filtering strategies. The adversarial filter must be a column with no supporting+ btree: a :models filter would be served exactly by the (model, model_id) unique constraint at this table+ size, bypassing the HNSW scan the matrix is exercising.+ - :decoy lies beyond the cutoff and must never be returned."+ [{:band :restricted :n 8 :model "card" :d0 0.020 :dstep 0.005}+ {:band :core :n 180 :model "card" :d0 0.100 :dstep 0.0015}+ {:band :mid :n 74 :model "card" :d0 0.400 :dstep 0.0008}+ {:band :pinned :n 6 :model "card" :d0 0.465 :dstep 0.002 :pinned true}+ {:band :rare :n 12 :model "card" :d0 0.550 :dstep 0.008 :verified true}+ {:band :decoy :n 20 :model "card" :d0 0.750 :dstep 0.007}])++(defn- vector-at-distance+ "A unit `dims`-d vector at exact cosine distance `d` (pgvector `<=>`) from the probe `[1 0 0 ...]`."+ [^Random rng dims d]+ ;; the probe component is fixed by `d` and the remainder is a seeded random unit direction orthogonal to+ ;; the probe axis, so the cosine against the probe is exactly 1 - d+ (let [u (loop []+ (let [g (repeatedly (dec dims) #(.nextGaussian rng))+ n (Math/sqrt (double (reduce + (map * g g))))]+ (if (< n 1e-6) (recur) (mapv #(/ % n) g))))+ a (- 1.0 d)+ b (Math/sqrt (- 1.0 (* a a)))]+ (into [a] (map #(* b %)) u)))++(defn- make-dataset+ "Build the deterministic banded dataset.+ Returns {:docs [...], :embeddings {text -> vec}, :id->distance {id -> d}, :id->band {id -> band}}.+ The creator alternates with id parity (odd -> crowberto, even -> rasta), so a `created-by` filter is+ half-selective in every band.+ Doc names share no stem with [[probe-text]], so the keyword side of the hybrid query never matches and the+ RRF rank reduces to the pure semantic (distance) rank."+ [{:keys [seed restricted-collection-id]}]+ (let [rng (Random. (long seed))+ rows (for [{:keys [band n model d0 dstep pinned verified]} bands+ i (range n)]+ {:band band :model model :pinned pinned :verified verified :d (+ d0 (* i dstep))})+ rows (map-indexed (fn [idx row]+ (let [{:keys [band model]} row]+ (assoc row+ :id (inc idx)+ :name (format "%s %s %03d" (name band) model (inc idx))+ :vector (vector-at-distance rng 4 (:d row)))))+ rows)]+ {:docs (vec (for [{:keys [band model pinned verified d id name]} rows]+ (let [collection-id (when (= band :restricted) restricted-collection-id)]+ {:model model+ :id id+ :name name+ :searchable_text name+ :embeddable_text name+ :created_at #t "2025-01-01T12:00:00Z"+ :creator_id (mt/user->id (if (odd? id) :crowberto :rasta))+ :archived false+ :pinned (boolean pinned)+ :verified (boolean verified)+ :collection_id collection-id+ ;; the permission filter runs on the decoded legacy_input, so like the real+ ;; ingestion documents it must carry collection_id itself+ :legacy_input {:id id :model model :name name :collection_id collection-id}+ :metadata {:title name :distance d}})))+ :embeddings (into {probe-text probe-vector} (map (juxt :name :vector)) rows)+ :id->distance (into {} (map (juxt :id :d)) rows)+ :id->band (into {} (map (juxt :id :band)) rows)}))++(defn- exact-ids+ "Ids of docs satisfying `pred`, within the distance cutoff, in exact ascending-distance order.+ This is the pure-Clojure ground truth the SQL retrieval is measured against."+ [{:keys [docs id->distance]} pred]+ (->> docs+ (filter pred)+ (filter #(<= (id->distance (:id %)) cutoff))+ (sort-by #(id->distance (:id %)))+ (mapv :id)))++(defn- band-ids+ [{:keys [id->band]} band]+ (set (keep (fn [[id b]] (when (= b band) id)) id->band)))++(deftest dataset-sanity-test+ (let [ds (make-dataset {:seed 42 :restricted-collection-id -1})]+ (testing "300 docs, distinct strictly-positive distances, unit vectors"+ (is (= 300 (count (:docs ds))))+ (is (= 300 (count (distinct (vals (:id->distance ds))))))+ (is (every? pos? (vals (:id->distance ds))))+ (is (every? #(< (abs (- 1.0 (reduce + (map * % %)))) 1e-9)+ (vals (dissoc (:embeddings ds) probe-text)))))+ (testing "geometry the assertions rely on"+ (let [top-20 (set (take 20 (exact-ids ds (constantly true))))]+ (testing "exact top-20 = the whole restricted band + nearest core docs, nothing verified"+ (is (every? (some-fn (band-ids ds :restricted) (band-ids ds :core)) top-20))+ (is (every? top-20 (band-ids ds :restricted))))+ (testing "rare verified docs are all inside the cutoff, decoys all outside"+ (is (= (band-ids ds :rare) (set (exact-ids ds :verified))))+ (is (empty? (filter (band-ids ds :decoy) (exact-ids ds (constantly true))))))+ (testing "at least a full pool of accessible docs lies within the cutoff"+ (is (<= 20 (count (remove (band-ids ds :restricted) (exact-ids ds (constantly true)))))))+ (testing "creator parity makes a created-by filter half-selective: 10 odd-id docs in the top-20, and+ the nearest 20 odd-id docs are 4 restricted + 16 core"+ (is (= 10 (count (filter odd? top-20))))+ (let [odd-top-20 (take 20 (exact-ids ds (comp odd? :id)))]+ (is (= 20 (count odd-top-20)))+ (is (= 4 (count (filter (band-ids ds :restricted) odd-top-20))))+ (is (= 16 (count (filter (band-ids ds :core) odd-top-20))))))))))++;;;; fixture++(def ^:private results-limit+ "The candidate-pool size every variant retrieves, well below the 300-doc dataset so nothing saturates."+ 20)++(def ^:private iterative-ef-search+ "Below [[results-limit]], so the iterative strategies must run more than one scan iteration."+ 16)++(defn- assert-pgvector-prereqs!+ "Fail fast unless pgvector supports iterative scans and the server ef_search default is what the naive+ :hnsw cells assume (there is deliberately no per-context ef knob for the naive strategy)."+ []+ (jdbc/with-transaction [tx (semantic.env/get-pgvector-datasource!)]+ ;; a vector op loads the extension library, which registers the hnsw.* GUCs for SHOW+ (jdbc/execute! tx ["SELECT '[1,2,3]'::vector <=> '[1,2,3]'::vector"])+ (let [version (-> (jdbc/execute-one! tx ["SELECT extversion FROM pg_extension WHERE extname = 'vector'"])+ vals first str)+ ef-search (-> (jdbc/execute-one! tx ["SHOW hnsw.ef_search"]) vals first)]+ (when-not (<= 0 (compare (mapv parse-long (take 2 (str/split version #"\.")))+ [0 8]))+ (throw (ex-info "vector-strategy matrix needs pgvector >= 0.8.0 (iterative scans)"+ {:extversion version})))+ (when-not (= "40" ef-search)+ (throw (ex-info "vector-strategy matrix assumes the stock hnsw.ef_search default"+ {:hnsw.ef_search ef-search}))))))++(def ^:private ^:dynamic *index*+ "The semantic index the helpers below operate on; [[graph-approximation-test]] rebinds it."+ semantic.tu/mock-index)++(defn- drop-hnsw-index!+ []+ (jdbc/execute! (semantic.env/get-pgvector-datasource!)+ [(str "DROP INDEX IF EXISTS " (semantic.index/hnsw-index-name *index*))]))++(defn- build-hnsw-index!+ "Build the HNSW index on [[*index*]]'s table."+ []+ ;; the graph is NOT reproducible: pgvector draws node levels from the per-backend global PRNG, which+ ;; `setseed` does not touch, so every assertion against the index must hold for any random graph+ (jdbc/with-transaction [tx (semantic.env/get-pgvector-datasource!)]+ ;; serial build keeps insertion order fixed and removes worker-count variation+ (jdbc/execute! tx ["SET LOCAL max_parallel_maintenance_workers = 0"])+ ;; m/ef_construction are the current pgvector defaults, pinned against future default changes+ (jdbc/execute! tx [(format (str "CREATE INDEX %s ON %s USING hnsw (embedding vector_cosine_ops) "+ "WITH (m = 16, ef_construction = 64)")+ (semantic.index/hnsw-index-name *index*)+ (:table-name *index*))])))++(defn- do-with-matrix-fixture!+ "Run `(f dataset)` against a temp pgvector DB whose index table is populated with the matrix dataset but has+ NO HNSW index yet. `f` runs its exact cells first, then calls [[build-hnsw-index!]] for the indexed ones."+ [f]+ (mt/with-premium-features #{:semantic-search}+ (mt/with-temporary-setting-values [semantic-search-results-limit results-limit]+ (mt/with-temp [:model/Collection {restricted-id :id} {:name "Matrix Restricted Collection"}]+ (perms/revoke-collection-permissions! (perms-group/all-users) restricted-id)+ (let [ds (make-dataset {:seed 42 :restricted-collection-id restricted-id})]+ (semantic.tu/with-mock-embeddings (:embeddings ds)+ (semantic.tu/with-test-db! {:mode :mock-initialized}+ (assert-pgvector-prereqs!)+ (drop-hnsw-index!)+ (semantic.tu/upsert-index! (:docs ds) :serial? true)+ ;; pin the stats the planner sees, so plan choice cannot vary with autoanalyze timing+ (jdbc/execute! (semantic.env/get-pgvector-datasource!)+ [(str "ANALYZE " (:table-name *index*))])+ (f ds))))))))++(def ^:private base-ctx+ {:search-string probe-text+ :search-engine "semantic"+ :archived? false})++(defn- variant-ctx+ "Per-variant search-context overrides."+ [variant]+ (case variant+ ;; the :hnsw query shape run before build-hnsw-index!, to measure exact pre-index retrieval. Opt out of+ ;; the missing-index fail-fast, which production traffic gets but this baseline deliberately bypasses.+ :no-index {:vector-search-strategy :hnsw+ :vector-search-allow-missing-index? true}+ :brute-force {:vector-search-strategy :brute-force}+ ;; force-index because the planner would otherwise always seq-scan a 300-row table, making every+ ;; variant silently exact+ :hnsw {:vector-search-strategy :hnsw+ :vector-search-force-index? true}+ (:hnsw-iterative-relaxed :hnsw-iterative-strict)+ ;; both GUC knobs pinned so nothing depends on settings; max_scan_tuples exceeds the table so it never+ ;; binds except where a cell overrides it+ {:vector-search-strategy variant+ :vector-search-force-index? true+ :vector-search-ef-search iterative-ef-search+ :vector-search-max-scan-tuples 10000}))++(defn- query-results!+ "Run the full [[semantic.index/query-index]] pipeline for `variant`.+ Returns query-index's {:results ... :raw-count ...}."+ [variant extra-ctx]+ (memoize/memo-clear! #'semantic.scoring/view-count-percentiles)+ (semantic.index/query-index (semantic.env/get-pgvector-datasource!)+ *index*+ (merge base-ctx+ {:current-user-id (mt/user->id :rasta)}+ (variant-ctx variant)+ extra-ctx)))++(defn- query-ids!+ "Ordered result ids for `variant`, through the full pipeline."+ [variant & {:as extra-ctx}]+ (mapv :id (:results (query-results! variant extra-ctx))))++;;;; if one of these tests flakes+;;+;; The HNSW graph is rebuilt randomly on every run (see [[build-hnsw-index!]] -- it cannot be seeded), so a+;; failure here is one of three things:+;;+;; 1. An EXACT matrix cell failing intermittently. These cells assert outcomes that are invariant across+;; random graphs (each was validated against 30-60 independently built graphs on pgvector 0.8.1), so an+;; intermittent failure means the invariance itself broke -- almost certainly a pgvector behavior change+;; (graph build defaults, scan-phase sizing, max_scan_tuples semantics). Do not loosen the assertion+;; blindly: re-validate with the loop below, work out which scan behavior changed, and either restore the+;; invariant by pinning the changed parameter or convert the cell to a band derived from fresh+;; measurements.+;; 2. A BANDED cell in [[graph-approximation-test]] failing. The edges sit several misses beyond the range+;; observed across dozens of graphs (the observed range is noted per cell), but a rare excursion past an+;; edge is possible. Re-derive the edges rather than nudging by one miss: run the loop below, take the+;; min/max per cell, and set edges ~0.1-0.15 of recall beyond them. Keep upper edges strictly below 1.0;+;; a band touching 1.0 no longer demonstrates approximation, and the fix is a harder dataset (higher+;; dims, denser packing), not a wider band.+;; 3. A deterministic failure after an environment change (new pgvector version, changed server defaults).+;; [[assert-pgvector-prereqs!]] catches the known dependencies on server state; extend it when a new one+;; is discovered.+;;+;; Each test run builds a fresh graph, so re-measuring a cell's distribution is just repetition:+;;+;; (frequencies (map (fn [_] (dissoc (clojure.test/run-test <the-deftest>) :type)) (range 30)))+;;+;; For per-cell values, evaluate the cell's recall expression inside the fixture across runs (bind an atom,+;; collect, take min/max). Beware deriving bands from raw SQL probes instead: in-pipeline recall has fatter+;; tails (an ef8-vs-ef64 margin assertion derived from SQL probes flaked within 10 in-pipeline runs).++;;;; layer 1: distance-level retrieval vs exact ground truth++(deftest distance-recall-matrix-test+ (do-with-matrix-fixture!+ (fn [ds]+ ;; isolate retrieval: weights reduce ranking to the distance rank, permission filtering is bypassed+ (semantic.tu/with-only-semantic-weights+ (mt/with-dynamic-fn-redefs [semantic.index/filter-read-permitted identity]+ (let [exact-all (exact-ids ds (constantly true))+ exact-rare (exact-ids ds :verified)]+ (testing "without an HNSW index"+ (testing "retrieval is exact: the pool is the true top-20 in distance order"+ (is (= (take 20 exact-all) (query-ids! :no-index)))+ (is (= (take 20 exact-all) (query-ids! :brute-force))))+ (testing "but post-filtering still finds NOTHING for an anti-correlated :verified filter"+ ;; the exact global top-20 contains nothing verified, so filtering it afterwards yields zero;+ ;; only filter-first (:brute-force) retrieval is immune+ (is (= [] (query-ids! :no-index :verified true)))+ (is (= exact-rare (query-ids! :brute-force :verified true)))))+ (build-hnsw-index!)+ (testing "with the HNSW index (forced)"+ (testing "unfiltered retrieval is exact: graph approximation error does not manifest at this scale"+ ;; a 300-doc 4-d graph returns the true top-20 even at ef_search 2, so these cells pin+ ;; exactness rather than a recall band; the approximation-error axis is a scale phenomenon+ ;; and is measured by the at-scale benchmarking harness, not this matrix+ (doseq [variant [:hnsw :hnsw-iterative-relaxed :hnsw-iterative-strict]]+ (let [ids (query-ids! variant)]+ (testing variant+ (is (= (take 20 exact-all) ids))+ (is (= 1.0 (ndcg-at 20 (:id->distance ds) exact-all ids)))))))+ (testing "naive :hnsw post-filters the same global pool, so the adversarial filter finds nothing"+ ;; structural, not graph luck: the scan's top-20 would need fewer than 20 of the 262 docs+ ;; nearer than the rare band for anything verified to enter the pool+ (is (= [] (query-ids! :hnsw :verified true))))+ (testing "iterative scans pull neighbours past the filter and fully recover, in exact order"+ ;; the inner limit (20) exceeds the 12 matches, so the scan runs until the graph is exhausted+ (doseq [variant [:hnsw-iterative-relaxed :hnsw-iterative-strict]]+ (testing variant+ (is (= exact-rare (query-ids! variant :verified true))))))+ (testing "hnsw.max_scan_tuples truncates an iterative scan before it reaches far-away matches"+ ;; the rare band sits at distance ranks 269-280, far past a 64-tuple visit budget; the cap+ ;; has scan-phase granularity, but no phase overshoots by the ~200 tuples that would need+ (is (= [] (query-ids! :hnsw-iterative-relaxed+ :verified true+ :vector-search-max-scan-tuples 64)))))))))))++;;;; layer 2: final top results vs an exhaustive run of the same scoring pipeline++(deftest full-pipeline-recall-matrix-test+ (do-with-matrix-fixture!+ (fn [ds]+ (mt/with-dynamic-fn-redefs [semantic.index/filter-read-permitted identity]+ (testing "re-ranking: a deterministic :pinned boost promotes docs the candidate pool never contains"+ (semantic.tu/with-weights {:rrf 1 :pinned 10}+ (let [ideal (mt/with-temporary-setting-values [semantic-search-results-limit 1000]+ (query-ids! :brute-force))+ ideal-top (take 10 ideal)+ exact-all (exact-ids ds (constantly true))]+ (testing "the ideal top-10 is the pinned band plus the nearest docs"+ (is (= (into (band-ids ds :pinned) (take 4 exact-all))+ (set ideal-top))))+ (testing "exact retrieval scores only its truncated pool: pinned docs sit at distance ranks 263-268"+ (is (= 0.4 (recall-at 10 ideal-top (query-ids! :no-index)))))+ (build-hnsw-index!)+ (testing "indexed retrieval is stuck at the same recall: iterating does not fix limit truncation"+ ;; the filters are non-selective here, so every variant's pool is the same exact top-20+ (doseq [variant [:hnsw :hnsw-iterative-relaxed :hnsw-iterative-strict]]+ (testing variant+ (is (= 0.4 (recall-at 10 ideal-top (query-ids! variant))))))))))+ (testing "under a selective filter the indexed iterative variants are strictly better end-to-end"+ (semantic.tu/with-only-semantic-weights+ (let [ideal-rare (mt/with-temporary-setting-values [semantic-search-results-limit 1000]+ (query-ids! :brute-force :verified true))]+ (is (= 0.0 (recall-at 10 ideal-rare (query-ids! :no-index :verified true))))+ (is (= 0.0 (recall-at 10 ideal-rare (query-ids! :hnsw :verified true))))+ (doseq [variant [:hnsw-iterative-relaxed :hnsw-iterative-strict]]+ (testing variant+ (is (= 1.0 (recall-at 10 ideal-rare (query-ids! variant :verified true)))))))))))))++;;;; layer 3: under-fetch from post-retrieval permission filtering++(deftest permission-under-fetch-matrix-test+ (do-with-matrix-fixture!+ (fn [_ds]+ (semantic.tu/with-only-semantic-weights+ (let [run! (fn [variant user & {:as extra-ctx}]+ (mt/with-test-user user+ (let [{:keys [results raw-count]} (query-results! variant+ (merge {:current-user-id (mt/user->id user)}+ extra-ctx))]+ {:raw-count raw-count :returned (count results)})))]+ (testing "exact variants: the restricted band fills 8 of the 20 pool slots, then gets dropped"+ (doseq [variant [:no-index :brute-force]]+ (testing variant+ (is (= {:raw-count 20 :returned 12} (run! variant :rasta))))))+ (build-hnsw-index!)+ (testing "indexed variants under-fetch identically; iterating cannot help (the check runs in Clojure)"+ (doseq [variant [:hnsw :hnsw-iterative-relaxed :hnsw-iterative-strict]]+ (testing variant+ (is (= {:raw-count 20 :returned 12} (run! variant :rasta))))))+ (testing "an admin sees the full pool, pinning the loss on permissions rather than retrieval"+ (is (= {:raw-count 20 :returned 20} (run! :hnsw :crowberto))))+ ;; The half-selective created-by filter (odd ids -> crowberto) keeps 10 of the exact top-20+ ;; (4 restricted + 6 core). Naive :hnsw post-filters that fixed pool, so it arrives at the+ ;; permission check already halved; filter-first and iterative scans refill to a full pool of 20+ ;; (4 restricted + 16 core). Structural, not graph luck: the forced top-20 is exact at this scale.+ ;; (At scale, graph approximation can shift this either way -- missing nearby restricted docs+ ;; lessens the permission drop, missing nearby permitted docs worsens it -- so only the SQL-stage+ ;; effect is pinned here.)+ (testing "a half-selective filter compounds with permissions for the post-filtering strategy only"+ (let [created-by [(mt/user->id :crowberto)]]+ (testing "naive :hnsw under-fetches at the SQL stage, then again at the permission stage"+ (is (= {:raw-count 10 :returned 6} (run! :hnsw :rasta :created-by created-by))))+ (testing "filter-first and iterative variants under-fetch only at the permission stage"+ (doseq [variant [:brute-force :hnsw-iterative-relaxed :hnsw-iterative-strict]]+ (testing variant+ (is (= {:raw-count 20 :returned 16} (run! variant :rasta :created-by created-by)))))))))))))++;;;; graph approximation: a 32-d densely packed dataset where HNSW genuinely misses++;; a separate 32-d index because graph approximation error needs dimensionality, not tree size: a 4-d graph+;; stays exact even with thousands of packed fillers+(def ^:private packed-index+ "The index for [[graph-approximation-test]]'s 32-d packed dataset."+ (semantic.index/default-index {:provider "mock" :model-name "packed" :vector-dimensions 32}+ :table-name "index_table_packed_test"))++(defn- make-packed-dataset+ "`n` filler cards packed at near-identical distances from the probe, so the true top-20 differ from any+ approximate top-20 by margins the graph cannot resolve.+ Doc ids are assigned in ascending-distance order, so the exact top-k is simply ids 1..k."+ [{:keys [seed n]}]+ (let [rng (Random. (long seed))+ rows (mapv (fn [i]+ {:id (inc i)+ :name (format "packed card %04d" (inc i))+ :vector (vector-at-distance rng 32 (+ 0.10 (* 2e-5 i)))})+ (range n))]+ {:docs (vec (for [{:keys [id name]} rows]+ {:model "card"+ :id id+ :name name+ :searchable_text name+ :embeddable_text name+ :created_at #t "2025-01-01T12:00:00Z"+ :creator_id (mt/user->id :rasta)+ :archived false+ :legacy_input {:id id :model "card" :name name}+ :metadata {:title name}}))+ :embeddings (into {probe-text (into [1.0] (repeat 31 0.0))} (map (juxt :name :vector)) rows)+ :exact-ids (mapv :id rows)}))++(deftest graph-approximation-test+ (mt/with-premium-features #{:semantic-search}+ (mt/with-temporary-setting-values [semantic-search-results-limit results-limit]+ (let [ds (make-packed-dataset {:seed 42 :n 2000})]+ (semantic.tu/with-mock-embeddings (:embeddings ds)+ (semantic.tu/with-test-db! {}+ (binding [*index* packed-index]+ (semantic.index/create-index-table-if-not-exists! (semantic.env/get-pgvector-datasource!)+ packed-index)+ (drop-hnsw-index!)+ (semantic.tu/upsert-index! (:docs ds) :index packed-index :serial? true)+ (jdbc/execute! (semantic.env/get-pgvector-datasource!)+ [(str "ANALYZE " (:table-name packed-index))])+ (semantic.tu/with-only-semantic-weights+ (mt/with-dynamic-fn-redefs [semantic.index/filter-read-permitted identity]+ (let [exact-top (take 20 (:exact-ids ds))+ recall20 (fn [variant & {:as extra-ctx}]+ (recall-at 20 exact-top (query-ids! variant extra-ctx)))]+ (testing "without an index, retrieval stays exact even at 32 dims"+ (is (= exact-top (query-ids! :no-index))))+ (build-hnsw-index!)+ ;; the graph differs every build, so these cells assert bands; each edge sits several+ ;; misses beyond everything observed across dozens of independently built graphs+ ;; (the observed range is noted per cell) -- a value at 1.0 or near 0 means the cell broke+ (testing "naive :hnsw genuinely misses: recall@20 lands in a band strictly below 1"+ ;; observed 0.5-0.85+ (is (<= 0.3 (recall20 :hnsw) 0.95)))+ (testing "iterative scans with ef_search 16 (below the limit of 20) miss more"+ ;; observed 0.35-0.6+ (is (<= 0.15 (recall20 :hnsw-iterative-relaxed) 0.85))+ ;; observed 0.15-0.55, below relaxed on every graph measured: strict_order discards+ ;; more per visit in exchange for emission order, which the outer sort makes worthless+ ;; in this pipeline (the gap can shrink to one miss, so no cross-assertion)+ (is (<= 0.05 (recall20 :hnsw-iterative-strict) 0.8)))+ (testing "a saturating ef_search dials approximation away again, on the same graph"+ (is (= exact-top (query-ids! :hnsw-iterative-relaxed+ :vector-search-ef-search 200))))))))))))))diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex aa2784f4ead..0ceb2b8dd11 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -400,6 +400,19 @@(prometheus/counter :metabase-search/semantic-db-query-ms{:description "Total number of ms spent querying the search index":labels [:embedding-model]})+ ;; the four semantic-vector-* diagnostics are emitted only while the semantic-search-explain setting is on+ (prometheus/counter :metabase-search/semantic-vector-inner-ms+ {:description "Total ms spent in the inner vector subquery (diagnostic, off by default)"+ :labels [:strategy]})+ (prometheus/counter :metabase-search/semantic-vector-tuples-scanned+ {:description "Total tuples the vector scan visited, returned plus filter-removed (diagnostic, off by default)"+ :labels [:strategy]})+ (prometheus/counter :metabase-search/semantic-prefilter-pool-size+ {:description "Total rows a filter-first (brute-force) scan would have computed distances over (diagnostic, off by default)"+ :labels [:strategy]})+ (prometheus/counter :metabase-search/semantic-vector-scan-used-index+ {:description "Count of vector searches by strategy and the plan node chosen for the index-table scan (diagnostic, off by default)"+ :labels [:strategy :plan-node]})(prometheus/counter :metabase-search/semantic-appdb-scores-ms{:description "Total number of ms spent adding appdb-based scores"})(prometheus/counter :metabase-search/semantic-fallback-triggereddiff --git a/src/metabase/search/api.clj b/src/metabase/search/api.cljindex 7ad0d6fa6d5..cae6faa6021 100644--- a/src/metabase/search/api.clj+++ b/src/metabase/search/api.clj@@ -179,6 +179,11 @@[:model_ancestors {:default false} [:maybe :boolean]][:search_engine {:optional true} [:maybe string?]][:vector_search_strategy {:optional true} [:maybe (into [:enum] (map name) search.config/vector-search-strategies)]]+ ;; bounded to pgvector's GUC ranges so out-of-range values get a 400 instead of erroring the+ ;; SET LOCAL mid-transaction (a 500)+ [:vector_search_ef_search {:optional true} [:maybe [:int {:min 1 :max 1000}]]]+ [:vector_search_max_scan_tuples {:optional true} [:maybe [:int {:min 1 :max 2147483647}]]]+ [:vector_search_explain {:optional true} [:maybe :boolean]][:search_native_query {:optional true} [:maybe :boolean]][:verified {:optional true} [:maybe true?]][:ids {:optional true} [:maybe (ms/QueryVectorOf ms/PositiveInt)]]@@ -209,6 +214,9 @@model-ancestors :model_ancestorssearch-engine :search_enginevector-search-strategy :vector_search_strategy+ vector-search-ef-search :vector_search_ef_search+ vector-search-max-scan-tuples :vector_search_max_scan_tuples+ vector-search-explain :vector_search_explainsearch-native-query :search_native_querytable-db-id :table_db_idinclude-metadata :include_metadata@@ -234,6 +242,9 @@:offset (request/offset):search-engine search-engine:vector-search-strategy vector-search-strategy+ :vector-search-ef-search vector-search-ef-search+ :vector-search-max-scan-tuples vector-search-max-scan-tuples+ :vector-search-explain? vector-search-explain:search-native-query search-native-query:search-string (some-> q str/trim not-empty):table-db-id table-db-id@@ -265,7 +276,12 @@- `last_edited_at`: search for items last edited at a specific timestamp- `last_edited_by`: search for items last edited by a specific user- `search_native_query`: set to true to search the content of native queries- - `vector_search_strategy`: for the semantic engine, `hnsw` (approximate index search, default) or `brute-force` (exact filter-first search); ignored by other engines+ - `vector_search_strategy`: for the semantic engine: `hnsw` (approximate index search, default),+ `brute-force` (exact filter-first search), or `hnsw-iterative-relaxed` / `hnsw-iterative-strict`+ (index-backed iterative scans with inline filters); ignored by other engines+ - `vector_search_ef_search`: override pgvector's `hnsw.ef_search` (1-1000) for the iterative strategies; (admin only)+ - `vector_search_max_scan_tuples`: override pgvector's `hnsw.max_scan_tuples` for the iterative strategies; (admin only)+ - `vector_search_explain`: set to true to record vector-scan instrumentation for this search (expensive); (admin only)- `verified`: set to true to search for verified items only (requires Content Management or Official Collections premium feature)- `ids`: search for items with those ids, works iff single value passed to `models`- `display_type`: search for cards/models with specific display types@@ -282,6 +298,12 @@[_route-paramsquery-params :- search-request-schema](api/check-valid-page-params (request/limit) (request/offset))+ ;; tuning/diagnostic knobs are admin-only: explain re-executes the vector scan and counts the whole index+ ;; table per request, and the scan-budget knobs let a request inflate its own cost+ (when (or (:vector_search_ef_search query-params)+ (:vector_search_max_scan_tuples query-params)+ (some? (:vector_search_explain query-params)))+ (api/check-superuser))(try(u/prog1 (search/search (params->search-context query-params))(analytics/inc! :metabase-search/response-ok)diff --git a/src/metabase/search/config.clj b/src/metabase/search/config.cljindex 8af1d7d5766..f0c5f1f90b9 100644--- a/src/metabase/search/config.clj+++ b/src/metabase/search/config.clj@@ -317,8 +317,24 @@"Valid semantic-search vector-search strategies, as keywords. Mastered here (rather than in the EE module)so the OSS search API param and the EE semantic-search setting validation share one definition.Note: the per-strategy dispatch in [[metabase-enterprise.semantic-search.index/semantic-search-query]] is a- separate `case` (with an `:hnsw` default) and must be updated by hand when adding a strategy."- [:hnsw :brute-force])+ separate dispatch (unknown strategies fall back to `:brute-force`) and must be updated by hand when+ adding a strategy.++ - `:hnsw` approximate, index-backed, post-filters the candidate set+ - `:brute-force` exact, filter-first full scan (skips the index)+ - `:hnsw-iterative-relaxed` index-backed iterative scan, filters inline, results in approximate distance+ order (pgvector `hnsw.iterative_scan = relaxed_order`)+ - `:hnsw-iterative-strict` as above but exact distance order (`hnsw.iterative_scan = strict_order`)++ The iterative scan keeps pulling neighbours until the limit is met or `hnsw.max_scan_tuples` is hit; the+ `ef-search` and `max-scan-tuples` knobs tune it further."+ [:hnsw :brute-force :hnsw-iterative-relaxed :hnsw-iterative-strict])++(def hnsw-index-backed-strategies+ "The subset of [[vector-search-strategies]] that read the HNSW index. Selecting one of these must build the+ index just-in-time, and a query under one must fail fast when the index is missing. `:brute-force` is the+ only strategy that needs no index."+ #{:hnsw :hnsw-iterative-relaxed :hnsw-iterative-strict})(def ^:private ui-contexts"Search `context` values issued by the frontend, one per UI surface.@@ -411,9 +427,20 @@[:models [:set SearchableModel]];; TODO this is optional only for tests, clean those up![:search-engine {:optional true} keyword?]- ;; Semantic-engine vector-search strategy (:hnsw or :brute-force). When absent, the engine uses its- ;; configured default setting.- [:vector-search-strategy {:optional true} [:maybe keyword?]]+ ;; Semantic-engine vector-search strategy (see [[search.config/vector-search-strategies]]). When absent,+ ;; the engine uses its configured default setting. The remaining vector-search-* knobs are experimental+ ;; tuning parameters; they only affect the `:hnsw-iterative-*` strategies (except `explain?`, which works+ ;; for any strategy). Each falls back to its EE setting default when absent.+ [:vector-search-strategy {:optional true} [:maybe keyword?]]+ ;; pgvector `hnsw.ef_search` -- HNSW candidate-list size+ [:vector-search-ef-search {:optional true} [:maybe pos-int?]]+ ;; pgvector `hnsw.max_scan_tuples` -- soft cap on tuples an iterative scan visits+ [:vector-search-max-scan-tuples {:optional true} [:maybe pos-int?]]+ ;; true to run gated EXPLAIN (ANALYZE) instrumentation of the inner vector subquery (expensive)+ [:vector-search-explain? {:optional true} [:maybe :boolean]]+ ;; true to `SET LOCAL enable_seqscan = off` to force the planner onto the HNSW index (experiment knob;+ ;; deliberately not exposed over HTTP)+ [:vector-search-force-index? {:optional true} [:maybe :boolean]][:search-string {:optional true} [:maybe ms/NonBlankString]][:weights {:optional true} [:maybe [:map-of :keyword number?]]];;diff --git a/src/metabase/search/impl.clj b/src/metabase/search/impl.cljindex 812be93eb3d..35b2bc21516 100644--- a/src/metabase/search/impl.clj+++ b/src/metabase/search/impl.clj@@ -273,6 +273,10 @@[:table-db-id {:optional true} [:maybe ms/PositiveInt]][:search-engine {:optional true} [:maybe string?]][:vector-search-strategy {:optional true} [:maybe string?]]+ [:vector-search-ef-search {:optional true} [:maybe ms/PositiveInt]]+ [:vector-search-max-scan-tuples {:optional true} [:maybe ms/PositiveInt]]+ [:vector-search-explain? {:optional true} [:maybe boolean?]]+ [:vector-search-force-index? {:optional true} [:maybe boolean?]][:search-native-query {:optional true} [:maybe boolean?]][:model-ancestors? {:optional true} [:maybe boolean?]][:verified {:optional true} [:maybe true?]]@@ -313,6 +317,10 @@offsetsearch-enginevector-search-strategy+ vector-search-ef-search+ vector-search-max-scan-tuples+ vector-search-explain?+ vector-search-force-index?search-native-querysearch-stringtable-db-id@@ -358,6 +366,10 @@(some? limit) (assoc :limit-int limit)(some? offset) (assoc :offset-int offset)(not (str/blank? vector-search-strategy)) (assoc :vector-search-strategy (keyword vector-search-strategy))+ (some? vector-search-ef-search) (assoc :vector-search-ef-search vector-search-ef-search)+ (some? vector-search-max-scan-tuples) (assoc :vector-search-max-scan-tuples vector-search-max-scan-tuples)+ (some? vector-search-explain?) (assoc :vector-search-explain? vector-search-explain?)+ (some? vector-search-force-index?) (assoc :vector-search-force-index? vector-search-force-index?)(some? search-native-query) (assoc :search-native-query search-native-query)(some? verified) (assoc :verified verified)(some? curated) (assoc :curated? curated)diff --git a/test/metabase/search/api_test.clj b/test/metabase/search/api_test.cljindex 93d31ace555..42b18aa4554 100644--- a/test/metabase/search/api_test.clj+++ b/test/metabase/search/api_test.clj@@ -334,6 +334,38 @@(is (=? {:engine string?}(mt/user-http-request :crowberto :get 200 "search" :q "x" :context "search-app"))))))+(deftest vector-search-knobs-test+ (testing "the vector-search tuning/diagnostic knobs are admin-only"+ (doseq [[param value] {:vector_search_ef_search 100+ :vector_search_max_scan_tuples 50000+ :vector_search_explain true}]+ (testing param+ (is (= "You don't have permissions to do that."+ (mt/user-http-request :rasta :get 403 "search" :q "x" param value))))))+ (testing "an admin's knobs thread through into the search context"+ (let [captured (atom nil)]+ (mt/with-dynamic-fn-redefs [search/search (fn [search-ctx]+ (reset! captured search-ctx)+ {:data [] :total 0 :engine "test"})]+ (mt/user-http-request :crowberto :get 200 "search" :q "x"+ :vector_search_strategy "hnsw-iterative-strict"+ :vector_search_ef_search 100+ :vector_search_max_scan_tuples 50000+ :vector_search_explain true)+ (is (=? {:vector-search-strategy :hnsw-iterative-strict+ :vector-search-ef-search 100+ :vector-search-max-scan-tuples 50000+ :vector-search-explain? true}+ @captured))+ (testing "absent knobs stay absent, so the semantic engine falls back to its setting defaults"+ (mt/user-http-request :crowberto :get 200 "search" :q "x")+ (is (not-any? @captured [:vector-search-strategy :vector-search-ef-search+ :vector-search-max-scan-tuples :vector-search-explain?+ :vector-search-force-index?]))))))+ (testing "values outside pgvector's GUC ranges are rejected up front"+ (is (=? {:errors {:vector_search_ef_search some?}}+ (mt/user-http-request :crowberto :get 400 "search" :q "x" :vector_search_ef_search 2000)))))+(deftest basic-test(testing "Basic search, should find 1 of each entity type, all items in the root collection"(with-search-items-in-root-collection "test"diff --git a/test/metabase/search/impl_test.clj b/test/metabase/search/impl_test.cljindex 7f0929959d9..e19b758c8af 100644--- a/test/metabase/search/impl_test.clj+++ b/test/metabase/search/impl_test.clj@@ -36,6 +36,21 @@(testing "Subclasses"(is (= :search.engine/hybrid (#'search.impl/parse-engine "hybrid"))))))+(deftest vector-search-knobs-absent-test+ ;; threading and gating of the knobs is covered end-to-end in [[metabase.search.api-test]]; this pins the+ ;; contract that search-context must never DEFAULT them -- absence is what lets the semantic engine fall+ ;; back to its setting-backed defaults, and the schema's default-value transformer makes that easy to+ ;; break by adding an innocent-looking :default+ (let [ctx (search.impl/search-context {:current-user-id 1+ :current-user-perms #{"/"}+ :is-superuser? true+ :is-impersonated-user? false+ :is-sandboxed-user? false+ :models nil+ :search-string "x"})]+ (is (not-any? ctx [:vector-search-strategy :vector-search-ef-search :vector-search-max-scan-tuples+ :vector-search-explain? :vector-search-force-index?]))))+(deftest ^:parallel order-clause-test(testing "it includes all columns and normalizes the query"(is (= [[:case