Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Address minor tech debt in Search code (#76240)

Chris Truterb57bcab6f082
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
index 128d0377869..2dce56af79d 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
@@ -45,10 +45,12 @@
(some? semantic.db.datasource/db-url)
(semantic.settings/semantic-search-enabled)))
-(defn- with-zero-semantic-distance
- "Record `:semantic-distance` 0 on `results` that lack it, for a consistent merged-result score breakdown."
+(defn- with-zero-semantic-distance-score
+ "Record a 0 `:semantic-distance` score on `results` that lack it, for a consistent merged-result score breakdown."
[search-ctx results]
;; Fallback (appdb/in-place) results never went through vector search, so they carry no semantic distance.
+ ;; Score them as least-relevant: the 0 below is a score, not a distance -- the opposite of a zero cosine
+ ;; distance, which would be a perfect match.
(let [entry {:name :semantic-distance
:score 0
:weight (search.config/weight search-ctx :semantic-distance)
@@ -101,7 +103,7 @@
[]))
fallback-results (take total-limit fallback-results)
_ (analytics/observe! :metabase-search/semantic-fallback-results-usage (count fallback-results))
- combined-results (concat results (with-zero-semantic-distance search-ctx fallback-results))
+ combined-results (concat results (with-zero-semantic-distance-score search-ctx fallback-results))
deduped-results (m/distinct-by (juxt :model :id) combined-results)]
(take total-limit deduped-results)))))
(catch Exception e
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
index 7a6c7bda3f2..6ca731e0025 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
@@ -641,16 +641,20 @@
(defn- hnsw-search-query
"Build the semantic vector subquery using the HNSW index, applying `filters` after candidate selection."
[index embedding-literal filters]
- ;; The inner `vector_candidates` CTE is a pure vector search (ORDER BY distance LIMIT) so the planner
- ;; uses the HNSW index. The filters only run in the outer query, so this is approximate: when the
- ;; globally-closest rows are dominated by a cluster the filters reject, slightly-further survivors that
- ;; would have passed the filters never enter the candidate set. `brute-force-search-query` is exact.
+ ;; The inner `vector_candidates` CTE is a pure vector search (ORDER BY distance LIMIT) so the planner uses
+ ;; the HNSW index. HNSW is an approximate-nearest-neighbour index, so its results are approximate regardless
+ ;; of filtering -- that's the trade-off we accept for its speed. Running the filters only in the outer query
+ ;; adds a separate problem: under-fetch. The fixed-size candidate set is picked before the filters apply, so
+ ;; when the globally-closest rows are dominated by a cluster the filters reject, we can return fewer rows than
+ ;; asked for -- or none at all. `brute-force-search-query` sidesteps both by skipping the index and scanning
+ ;; every filtered row.
;; TODO: only pull in necessary extra columns from configured filters
(let [hnsw-query {:select (into common-search-columns
[[[:raw (str "embedding <=> " embedding-literal)] :distance]])
:from [(keyword (:table-name index))]
:order-by [[[:raw (str "embedding <=> " embedding-literal)] :asc]]
:limit (semantic-settings/semantic-search-results-limit)}
+ ;; `semantic_rank` feeds the RRF scorer; `semantic_distance` feeds the semantic-distance scorer.
base-query {:with [[:vector_candidates hnsw-query]]
:select (into common-search-columns
[[[:raw "row_number() OVER (ORDER BY distance ASC)"] :semantic_rank]
@@ -696,10 +700,13 @@
`:brute-force` is exact and filter-first; `:hnsw` (the default) is approximate and index-backed."
[index embedding search-context]
(let [filters (search-filters search-context)
- embedding-literal (format-embedding embedding)]
- (case (vector-search-strategy search-context)
+ embedding-literal (format-embedding embedding)
+ strategy (vector-search-strategy search-context)]
+ (case strategy
:brute-force (brute-force-search-query index embedding-literal filters)
- (hnsw-search-query index embedding-literal filters))))
+ :hnsw (hnsw-search-query index embedding-literal filters)
+ (do (log/warnf "Unknown vector-search strategy %s; falling back to :hnsw" (pr-str strategy))
+ (hnsw-search-query index embedding-literal filters)))))
(defn- flatten-ctes
"Flatten nested :with clauses into a single top-level :with.
@@ -711,10 +718,11 @@
(if-not (:with query)
{:ctes [] :query query}
(let [ctes (reduce
- ;; `opts` carries any trailing CTE options (e.g. `:materialized`) so they survive hoisting.
- (fn [acc [cte-name cte-query & opts]]
+ ;; `assoc` swaps the (possibly flattened) inner query back into the binding while preserving the
+ ;; CTE name and any trailing opts (e.g. `:materialized`), so they survive hoisting.
+ (fn [acc [_cte-name cte-query & _opts :as cte-binding]]
(let [{:keys [ctes query]} (flatten-ctes cte-query)]
- (into acc (conj ctes (into [cte-name query] opts)))))
+ (into acc (conj ctes (assoc cte-binding 1 query)))))
[]
(:with query))]
{:ctes ctes
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj
index ec7a06a6b2a..0bc217d8461 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj
@@ -77,9 +77,8 @@
"Map a cosine `distance` (pgvector `<=>`, range [0, 2]) to a [0, 1] score.
Linear for now: distance 0 scores 1, the maximum distance of 2 scores 0."
[distance]
- ;; TODO (Chris 2026-06-02) -- consider rescaling against the retrieval cutoff (index/max-cosine-distance) so the
- ;; [0, cutoff] band spans the full [0, 1], and/or a non-linear curve. For now keep the raw distance as transparent
- ;; as possible.
+ ;; TODO (Chris 2026-06-22) -- a non-linear curve might rank better than this straight-line map. Keeping it
+ ;; linear for now so the score stays as transparent as possible.
[:- [:inline 1] [:/ distance [:inline cosine-distance-ceiling]]])
(defn base-scorers
@@ -90,8 +89,9 @@
;; NOTE: we calculate scores even if the weight is zero, so that it's easy to consider how we could affect any
;; given set of results. At some point, we should optimize away the irrelevant scores for any given context.
{:rrf rrf-rank-exp
- ;; Keyword-only hits have no vector distance, so treat them as maximally distant (score 0).
- :semantic-distance (semantic-distance-score-expr [:coalesce :semantic_distance [:inline cosine-distance-ceiling]])
+ ;; Keyword-only hits have no vector distance; score them 0 directly rather than feeding the ceiling
+ ;; distance through the linear map.
+ :semantic-distance [:coalesce (semantic-distance-score-expr :semantic_distance) [:inline 0]]
:view-count (view-count-expr index-table search.config/view-count-scaling-percentile)
:pinned (search.scoring/truthy :pinned)
:recency (search.scoring/inverse-duration
diff --git a/test/metabase/search/appdb/scoring_test.clj b/test/metabase/search/appdb/scoring_test.clj
index da2c42c600e..a6b9d1dc911 100644
--- a/test/metabase/search/appdb/scoring_test.clj
+++ b/test/metabase/search/appdb/scoring_test.clj
@@ -131,15 +131,10 @@
(with-index-contents
[{:model "card" :id 1 :name "Sales, Revenue"}
{:model "card" :id 2 :name "Sales Revenue Report"}]
- (case (mdb/db-type)
- :postgres
- (testing "Exact matching ignores commas and collapses whitespace runs"
- (is (= [["card" 1 "Sales, Revenue"]
- ["card" 2 "Sales Revenue Report"]]
- (search-results :exact "sales revenue"))))
- :h2
- ;; TODO text ranking (probably in-memory)
- nil)))
+ (testing "Exact matching ignores commas and collapses whitespace runs"
+ (is (= [["card" 1 "Sales, Revenue"]
+ ["card" 2 "Sales Revenue Report"]]
+ (search-results :exact "sales revenue"))))))
(deftest ^:parallel prefix-test
(with-index-contents
@@ -152,10 +147,12 @@
(deftest ^:parallel prefix-normalization-test
(with-index-contents
- [{:model "card" :id 1 :name "Sales, Revenue Quarterly"}
+ ;; The whitespace run sits inside the matched prefix ("Sales, Revenue"), so the LIKE only matches
+ ;; "sales revenue%" once commas are dropped and the run is collapsed -- a stray double space would miss.
+ [{:model "card" :id 1 :name "Sales, Revenue Quarterly"}
{:model "card" :id 2 :name "Revenue and Sales"}]
(testing "Prefix matching ignores commas and collapses whitespace runs"
- (is (= [["card" 1 "Sales, Revenue Quarterly"]
+ (is (= [["card" 1 "Sales, Revenue Quarterly"]
["card" 2 "Revenue and Sales"]]
(search-results :prefix "sales revenue"))))))