The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Address minor tech debt in Search code (#76240)
Chris Truterb57bcab6f082
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/core.cljindex 128d0377869..2dce56af79d 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj@@ -45,10 +45,12 @@(some? semantic.db.datasource/db-url)(semantic.settings/semantic-search-enabled)))-(defn- with-zero-semantic-distance- "Record `:semantic-distance` 0 on `results` that lack it, for a consistent merged-result score breakdown."+(defn- with-zero-semantic-distance-score+ "Record a 0 `:semantic-distance` score on `results` that lack it, for a consistent merged-result score breakdown."[search-ctx results];; Fallback (appdb/in-place) results never went through vector search, so they carry no semantic distance.+ ;; Score them as least-relevant: the 0 below is a score, not a distance -- the opposite of a zero cosine+ ;; distance, which would be a perfect match.(let [entry {:name :semantic-distance:score 0:weight (search.config/weight search-ctx :semantic-distance)@@ -101,7 +103,7 @@[]))fallback-results (take total-limit fallback-results)_ (analytics/observe! :metabase-search/semantic-fallback-results-usage (count fallback-results))- combined-results (concat results (with-zero-semantic-distance search-ctx fallback-results))+ combined-results (concat results (with-zero-semantic-distance-score search-ctx fallback-results))deduped-results (m/distinct-by (juxt :model :id) combined-results)](take total-limit deduped-results)))))(catch Exception ediff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.cljindex 7a6c7bda3f2..6ca731e0025 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj@@ -641,16 +641,20 @@(defn- hnsw-search-query"Build the semantic vector subquery using the HNSW index, applying `filters` after candidate selection."[index embedding-literal filters]- ;; The inner `vector_candidates` CTE is a pure vector search (ORDER BY distance LIMIT) so the planner- ;; uses the HNSW index. The filters only run in the outer query, so this is approximate: when the- ;; globally-closest rows are dominated by a cluster the filters reject, slightly-further survivors that- ;; would have passed the filters never enter the candidate set. `brute-force-search-query` is exact.+ ;; The inner `vector_candidates` CTE is a pure vector search (ORDER BY distance LIMIT) so the planner uses+ ;; the HNSW index. HNSW is an approximate-nearest-neighbour index, so its results are approximate regardless+ ;; of filtering -- that's the trade-off we accept for its speed. Running the filters only in the outer query+ ;; adds a separate problem: under-fetch. The fixed-size candidate set is picked before the filters apply, so+ ;; when the globally-closest rows are dominated by a cluster the filters reject, we can return fewer rows than+ ;; asked for -- or none at all. `brute-force-search-query` sidesteps both by skipping the index and scanning+ ;; every filtered row.;; TODO: only pull in necessary extra columns from configured filters(let [hnsw-query {:select (into common-search-columns[[[:raw (str "embedding <=> " embedding-literal)] :distance]]):from [(keyword (:table-name index))]:order-by [[[:raw (str "embedding <=> " embedding-literal)] :asc]]:limit (semantic-settings/semantic-search-results-limit)}+ ;; `semantic_rank` feeds the RRF scorer; `semantic_distance` feeds the semantic-distance scorer.base-query {:with [[:vector_candidates hnsw-query]]:select (into common-search-columns[[[:raw "row_number() OVER (ORDER BY distance ASC)"] :semantic_rank]@@ -696,10 +700,13 @@`:brute-force` is exact and filter-first; `:hnsw` (the default) is approximate and index-backed."[index embedding search-context](let [filters (search-filters search-context)- embedding-literal (format-embedding embedding)]- (case (vector-search-strategy search-context)+ embedding-literal (format-embedding embedding)+ strategy (vector-search-strategy search-context)]+ (case strategy:brute-force (brute-force-search-query index embedding-literal filters)- (hnsw-search-query index embedding-literal filters))))+ :hnsw (hnsw-search-query index embedding-literal filters)+ (do (log/warnf "Unknown vector-search strategy %s; falling back to :hnsw" (pr-str strategy))+ (hnsw-search-query index embedding-literal filters)))))(defn- flatten-ctes"Flatten nested :with clauses into a single top-level :with.@@ -711,10 +718,11 @@(if-not (:with query){:ctes [] :query query}(let [ctes (reduce- ;; `opts` carries any trailing CTE options (e.g. `:materialized`) so they survive hoisting.- (fn [acc [cte-name cte-query & opts]]+ ;; `assoc` swaps the (possibly flattened) inner query back into the binding while preserving the+ ;; CTE name and any trailing opts (e.g. `:materialized`), so they survive hoisting.+ (fn [acc [_cte-name cte-query & _opts :as cte-binding]](let [{:keys [ctes query]} (flatten-ctes cte-query)]- (into acc (conj ctes (into [cte-name query] opts)))))+ (into acc (conj ctes (assoc cte-binding 1 query)))))[](:with query))]{:ctes ctesdiff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.cljindex ec7a06a6b2a..0bc217d8461 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj@@ -77,9 +77,8 @@"Map a cosine `distance` (pgvector `<=>`, range [0, 2]) to a [0, 1] score.Linear for now: distance 0 scores 1, the maximum distance of 2 scores 0."[distance]- ;; TODO (Chris 2026-06-02) -- consider rescaling against the retrieval cutoff (index/max-cosine-distance) so the- ;; [0, cutoff] band spans the full [0, 1], and/or a non-linear curve. For now keep the raw distance as transparent- ;; as possible.+ ;; TODO (Chris 2026-06-22) -- a non-linear curve might rank better than this straight-line map. Keeping it+ ;; linear for now so the score stays as transparent as possible.[:- [:inline 1] [:/ distance [:inline cosine-distance-ceiling]]])(defn base-scorers@@ -90,8 +89,9 @@;; NOTE: we calculate scores even if the weight is zero, so that it's easy to consider how we could affect any;; given set of results. At some point, we should optimize away the irrelevant scores for any given context.{:rrf rrf-rank-exp- ;; Keyword-only hits have no vector distance, so treat them as maximally distant (score 0).- :semantic-distance (semantic-distance-score-expr [:coalesce :semantic_distance [:inline cosine-distance-ceiling]])+ ;; Keyword-only hits have no vector distance; score them 0 directly rather than feeding the ceiling+ ;; distance through the linear map.+ :semantic-distance [:coalesce (semantic-distance-score-expr :semantic_distance) [:inline 0]]:view-count (view-count-expr index-table search.config/view-count-scaling-percentile):pinned (search.scoring/truthy :pinned):recency (search.scoring/inverse-durationdiff --git a/test/metabase/search/appdb/scoring_test.clj b/test/metabase/search/appdb/scoring_test.cljindex da2c42c600e..a6b9d1dc911 100644--- a/test/metabase/search/appdb/scoring_test.clj+++ b/test/metabase/search/appdb/scoring_test.clj@@ -131,15 +131,10 @@(with-index-contents[{:model "card" :id 1 :name "Sales, Revenue"}{:model "card" :id 2 :name "Sales Revenue Report"}]- (case (mdb/db-type)- :postgres- (testing "Exact matching ignores commas and collapses whitespace runs"- (is (= [["card" 1 "Sales, Revenue"]- ["card" 2 "Sales Revenue Report"]]- (search-results :exact "sales revenue"))))- :h2- ;; TODO text ranking (probably in-memory)- nil)))+ (testing "Exact matching ignores commas and collapses whitespace runs"+ (is (= [["card" 1 "Sales, Revenue"]+ ["card" 2 "Sales Revenue Report"]]+ (search-results :exact "sales revenue"))))))(deftest ^:parallel prefix-test(with-index-contents@@ -152,10 +147,12 @@(deftest ^:parallel prefix-normalization-test(with-index-contents- [{:model "card" :id 1 :name "Sales, Revenue Quarterly"}+ ;; The whitespace run sits inside the matched prefix ("Sales, Revenue"), so the LIKE only matches+ ;; "sales revenue%" once commas are dropped and the run is collapsed -- a stray double space would miss.+ [{:model "card" :id 1 :name "Sales, Revenue Quarterly"}{:model "card" :id 2 :name "Revenue and Sales"}](testing "Prefix matching ignores commas and collapses whitespace runs"- (is (= [["card" 1 "Sales, Revenue Quarterly"]+ (is (= [["card" 1 "Sales, Revenue Quarterly"]["card" 2 "Revenue and Sales"]](search-results :prefix "sales revenue"))))))