The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add semantic-distance scorer for semantic search (#75128)
Chris Truter814398c83511
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/core.cljindex c6a2d79746c..9454883386d 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj@@ -13,6 +13,7 @@[metabase-enterprise.semantic-search.settings :as semantic.settings][metabase.analytics-interface.core :as analytics][metabase.premium-features.core :refer [defenterprise]]+ [metabase.search.config :as search.config][metabase.search.engine :as search.engine][metabase.tracing.core :as tracing][metabase.util :as u]@@ -44,6 +45,21 @@(some? semantic.db.datasource/db-url)(semantic.settings/semantic-search-enabled)))+(defn- with-zero-semantic-distance+ "Record `:semantic-distance` 0 on `results` that lack it, for a consistent merged-result score breakdown."+ [search-ctx results]+ ;; Fallback (appdb/in-place) results never went through vector search, so they carry no semantic distance.+ (let [entry {:name :semantic-distance+ :score 0+ :weight (search.config/weight search-ctx :semantic-distance)+ :contribution 0}]+ (map (fn [result]+ (cond-> result+ (and (contains? result :all-scores)+ (not-any? (comp #{:semantic-distance} :name) (:all-scores result)))+ (update :all-scores conj entry)))+ results)))+(defenterprise results"Enterprise implementation of semantic search results with improved fallback logic. Falls back to appdb search onlywhen semantic search returns too few results and some results were filtered out (e.g. due to permission checks)."@@ -85,7 +101,7 @@[]))fallback-results (take total-limit fallback-results)_ (analytics/observe! :metabase-search/semantic-fallback-results-usage (count fallback-results))- combined-results (concat results fallback-results)+ combined-results (concat results (with-zero-semantic-distance search-ctx fallback-results))deduped-results (m/distinct-by (juxt :model :id) combined-results)](take total-limit deduped-results)))))(catch Exception ediff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.cljindex fb1862e9633..5544c7e7763 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj@@ -637,7 +637,7 @@base-query {:with [[:vector_candidates hnsw-query]]:select (into common-search-columns[[[:raw "row_number() OVER (ORDER BY distance ASC)"] :semantic_rank]- [:distance :semantic_score]])+ [:distance :semantic_distance]]):from [:vector_candidates]:where [:<= :distance max-cosine-distance]:order-by [[:semantic_rank :asc]]}]@@ -662,7 +662,7 @@{:with [[:vector_candidates filtered-query :materialized]]:select (into common-search-columns[[[:raw "row_number() OVER (ORDER BY distance ASC)"] :semantic_rank]- [:distance :semantic_score]])+ [:distance :semantic_distance]]):from [:vector_candidates]:where [:<= :distance max-cosine-distance]:order-by [[:semantic_rank :asc]]@@ -723,6 +723,7 @@:select (into(mapv hybrid-select common-search-columns)[[:v.semantic_rank :semantic_rank]+ [:v.semantic_distance :semantic_distance][:t.keyword_rank :keyword_rank]]):from [[:vector_results :v]]:full-join [[:text_results :t] [:= :v.id :t.id]]@@ -743,7 +744,7 @@{:keys [ctes query]} (flatten-ctes hybrid-query)all-ctes (conj ctes [:hybrid_results query])full-query {:with all-ctes- :select [:id :model_id :model :content :verified :legacy_input :semantic_rank :keyword_rank]+ :select [:id :model_id :model :content :verified :legacy_input :semantic_rank :semantic_distance :keyword_rank]:from [[:hybrid_results :search_index]]:limit (semantic-settings/semantic-search-results-limit)}](scoring/with-scores search-context scorers full-query)))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.cljindex 2b1e1eacdd0..ec7a06a6b2a 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj@@ -69,6 +69,19 @@[:* [:cast keyword-weight :float][:coalesce [:/ 1.0 [:+ k :keyword_rank]] 0]]]))+(def ^:private cosine-distance-ceiling+ "Largest possible pgvector cosine distance (`<=>`): `1 - cos(theta)` tops out at 2 for opposed vectors."+ 2.0)++(defn- semantic-distance-score-expr+ "Map a cosine `distance` (pgvector `<=>`, range [0, 2]) to a [0, 1] score.+ Linear for now: distance 0 scores 1, the maximum distance of 2 scores 0."+ [distance]+ ;; TODO (Chris 2026-06-02) -- consider rescaling against the retrieval cutoff (index/max-cosine-distance) so the+ ;; [0, cutoff] band spans the full [0, 1], and/or a non-linear curve. For now keep the raw distance as transparent+ ;; as possible.+ [:- [:inline 1] [:/ distance [:inline cosine-distance-ceiling]]])+(defn base-scorers"The default constituents of the search ranking scores."[index-table {:keys [search-string] :as search-ctx}]@@ -77,6 +90,8 @@;; NOTE: we calculate scores even if the weight is zero, so that it's easy to consider how we could affect any;; given set of results. At some point, we should optimize away the irrelevant scores for any given context.{:rrf rrf-rank-exp+ ;; Keyword-only hits have no vector distance, so treat them as maximally distant (score 0).+ :semantic-distance (semantic-distance-score-expr [:coalesce :semantic_distance [:inline cosine-distance-ceiling]]):view-count (view-count-expr index-table search.config/view-count-scaling-percentile):pinned (search.scoring/truthy :pinned):recency (search.scoring/inverse-duration@@ -88,12 +103,14 @@:model (search.scoring/model-rank-expr search-ctx):mine (search.scoring/equal :creator_id (:current-user-id search-ctx)):exact (if search-string- ;; perform the lower casing within the database, in case it behaves differently to our helper- (search.scoring/equal [:lower :name] [:lower search-string])+ ;; normalize both sides in the database, in case it behaves differently to our helper+ (search.scoring/equal (search.scoring/normalize-text-expr :postgres :name)+ (search.scoring/normalize-text-expr :postgres search-string))[:inline 0]):prefix (if search-string;; in this case, we need to transform the string into a pattern in code, so forced to use helper- (search.scoring/prefix [:lower :name] (u/lower-case-en search-string))+ (search.scoring/prefix (search.scoring/normalize-text-expr :postgres :name)+ (search.scoring/normalize-text search-string))[:inline 0]):library (search.scoring/library-score-expr):data-layer (search.scoring/data-layer-score-expr search-ctx)}))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.cljindex 985e0fca899..534e04e5e58 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj@@ -160,6 +160,27 @@(is (= 1 (:metabase-search/semantic-fallback-triggered @metrics)))(is (= 1 (:metabase-search/semantic-results-before-fallback @metrics)))))))))))))+(deftest fallback-results-zero-semantic-distance-test+ (testing "fallback results record :semantic-distance as 0 in their score breakdown, leaving other scores intact"+ (mt/with-premium-features #{:semantic-search}+ (mt/with-temporary-setting-values [semantic-search-min-results-threshold 3]+ (let [semantic-result (assoc (make-card-result 1 "semantic-card" :score 0.9)+ :all-scores [{:name :semantic-distance :score 0.95 :weight 10 :contribution 9.5}])+ fallback-results [(assoc (make-card-result 2 "fallback-card" :score 0.5)+ :all-scores [{:name :text :score 1 :weight 5 :contribution 5}])]+ search-ctx (assoc search-context :weights {:semantic-distance 10})]+ (with-search-engine-mocks! [semantic-result] fallback-results+ (fn []+ (let [results (vec (semantic.core/results search-ctx))+ result-by-id (fn [id] (some #(when (= id (:id %)) %) results))]+ (testing "fallback result gains a zero semantic-distance entry, keeping its existing scores"+ (is (=? [{:name :text :score 1}+ {:name :semantic-distance :score 0 :weight 10 :contribution 0}]+ (:all-scores (result-by-id 2)))))+ (testing "genuine semantic result keeps its computed distance"+ (is (=? [{:name :semantic-distance :score 0.95}]+ (:all-scores (result-by-id 1)))))))))))))+(deftest test-semantic-search-fallback-failure-resilient(testing "semantic search is resilient to fallback engine failure"(mt/with-premium-features #{:semantic-search}diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.cljindex 13e39e2644a..fff87545591 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.clj@@ -94,6 +94,27 @@["card" 3 "classified"]](search-results :rrf "order"))))))))+(deftest semantic-distance-test+ (mt/with-premium-features #{:semantic-search}+ ;; Bind embeddings around indexing *and* querying: cosine distance scores the stored doc embedding against the+ ;; query embedding, so the docs must be indexed with these vectors, not the default fallback.+ ;; Cosine distance ignores magnitude, so these order purely by direction relative to the query: "near" shares the+ ;; query's direction (distance 0 -> score 1), "mid" diverges a little, "far" the most. All stay under the 0.7+ ;; cutoff so every doc survives semantic retrieval.+ (semantic.tu/with-mock-embeddings {"animal" [1.0 0.0 0.0 0.0]+ "animal near" [1.0 0.0 0.0 0.0]+ "animal mid" [1.0 0.5 0.0 0.0]+ "animal far" [1.0 1.0 0.0 0.0]}+ (with-index-contents!+ [{:model "card" :id 1 :name "animal near"}+ {:model "card" :id 2 :name "animal mid"}+ {:model "card" :id 3 :name "animal far"}]+ (testing "Closer cosine distance ranks higher"+ (is (= [["card" 1 "animal near"]+ ["card" 2 "animal mid"]+ ["card" 3 "animal far"]]+ (search-results :semantic-distance "animal"))))))))+(deftest exact-test(mt/with-premium-features #{:semantic-search}(with-index-contents!@@ -104,6 +125,16 @@["card" 2 "stop words"]](search-results :exact "the any most of stop words very")))))))+(deftest exact-normalization-test+ (mt/with-premium-features #{:semantic-search}+ (with-index-contents!+ [{:model "card" :id 1 :name "Sales, Revenue"}+ {:model "card" :id 2 :name "Sales Revenue Report"}]+ (testing "Exact matching ignores commas and collapses whitespace runs"+ (is (= [["card" 1 "Sales, Revenue"]+ ["card" 2 "Sales Revenue Report"]]+ (search-results :exact "sales revenue")))))))+(deftest prefix-test(mt/with-premium-features #{:semantic-search}(with-index-contents!@@ -114,6 +145,16 @@["card" 2 "a prefix this is not, unfortunately"]](search-results :prefix "this is a prefix")))))))+(deftest prefix-normalization-test+ (mt/with-premium-features #{:semantic-search}+ (with-index-contents!+ [{:model "card" :id 1 :name "Sales, Revenue Quarterly"}+ {:model "card" :id 2 :name "Revenue and Sales"}]+ (testing "Prefix matching ignores commas and collapses whitespace runs"+ (is (= [["card" 1 "Sales, Revenue Quarterly"]+ ["card" 2 "Revenue and Sales"]]+ (search-results :prefix "sales revenue")))))))+(deftest pinned-test(mt/with-premium-features #{:semantic-search}(with-index-contents!diff --git a/src/metabase/search/appdb/scoring.clj b/src/metabase/search/appdb/scoring.cljindex b8a289b907b..e6f0311f0f6 100644--- a/src/metabase/search/appdb/scoring.clj+++ b/src/metabase/search/appdb/scoring.clj@@ -56,12 +56,14 @@:model (search.scoring/model-rank-expr search-ctx):mine (search.scoring/equal :search_index.creator_id (:current-user-id search-ctx)):exact (if search-string- ;; perform the lower casing within the database, in case it behaves differently to our helper- (search.scoring/equal [:lower :search_index.name] [:lower search-string])+ ;; normalize both sides in the database, in case it behaves differently to our helper+ (search.scoring/equal (search.scoring/normalize-text-expr :search_index.name)+ (search.scoring/normalize-text-expr search-string))[:inline 0]):prefix (if search-string;; in this case, we need to transform the string into a pattern in code, so forced to use helper- (search.scoring/prefix [:lower :search_index.name] (u/lower-case-en search-string))+ (search.scoring/prefix (search.scoring/normalize-text-expr :search_index.name)+ (search.scoring/normalize-text search-string))[:inline 0]):library (search.scoring/library-score-expr):data-layer (search.scoring/data-layer-score-expr search-ctx)}))diff --git a/src/metabase/search/config.clj b/src/metabase/search/config.cljindex 9066cc4e02d..525a371509c 100644--- a/src/metabase/search/config.clj+++ b/src/metabase/search/config.clj@@ -97,7 +97,9 @@:official-collection 80:verified 80;; RRF is the "Reciprocal Rank Fusion" score used by the semantic search backend to blend semantic and keyword scores- :rrf 500}+ :rrf 500+ ;; Maps the backend's cosine distance to a [0, 1] score: 1 = identical vector, 0 = maximally distant or keyword-only hit.+ :semantic-distance 10}:command-palette{:prefix 5:model/collection 1diff --git a/src/metabase/search/scoring.clj b/src/metabase/search/scoring.cljindex 8d8f1a523f6..3d76e2ceddf 100644--- a/src/metabase/search/scoring.clj+++ b/src/metabase/search/scoring.clj@@ -5,6 +5,7 @@[metabase.app-db.core :as mdb][metabase.collections.models.collection :as collection][metabase.search.config :as search.config]+ [metabase.util :as u][metabase.util.honey-sql-2 :as h2x]))(def ^:private seconds-in-a-day 86400)@@ -24,6 +25,26 @@[column value][:coalesce [:case [:like column (str (str/replace value "%" "%%") "%")] [:inline 1] :else [:inline 0]] [:inline 0]])+(defn normalize-text-expr+ "Wrap a string column/value SQL expr with the normalization the text scorers compare on:+ lower-case, replace commas with spaces, collapse whitespace runs to one space, trim.+ `db-type` picks the regexp_replace dialect -- Postgres needs the 'g' flag; H2 replaces all by default and+ rejects 'g'."+ ([expr] (normalize-text-expr (mdb/db-type) expr))+ ([db-type expr]+ ;; Replace commas with a space, not nothing, so `a,b` doesn't collapse into `ab`.+ (let [stripped [:replace [:lower expr] [:inline ","] [:inline " "]]+ collapsed (case db-type+ :postgres [:regexp_replace stripped [:inline "\\s+"] [:inline " "] [:inline "g"]]+ :h2 [:regexp_replace stripped [:inline "\\s+"] [:inline " "]])]+ [:trim collapsed])))++(defn normalize-text+ "Clojure analogue of `normalize-text-expr`, for callers that must normalize a search string in code (the+ :prefix scorer builds a LIKE pattern). Keep in lock-step with the SQL version."+ [s]+ (-> (u/lower-case-en s) (str/replace "," " ") (str/replace #"\s+" " ") str/trim))+(defn size"Prefer items whose value is larger, up to some saturation point. Items beyond that point are equivalent."[column ceiling]diff --git a/test/metabase/search/appdb/scoring_test.clj b/test/metabase/search/appdb/scoring_test.cljindex cff8ab9c3dc..da2c42c600e 100644--- a/test/metabase/search/appdb/scoring_test.clj+++ b/test/metabase/search/appdb/scoring_test.clj@@ -127,6 +127,20 @@;; TODO text ranking (probably in-memorynil)))+(deftest ^:parallel exact-normalization-test+ (with-index-contents+ [{:model "card" :id 1 :name "Sales, Revenue"}+ {:model "card" :id 2 :name "Sales Revenue Report"}]+ (case (mdb/db-type)+ :postgres+ (testing "Exact matching ignores commas and collapses whitespace runs"+ (is (= [["card" 1 "Sales, Revenue"]+ ["card" 2 "Sales Revenue Report"]]+ (search-results :exact "sales revenue"))))+ :h2+ ;; TODO text ranking (probably in-memory)+ nil)))+(deftest ^:parallel prefix-test(with-index-contents[{:model "card" :id 1 :name "this is a prefix of something longer"}@@ -136,6 +150,15 @@["card" 2 "a prefix this is not, unfortunately"]](search-results :prefix "this is a prefix"))))))+(deftest ^:parallel prefix-normalization-test+ (with-index-contents+ [{:model "card" :id 1 :name "Sales, Revenue Quarterly"}+ {:model "card" :id 2 :name "Revenue and Sales"}]+ (testing "Prefix matching ignores commas and collapses whitespace runs"+ (is (= [["card" 1 "Sales, Revenue Quarterly"]+ ["card" 2 "Revenue and Sales"]]+ (search-results :prefix "sales revenue"))))))+(deftest ^:parallel model-test(with-index-contents[{:model "dataset" :id 1 :name "card ancient"}diff --git a/test/metabase/search/scoring_test.clj b/test/metabase/search/scoring_test.cljnew file mode 100644index 00000000000..0c8ce658aa4--- /dev/null+++ b/test/metabase/search/scoring_test.clj@@ -0,0 +1,14 @@+(ns metabase.search.scoring-test+ (:require+ [clojure.test :refer :all]+ [metabase.search.scoring :as scoring]))++(deftest normalize-text-test+ (testing "normalize-text lower-cases, strips commas, collapses whitespace runs, and trims"+ (are [in out] (= out (scoring/normalize-text in))+ "Sales, Revenue" "sales revenue"+ "Sales, Revenue" "sales revenue"+ " Sales Revenue " "sales revenue"+ "Sales,Revenue" "sales revenue"+ "SALES" "sales"+ "" "")))