Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

feat(search): split weight profiles and normalize search contexts (#75388)

Chris Truterad23a6079041
diff --git a/src/metabase/search/api.clj b/src/metabase/search/api.clj
index 88b21fbeb35..0efb9c84449 100644
--- a/src/metabase/search/api.clj
+++ b/src/metabase/search/api.clj
@@ -101,14 +101,13 @@
(when (= context :all)
(throw (ex-info "Cannot set weights for all context"
{:status-code 400})))
- (let [known-ranker? (set (keys (:default @#'search.config/static-weights)))
- rankers (into #{}
+ (let [rankers (into #{}
(map (fn [k]
(if (namespace k)
(keyword (namespace k))
k)))
(keys overrides))
- unknown-rankers (not-empty (remove known-ranker? rankers))]
+ unknown-rankers (not-empty (remove search.config/known-rankers rankers))]
(when unknown-rankers
(throw (ex-info (str "Unknown rankers: " (str/join ", " (map name (sort unknown-rankers))))
{:status-code 400})))
@@ -127,7 +126,8 @@
"Return the current weights being used to rank the search results"
[_route-params
{:keys [context]} :- [:map [:context {:default :default} :keyword]]]
- (search.config/weights {:context context}))
+ ;; normalize so the reported weights match what search actually applies for this context
+ (search.config/weights {:context (search.config/normalized-context context)}))
;; TODO (Cam 10/28/25) -- fix this endpoint so it uses kebab-case for query parameters for consistency with the rest
;; of the REST API
@@ -144,7 +144,9 @@
[:context {:default :default} :keyword]
[:search_engine {:optional true} :any]]]
;; remove cookie
- (let [overrides (-> overrides (dissoc :search_engine :context) (update-vals parse-double))]
+ ;; normalize so overrides are stored under the same key search reads them from
+ (let [context (search.config/normalized-context context)
+ overrides (-> overrides (dissoc :search_engine :context) (update-vals parse-double))]
(when (seq overrides)
(set-weights! context overrides))
(search.config/weights {:context context})))
diff --git a/src/metabase/search/config.clj b/src/metabase/search/config.clj
index 90af654a4fb..26cd238164e 100644
--- a/src/metabase/search/config.clj
+++ b/src/metabase/search/config.clj
@@ -7,6 +7,7 @@
[metabase.search.settings :as search.settings]
[metabase.util :as u]
[metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]))
(def ^:dynamic *db-max-results*
@@ -75,32 +76,36 @@
(assert (= all-models (set models-search-order)) "The models search order has to include all models")
-(def static-weights
- "Strength of the various scorers."
- {:default
- {:pinned 0
- :bookmarked 1
- :recency 1
- :user-recency 5
- :dashboard 0
- :model 2
- :view-count 2
- :text 5
- :mine 1
- :exact 5
- :prefix 0
- ;; User-curated tiers, from strongest to weakest.
- ;; :library at 200 exceeds :official-collection + :verified (80 each) combined,
- ;; so a library item always outranks a non-library one.
- ;; Base text/recency scorers (0–5 range) only break ties *within* a tier.
- :library 200
- :official-collection 80
- :verified 80
- ;; RRF is the "Reciprocal Rank Fusion" score used by the semantic search backend to blend semantic and keyword scores
- :rrf 500
- ;; Maps the backend's cosine distance to a [0, 1] score: 1 = identical vector, 0 = maximally distant or keyword-only hit.
- :semantic-distance 10}
- :command-palette
+(def static-default-weights
+ "Base scorer weights that every search inherits.
+ Each context layers its overrides on top (see [[static-context-weights]] and [[weights]])."
+ {:pinned 0
+ :bookmarked 1
+ :recency 1
+ :user-recency 5
+ :dashboard 0
+ :model 2
+ :view-count 2
+ :text 5
+ :mine 1
+ :exact 5
+ :prefix 0
+ ;; User-curated tiers, from strongest to weakest.
+ ;; :library at 200 exceeds :official-collection + :verified (80 each) combined,
+ ;; so a library item always outranks a non-library one.
+ ;; Base text/recency scorers (0–5 range) only break ties *within* a tier.
+ :library 200
+ :official-collection 80
+ :verified 80
+ ;; RRF is the "Reciprocal Rank Fusion" score used by the semantic search backend to blend semantic and keyword scores
+ :rrf 500
+ ;; Maps the backend's cosine distance to a [0, 1] score: 1 = identical vector, 0 = maximally distant or keyword-only hit.
+ :semantic-distance 10})
+
+(def static-context-weights
+ "Per-context scorer overrides, keyed by [[normalized-contexts]].
+ Merged onto [[static-default-weights]] for that context (see [[weights]])."
+ {:global
{:prefix 5
:model/collection 1
:model/dashboard 1
@@ -138,6 +143,15 @@
:data-layer/hidden 0.03 ; ≈ 1
}})
+(def known-rankers
+ "Scorer keys the weights API accepts as overrides: the union across [[static-default-weights]] and every
+ [[static-context-weights]] profile.
+ Namespaced scorer params (e.g. `:model/dashboard`) collapse to their namespace (`:model`)."
+ (into #{}
+ (map #(if (namespace %) (keyword (namespace %)) %))
+ (concat (keys static-default-weights)
+ (mapcat keys (vals static-context-weights)))))
+
(def ^:private FilterDef
"A relaxed definition, capturing how we can write the filter - with some fields omitted."
[:map {:closed true}
@@ -195,12 +209,12 @@
:display-type {:type :list, :field "display_type"}}))
(def ^:private filter-defaults-by-context
- {:default {:archived false
- ;; keys will typically those in [[filters]], but this is an atypical filter.
- ;; we plan to generify it, by precalculating it on the index.
- :filter-items-in-personal-collection "all"}
- :search-app {:filter-items-in-personal-collection "exclude-others"}
- :command-palette {:filter-items-in-personal-collection "exclude-others"}})
+ ;; Keyed by [[normalized-contexts]] plus a `:default` base; the broad-search surfaces share `:global`.
+ {:default {:archived false
+ ;; keys will typically those in [[filters]], but this is an atypical filter.
+ ;; we plan to generify it, by precalculating it on the index.
+ :filter-items-in-personal-collection "all"}
+ :global {:filter-items-in-personal-collection "exclude-others"}})
(defn filter-default
"Get the default value for the given filter in the given context."
@@ -208,6 +222,26 @@
(let [fetch (fn [ctx] (when ctx (-> filter-defaults-by-context (get ctx) (get filter-key))))]
(or (fetch context) (fetch :default))))
+(declare normalized-context)
+
+(defn- normalize-override-keys
+ "Re-key persisted weight overrides ([[search.settings/experimental-search-weight-overrides]]) by
+ [[normalized-context]], so an override saved under a context that has since been collapsed (e.g.
+ `:command-palette` -> `:global`) is still applied.
+ When a raw alias and its normalized context both carry overrides, the normalized one wins -- it's
+ what the weights API writes today."
+ [overrides]
+ (reduce-kv (fn [acc context weight-overrides]
+ (let [normalized (normalized-context context)]
+ (update acc normalized
+ (if (= context normalized)
+ #(merge % weight-overrides)
+ #(merge weight-overrides %)))))
+ {}
+ ;; sorted so the winner is deterministic when several aliases collapse to one normalized
+ ;; context (the normalized key still wins; among aliases the lowest-sorted one does)
+ (into (sorted-map) overrides)))
+
;; This gets called *a lot* during a search request, so we'll almost certainly need to optimize it. Maybe just TTL.
(defn weights
"Strength of the various scorers. Copied from metabase.search.in-place.scoring, but allowing divergence."
@@ -215,13 +249,13 @@
(weights {}))
([{request-overrides :weights, :keys [context]}]
(let [context (or context :default)
- system-overrides (search.settings/experimental-search-weight-overrides)]
+ system-overrides (normalize-override-keys (search.settings/experimental-search-weight-overrides))]
(if (= :all context)
- (merge-with merge static-weights system-overrides)
- (merge (get static-weights :default)
+ (merge-with merge (assoc static-context-weights :default static-default-weights) system-overrides)
+ (merge static-default-weights
;; Not sure which of the next two should have precedence, arguments for both "¯\_(ツ)_/¯"
(get system-overrides :default)
- (get static-weights context)
+ (get static-context-weights context)
(get system-overrides context)
request-overrides)))))
@@ -262,7 +296,7 @@
(def ^:private ui-contexts
"Search `context` values issued by the frontend, one per UI surface.
- Selects the ranking weights ([[static-weights]]) and filter defaults ([[filter-defaults-by-context]]).
+ Selects ranking weights ([[static-context-weights]]) and filter defaults ([[filter-defaults-by-context]]).
Keep in sync with the frontend `SearchContext` type (frontend/src/metabase-types/api/search.ts).
Give every value a comment naming the surface that issues it."
[:search-bar ; the global navbar search typeahead
@@ -293,6 +327,43 @@
The `:decode/string keyword` hook coerces the query-string value (e.g. \"search-app\") to a keyword."
(into [:enum {:decode/string keyword}] contexts))
+(def context->normalized-context
+ "Collapses search `context` values that should rank and filter alike onto a shared normalized context.
+ Only genuine remappings appear here; an unlisted context normalizes to itself and inherits `:default`.
+ The full-page search app, command palette, and type-filter lookup share `:global`, as opposed to the
+ scoped `:data-picker` / `:entity-picker` contexts."
+ ;; TODO (Chris 2026-06-08) -- the nav search bar is deliberately left off `:global` for now so it keeps
+ ;; the default filters/weights; decide whether any of its behaviour should be DRYed up with the other
+ ;; broad-search surfaces.
+ {:search-app :global
+ :command-palette :global
+ :type-filter :global})
+
+(assert (every? (set contexts) (keys context->normalized-context))
+ "context->normalized-context keys must be valid contexts")
+
+(def normalized-contexts
+ "Context values [[normalized-context]] can return: the remap targets plus every un-remapped context.
+ [[static-context-weights]] and [[filter-defaults-by-context]] are keyed by these (each also has a
+ `:default` base entry)."
+ (into (set (vals context->normalized-context))
+ (remove (set (keys context->normalized-context)))
+ contexts))
+
+(def NormalizedContext
+ "Malli enum schema for a normalized search context (see [[normalized-contexts]])."
+ (into [:enum] (sort normalized-contexts)))
+
+(defn normalized-context
+ "Normalize a search `context` for weight and filter-default lookup.
+ Contexts that should behave alike collapse to a shared value; all others map to themselves.
+ Also used by the `/api/search/weights` endpoints so stored/read override keys agree with search."
+ [context]
+ (get context->normalized-context context context))
+
+(assert (every? (partial mr/validate NormalizedContext) (keys static-context-weights))
+ "static-context-weights must be keyed by normalized contexts")
+
(def SearchContext
"Map with the various allowed search parameters, used to construct the SQL query."
[:map {:closed true}
diff --git a/src/metabase/search/impl.clj b/src/metabase/search/impl.clj
index 2904aa7d130..857e71da848 100644
--- a/src/metabase/search/impl.clj
+++ b/src/metabase/search/impl.clj
@@ -317,7 +317,8 @@
(premium-features/assert-has-any-features
[:content-verification :official-collections]
(deferred-tru "Content Management or Official Collections")))
- (let [models (if (seq models) models search.config/all-models)
+ (let [context (some-> context search.config/normalized-context)
+ models (if (seq models) models search.config/all-models)
engine (parse-engine search-engine)
fvalue (fn [filter-key] (search.config/filter-default engine context filter-key))
ctx (cond-> {:archived? (boolean (or archived (fvalue :archived)))
diff --git a/test/metabase/search/api_test.clj b/test/metabase/search/api_test.clj
index 8745f95f686..2276357f515 100644
--- a/test/metabase/search/api_test.clj
+++ b/test/metabase/search/api_test.clj
@@ -1831,6 +1831,16 @@
(mt/user-http-request :crowberto :put 200 (weights-url context {:model/dataset 5}))))
(is (= 5.0 (search.config/scorer-param search-ctx :model :dataset)))))))
+(deftest ^:synchronized weights-normalize-context-test
+ (mt/with-temporary-setting-values [experimental-search-weight-overrides nil]
+ (testing "the /weights endpoints normalize context, so introspection and overrides match search"
+ (testing "surfaces that share a normalized context report the same weights"
+ (is (= (mt/user-http-request :crowberto :get 200 (weights-url :global {}))
+ (mt/user-http-request :crowberto :get 200 (weights-url :command-palette {})))))
+ (testing "an override set via one normalized surface is read back via another"
+ (mt/user-http-request :crowberto :put 200 (weights-url :command-palette {:recency 9}))
+ (is (= 9.0 (:recency (mt/user-http-request :crowberto :get 200 (weights-url :search-app {})))))))))
+
(deftest ^:synchronized dashboard-questions
(testing "Dashboard questions get a dashboard_id when searched"
(let [search-name (random-uuid)
diff --git a/test/metabase/search/config_test.clj b/test/metabase/search/config_test.clj
index e007b56c2f8..922deaee484 100644
--- a/test/metabase/search/config_test.clj
+++ b/test/metabase/search/config_test.clj
@@ -7,22 +7,19 @@
;; All that matters is that this is not legacy search.
(def ^:private search-engine :search.engine/appdb)
+;; `filter-default` and `weights` take an already-normalized context (normalization happens once at the top
+;; of the search call tree); these tests pass the normalized values directly.
+
(deftest filter-default-test
(testing "Default values"
(is (= false (search.config/filter-default search-engine nil :archived)))
(is (= "all" (search.config/filter-default search-engine nil :filter-items-in-personal-collection))))
- (testing "No overrides"
- (is (= false (search.config/filter-default search-engine :command-palette :archived))))
- (testing "Overrides"
- (is (= "exclude-others"
- (search.config/filter-default search-engine :command-palette :filter-items-in-personal-collection)))
+ (testing "the :global profile excludes others' personal collections"
(is (= "exclude-others"
- (search.config/filter-default search-engine :search-app :filter-items-in-personal-collection))))
- (testing "Legacy search should respect context overrides (#UXW-3238)"
+ (search.config/filter-default search-engine :global :filter-items-in-personal-collection))))
+ (testing "Legacy search uses the same per-context defaults (#UXW-3238)"
(is (= "exclude-others"
- (search.config/filter-default :search.engine/in-place :command-palette :filter-items-in-personal-collection)))
- (is (= "exclude-others"
- (search.config/filter-default :search.engine/in-place :search-app :filter-items-in-personal-collection)))))
+ (search.config/filter-default :search.engine/in-place :global :filter-items-in-personal-collection)))))
(deftest context-schema-test
(testing "the HTTP `context` enum allows the UI surfaces, :api, and :metabot (accepted for debugging)"
@@ -31,3 +28,44 @@
(is (mr/validate search.config/Context :metabot)))
(testing "values outside the enum are rejected"
(is (not (mr/validate search.config/Context :bogus)))))
+
+(deftest normalized-context-test
+ (testing "the broad-search surfaces normalize to a single :global context"
+ (is (= [:global :global :global]
+ (map search.config/normalized-context [:search-app :command-palette :type-filter]))))
+ (testing "the nav search bar is intentionally not normalized (keeps default filters/weights for now)"
+ (is (= :search-bar (search.config/normalized-context :search-bar))))
+ (testing "every other context normalizes to itself"
+ (is (= [:entity-picker :data-picker :metabot :api]
+ (map search.config/normalized-context [:entity-picker :data-picker :metabot :api]))))
+ (testing "static-context-weights are keyed only by normalized contexts, never the :default base"
+ (is (not (contains? search.config/static-context-weights :default)))
+ (is (every? (set search.config/normalized-contexts) (keys search.config/static-context-weights)))))
+
+(deftest normalize-override-keys-test
+ (let [normalize #'search.config/normalize-override-keys]
+ (testing "an override persisted under a context that has since collapsed is re-keyed to its normalized context"
+ (is (= {:global {:exact 1}} (normalize {:command-palette {:exact 1}}))))
+ (testing "when a raw alias and its normalized context both have overrides, the normalized one wins"
+ (is (= {:global {:exact 2}} (normalize {:command-palette {:exact 1} :global {:exact 2}})))
+ (is (= {:global {:text 5 :exact 9}} (normalize {:type-filter {:text 5} :global {:exact 9}}))))
+ (testing "the :default base and un-remapped contexts pass through untouched"
+ (is (= {:default {:text 7} :entity-picker {:exact 3}}
+ (normalize {:default {:text 7} :entity-picker {:exact 3}})))))
+ (testing "several aliases collapsing to one normalized context resolve deterministically, regardless of
+ input order (the lowest-sorted alias wins among aliases)"
+ (let [normalize #'search.config/normalize-override-keys]
+ (is (= {:global {:exact 2}}
+ (normalize (array-map :search-app {:exact 1} :command-palette {:exact 2}))
+ (normalize (array-map :command-palette {:exact 2} :search-app {:exact 1})))))))
+
+(deftest weights-by-context-test
+ (testing "the broad-search surfaces share one weight profile that boosts prefix matches"
+ (is (= 5 (search.config/weight {:context :global} :prefix)))
+ (is (= 0 (search.config/weight {:context :default} :prefix)))))
+
+(deftest known-rankers-test
+ (testing "overridable rankers span every profile, not just :default"
+ ;; :library lives in :default; :data-layer only in :metabot
+ (is (contains? search.config/known-rankers :library))
+ (is (contains? search.config/known-rankers :data-layer))))