The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
feat(search): split weight profiles and normalize search contexts (#75388)
Chris Truterad23a6079041
diff --git a/src/metabase/search/api.clj b/src/metabase/search/api.cljindex 88b21fbeb35..0efb9c84449 100644--- a/src/metabase/search/api.clj+++ b/src/metabase/search/api.clj@@ -101,14 +101,13 @@(when (= context :all)(throw (ex-info "Cannot set weights for all context"{:status-code 400})))- (let [known-ranker? (set (keys (:default @#'search.config/static-weights)))- rankers (into #{}+ (let [rankers (into #{}(map (fn [k](if (namespace k)(keyword (namespace k))k)))(keys overrides))- unknown-rankers (not-empty (remove known-ranker? rankers))]+ unknown-rankers (not-empty (remove search.config/known-rankers rankers))](when unknown-rankers(throw (ex-info (str "Unknown rankers: " (str/join ", " (map name (sort unknown-rankers)))){:status-code 400})))@@ -127,7 +126,8 @@"Return the current weights being used to rank the search results"[_route-params{:keys [context]} :- [:map [:context {:default :default} :keyword]]]- (search.config/weights {:context context}))+ ;; normalize so the reported weights match what search actually applies for this context+ (search.config/weights {:context (search.config/normalized-context context)}));; TODO (Cam 10/28/25) -- fix this endpoint so it uses kebab-case for query parameters for consistency with the rest;; of the REST API@@ -144,7 +144,9 @@[:context {:default :default} :keyword][:search_engine {:optional true} :any]]];; remove cookie- (let [overrides (-> overrides (dissoc :search_engine :context) (update-vals parse-double))]+ ;; normalize so overrides are stored under the same key search reads them from+ (let [context (search.config/normalized-context context)+ overrides (-> overrides (dissoc :search_engine :context) (update-vals parse-double))](when (seq overrides)(set-weights! context overrides))(search.config/weights {:context context})))diff --git a/src/metabase/search/config.clj b/src/metabase/search/config.cljindex 90af654a4fb..26cd238164e 100644--- a/src/metabase/search/config.clj+++ b/src/metabase/search/config.clj@@ -7,6 +7,7 @@[metabase.search.settings :as search.settings][metabase.util :as u][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))(def ^:dynamic *db-max-results*@@ -75,32 +76,36 @@(assert (= all-models (set models-search-order)) "The models search order has to include all models")-(def static-weights- "Strength of the various scorers."- {:default- {:pinned 0- :bookmarked 1- :recency 1- :user-recency 5- :dashboard 0- :model 2- :view-count 2- :text 5- :mine 1- :exact 5- :prefix 0- ;; User-curated tiers, from strongest to weakest.- ;; :library at 200 exceeds :official-collection + :verified (80 each) combined,- ;; so a library item always outranks a non-library one.- ;; Base text/recency scorers (0–5 range) only break ties *within* a tier.- :library 200- :official-collection 80- :verified 80- ;; RRF is the "Reciprocal Rank Fusion" score used by the semantic search backend to blend semantic and keyword scores- :rrf 500- ;; Maps the backend's cosine distance to a [0, 1] score: 1 = identical vector, 0 = maximally distant or keyword-only hit.- :semantic-distance 10}- :command-palette+(def static-default-weights+ "Base scorer weights that every search inherits.+ Each context layers its overrides on top (see [[static-context-weights]] and [[weights]])."+ {:pinned 0+ :bookmarked 1+ :recency 1+ :user-recency 5+ :dashboard 0+ :model 2+ :view-count 2+ :text 5+ :mine 1+ :exact 5+ :prefix 0+ ;; User-curated tiers, from strongest to weakest.+ ;; :library at 200 exceeds :official-collection + :verified (80 each) combined,+ ;; so a library item always outranks a non-library one.+ ;; Base text/recency scorers (0–5 range) only break ties *within* a tier.+ :library 200+ :official-collection 80+ :verified 80+ ;; RRF is the "Reciprocal Rank Fusion" score used by the semantic search backend to blend semantic and keyword scores+ :rrf 500+ ;; Maps the backend's cosine distance to a [0, 1] score: 1 = identical vector, 0 = maximally distant or keyword-only hit.+ :semantic-distance 10})++(def static-context-weights+ "Per-context scorer overrides, keyed by [[normalized-contexts]].+ Merged onto [[static-default-weights]] for that context (see [[weights]])."+ {:global{:prefix 5:model/collection 1:model/dashboard 1@@ -138,6 +143,15 @@:data-layer/hidden 0.03 ; ≈ 1}})+(def known-rankers+ "Scorer keys the weights API accepts as overrides: the union across [[static-default-weights]] and every+ [[static-context-weights]] profile.+ Namespaced scorer params (e.g. `:model/dashboard`) collapse to their namespace (`:model`)."+ (into #{}+ (map #(if (namespace %) (keyword (namespace %)) %))+ (concat (keys static-default-weights)+ (mapcat keys (vals static-context-weights)))))+(def ^:private FilterDef"A relaxed definition, capturing how we can write the filter - with some fields omitted."[:map {:closed true}@@ -195,12 +209,12 @@:display-type {:type :list, :field "display_type"}}))(def ^:private filter-defaults-by-context- {:default {:archived false- ;; keys will typically those in [[filters]], but this is an atypical filter.- ;; we plan to generify it, by precalculating it on the index.- :filter-items-in-personal-collection "all"}- :search-app {:filter-items-in-personal-collection "exclude-others"}- :command-palette {:filter-items-in-personal-collection "exclude-others"}})+ ;; Keyed by [[normalized-contexts]] plus a `:default` base; the broad-search surfaces share `:global`.+ {:default {:archived false+ ;; keys will typically those in [[filters]], but this is an atypical filter.+ ;; we plan to generify it, by precalculating it on the index.+ :filter-items-in-personal-collection "all"}+ :global {:filter-items-in-personal-collection "exclude-others"}})(defn filter-default"Get the default value for the given filter in the given context."@@ -208,6 +222,26 @@(let [fetch (fn [ctx] (when ctx (-> filter-defaults-by-context (get ctx) (get filter-key))))](or (fetch context) (fetch :default))))+(declare normalized-context)++(defn- normalize-override-keys+ "Re-key persisted weight overrides ([[search.settings/experimental-search-weight-overrides]]) by+ [[normalized-context]], so an override saved under a context that has since been collapsed (e.g.+ `:command-palette` -> `:global`) is still applied.+ When a raw alias and its normalized context both carry overrides, the normalized one wins -- it's+ what the weights API writes today."+ [overrides]+ (reduce-kv (fn [acc context weight-overrides]+ (let [normalized (normalized-context context)]+ (update acc normalized+ (if (= context normalized)+ #(merge % weight-overrides)+ #(merge weight-overrides %)))))+ {}+ ;; sorted so the winner is deterministic when several aliases collapse to one normalized+ ;; context (the normalized key still wins; among aliases the lowest-sorted one does)+ (into (sorted-map) overrides)))+;; This gets called *a lot* during a search request, so we'll almost certainly need to optimize it. Maybe just TTL.(defn weights"Strength of the various scorers. Copied from metabase.search.in-place.scoring, but allowing divergence."@@ -215,13 +249,13 @@(weights {}))([{request-overrides :weights, :keys [context]}](let [context (or context :default)- system-overrides (search.settings/experimental-search-weight-overrides)]+ system-overrides (normalize-override-keys (search.settings/experimental-search-weight-overrides))](if (= :all context)- (merge-with merge static-weights system-overrides)- (merge (get static-weights :default)+ (merge-with merge (assoc static-context-weights :default static-default-weights) system-overrides)+ (merge static-default-weights;; Not sure which of the next two should have precedence, arguments for both "¯\_(ツ)_/¯"(get system-overrides :default)- (get static-weights context)+ (get static-context-weights context)(get system-overrides context)request-overrides)))))@@ -262,7 +296,7 @@(def ^:private ui-contexts"Search `context` values issued by the frontend, one per UI surface.- Selects the ranking weights ([[static-weights]]) and filter defaults ([[filter-defaults-by-context]]).+ Selects ranking weights ([[static-context-weights]]) and filter defaults ([[filter-defaults-by-context]]).Keep in sync with the frontend `SearchContext` type (frontend/src/metabase-types/api/search.ts).Give every value a comment naming the surface that issues it."[:search-bar ; the global navbar search typeahead@@ -293,6 +327,43 @@The `:decode/string keyword` hook coerces the query-string value (e.g. \"search-app\") to a keyword."(into [:enum {:decode/string keyword}] contexts))+(def context->normalized-context+ "Collapses search `context` values that should rank and filter alike onto a shared normalized context.+ Only genuine remappings appear here; an unlisted context normalizes to itself and inherits `:default`.+ The full-page search app, command palette, and type-filter lookup share `:global`, as opposed to the+ scoped `:data-picker` / `:entity-picker` contexts."+ ;; TODO (Chris 2026-06-08) -- the nav search bar is deliberately left off `:global` for now so it keeps+ ;; the default filters/weights; decide whether any of its behaviour should be DRYed up with the other+ ;; broad-search surfaces.+ {:search-app :global+ :command-palette :global+ :type-filter :global})++(assert (every? (set contexts) (keys context->normalized-context))+ "context->normalized-context keys must be valid contexts")++(def normalized-contexts+ "Context values [[normalized-context]] can return: the remap targets plus every un-remapped context.+ [[static-context-weights]] and [[filter-defaults-by-context]] are keyed by these (each also has a+ `:default` base entry)."+ (into (set (vals context->normalized-context))+ (remove (set (keys context->normalized-context)))+ contexts))++(def NormalizedContext+ "Malli enum schema for a normalized search context (see [[normalized-contexts]])."+ (into [:enum] (sort normalized-contexts)))++(defn normalized-context+ "Normalize a search `context` for weight and filter-default lookup.+ Contexts that should behave alike collapse to a shared value; all others map to themselves.+ Also used by the `/api/search/weights` endpoints so stored/read override keys agree with search."+ [context]+ (get context->normalized-context context context))++(assert (every? (partial mr/validate NormalizedContext) (keys static-context-weights))+ "static-context-weights must be keyed by normalized contexts")+(def SearchContext"Map with the various allowed search parameters, used to construct the SQL query."[:map {:closed true}diff --git a/src/metabase/search/impl.clj b/src/metabase/search/impl.cljindex 2904aa7d130..857e71da848 100644--- a/src/metabase/search/impl.clj+++ b/src/metabase/search/impl.clj@@ -317,7 +317,8 @@(premium-features/assert-has-any-features[:content-verification :official-collections](deferred-tru "Content Management or Official Collections")))- (let [models (if (seq models) models search.config/all-models)+ (let [context (some-> context search.config/normalized-context)+ models (if (seq models) models search.config/all-models)engine (parse-engine search-engine)fvalue (fn [filter-key] (search.config/filter-default engine context filter-key))ctx (cond-> {:archived? (boolean (or archived (fvalue :archived)))diff --git a/test/metabase/search/api_test.clj b/test/metabase/search/api_test.cljindex 8745f95f686..2276357f515 100644--- a/test/metabase/search/api_test.clj+++ b/test/metabase/search/api_test.clj@@ -1831,6 +1831,16 @@(mt/user-http-request :crowberto :put 200 (weights-url context {:model/dataset 5}))))(is (= 5.0 (search.config/scorer-param search-ctx :model :dataset)))))))+(deftest ^:synchronized weights-normalize-context-test+ (mt/with-temporary-setting-values [experimental-search-weight-overrides nil]+ (testing "the /weights endpoints normalize context, so introspection and overrides match search"+ (testing "surfaces that share a normalized context report the same weights"+ (is (= (mt/user-http-request :crowberto :get 200 (weights-url :global {}))+ (mt/user-http-request :crowberto :get 200 (weights-url :command-palette {})))))+ (testing "an override set via one normalized surface is read back via another"+ (mt/user-http-request :crowberto :put 200 (weights-url :command-palette {:recency 9}))+ (is (= 9.0 (:recency (mt/user-http-request :crowberto :get 200 (weights-url :search-app {})))))))))+(deftest ^:synchronized dashboard-questions(testing "Dashboard questions get a dashboard_id when searched"(let [search-name (random-uuid)diff --git a/test/metabase/search/config_test.clj b/test/metabase/search/config_test.cljindex e007b56c2f8..922deaee484 100644--- a/test/metabase/search/config_test.clj+++ b/test/metabase/search/config_test.clj@@ -7,22 +7,19 @@;; All that matters is that this is not legacy search.(def ^:private search-engine :search.engine/appdb)+;; `filter-default` and `weights` take an already-normalized context (normalization happens once at the top+;; of the search call tree); these tests pass the normalized values directly.+(deftest filter-default-test(testing "Default values"(is (= false (search.config/filter-default search-engine nil :archived)))(is (= "all" (search.config/filter-default search-engine nil :filter-items-in-personal-collection))))- (testing "No overrides"- (is (= false (search.config/filter-default search-engine :command-palette :archived))))- (testing "Overrides"- (is (= "exclude-others"- (search.config/filter-default search-engine :command-palette :filter-items-in-personal-collection)))+ (testing "the :global profile excludes others' personal collections"(is (= "exclude-others"- (search.config/filter-default search-engine :search-app :filter-items-in-personal-collection))))- (testing "Legacy search should respect context overrides (#UXW-3238)"+ (search.config/filter-default search-engine :global :filter-items-in-personal-collection))))+ (testing "Legacy search uses the same per-context defaults (#UXW-3238)"(is (= "exclude-others"- (search.config/filter-default :search.engine/in-place :command-palette :filter-items-in-personal-collection)))- (is (= "exclude-others"- (search.config/filter-default :search.engine/in-place :search-app :filter-items-in-personal-collection)))))+ (search.config/filter-default :search.engine/in-place :global :filter-items-in-personal-collection)))))(deftest context-schema-test(testing "the HTTP `context` enum allows the UI surfaces, :api, and :metabot (accepted for debugging)"@@ -31,3 +28,44 @@(is (mr/validate search.config/Context :metabot)))(testing "values outside the enum are rejected"(is (not (mr/validate search.config/Context :bogus)))))++(deftest normalized-context-test+ (testing "the broad-search surfaces normalize to a single :global context"+ (is (= [:global :global :global]+ (map search.config/normalized-context [:search-app :command-palette :type-filter]))))+ (testing "the nav search bar is intentionally not normalized (keeps default filters/weights for now)"+ (is (= :search-bar (search.config/normalized-context :search-bar))))+ (testing "every other context normalizes to itself"+ (is (= [:entity-picker :data-picker :metabot :api]+ (map search.config/normalized-context [:entity-picker :data-picker :metabot :api]))))+ (testing "static-context-weights are keyed only by normalized contexts, never the :default base"+ (is (not (contains? search.config/static-context-weights :default)))+ (is (every? (set search.config/normalized-contexts) (keys search.config/static-context-weights)))))++(deftest normalize-override-keys-test+ (let [normalize #'search.config/normalize-override-keys]+ (testing "an override persisted under a context that has since collapsed is re-keyed to its normalized context"+ (is (= {:global {:exact 1}} (normalize {:command-palette {:exact 1}}))))+ (testing "when a raw alias and its normalized context both have overrides, the normalized one wins"+ (is (= {:global {:exact 2}} (normalize {:command-palette {:exact 1} :global {:exact 2}})))+ (is (= {:global {:text 5 :exact 9}} (normalize {:type-filter {:text 5} :global {:exact 9}}))))+ (testing "the :default base and un-remapped contexts pass through untouched"+ (is (= {:default {:text 7} :entity-picker {:exact 3}}+ (normalize {:default {:text 7} :entity-picker {:exact 3}})))))+ (testing "several aliases collapsing to one normalized context resolve deterministically, regardless of+ input order (the lowest-sorted alias wins among aliases)"+ (let [normalize #'search.config/normalize-override-keys]+ (is (= {:global {:exact 2}}+ (normalize (array-map :search-app {:exact 1} :command-palette {:exact 2}))+ (normalize (array-map :command-palette {:exact 2} :search-app {:exact 1})))))))++(deftest weights-by-context-test+ (testing "the broad-search surfaces share one weight profile that boosts prefix matches"+ (is (= 5 (search.config/weight {:context :global} :prefix)))+ (is (= 0 (search.config/weight {:context :default} :prefix)))))++(deftest known-rankers-test+ (testing "overridable rankers span every profile, not just :default"+ ;; :library lives in :default; :data-layer only in :metabot+ (is (contains? search.config/known-rankers :library))+ (is (contains? search.config/known-rankers :data-layer))))