The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Align Metabot and MCP search with Semantic Layer (#82337)
Chris Truterddce6528b5a8
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex ae0589c33c2..1655cfcb0d2 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1456,6 +1456,7 @@{:team "Graphy":api #{metabase.measures.api metabase.measures.schema}:uses #{api+ collectionseventsliblib.be@@ -1472,7 +1473,10 @@warehouse-schema-overlaywarehouses.schema}:model-exports #{:model/Measure}- :model-imports #{:model/Collection :model/Table :model/User}}+ :model-imports #{:model/Collection+ :model/Table+ :model/TableUserSettings+ :model/User}}metabot{:team "Metabot"@@ -2550,6 +2554,7 @@:module-exports #{segments.rest}:api #{metabase.segments.schema}:uses #{api+ collectionsliblib.belib.schema@@ -2563,7 +2568,10 @@warehouse-schema-overlaywarehouses.schema}:model-exports #{:model/Segment}- :model-imports #{:model/Collection :model/Table :model/User}}+ :model-imports #{:model/Collection+ :model/Table+ :model/TableUserSettings+ :model/User}}segments.rest{:team "Graphy"diff --git a/frontend/src/metabase/api/ai-streaming/schemas.ts b/frontend/src/metabase/api/ai-streaming/schemas.tsindex e2fb5a8d2d7..148e824193d 100644--- a/frontend/src/metabase/api/ai-streaming/schemas.ts+++ b/frontend/src/metabase/api/ai-streaming/schemas.ts@@ -59,6 +59,8 @@ export type SearchResultItem = {display?: CardDisplayType;moderated_status?: string;database_id?: number;+ /** The base table of a measure or segment. */+ table_id?: number;database_schema?: string;database_name?: string;collection?: { id?: number; name: string };diff --git a/frontend/src/metabase/metabot/components/MetabotChat/MetabotChainOfThought/MetabotChainOfThought.unit.spec.tsx b/frontend/src/metabase/metabot/components/MetabotChat/MetabotChainOfThought/MetabotChainOfThought.unit.spec.tsxindex 3c56da1dcc7..f2d36dff0f6 100644--- a/frontend/src/metabase/metabot/components/MetabotChat/MetabotChainOfThought/MetabotChainOfThought.unit.spec.tsx+++ b/frontend/src/metabase/metabot/components/MetabotChat/MetabotChainOfThought/MetabotChainOfThought.unit.spec.tsx@@ -409,6 +409,52 @@ describe("MetabotChainOfThought", () => {expect(screen.getByText("PUBLIC", { exact: false })).toBeInTheDocument();});+ it("links measures to the explorer and segments through their base table", async () => {+ setup(+ chain({+ steps: [+ {+ kind: "tool",+ id: "t1",+ name: "search",+ title: "revenue",+ status: "ended",+ searchResults: {+ totalCount: 2,+ results: [+ {+ id: 7,+ type: "measure",+ name: "Total revenue",+ database_id: 1,+ table_id: 9,+ },+ {+ id: 8,+ type: "segment",+ name: "Big orders",+ database_id: 1,+ table_id: 9,+ },+ ],+ },+ },+ ],+ startedAtMs: 1000,+ endedAtMs: 2000,+ }),+ false,+ );+ await expandChain();+ await userEvent.click(screen.getByText("2 results"));+ expect(+ screen.getByRole("link", { name: /Total revenue/, hidden: true }),+ ).toHaveAttribute("href", "/explore?measureId=7");+ expect(+ screen.getByRole("link", { name: /Big orders/, hidden: true }),+ ).not.toHaveAttribute("href", "/404");+ });+it("labels an empty search with No results", async () => {setup(chain({diff --git a/frontend/src/metabase/metabot/components/MetabotChat/MetabotChainOfThought/SearchResults.tsx b/frontend/src/metabase/metabot/components/MetabotChat/MetabotChainOfThought/SearchResults.tsxindex eb8b07b590e..fba38801e1d 100644--- a/frontend/src/metabase/metabot/components/MetabotChat/MetabotChainOfThought/SearchResults.tsx+++ b/frontend/src/metabase/metabot/components/MetabotChat/MetabotChainOfThought/SearchResults.tsx@@ -13,7 +13,7 @@ import Animation from "metabase/css/core/animation.module.css";import { useGetIcon } from "metabase/hooks/use-icon";import type { MetabotChainStep } from "metabase/metabot/state";import { Text } from "metabase/ui";-import { modelToUrl } from "metabase/urls";+import { exploreMeasure, modelToUrl } from "metabase/urls";import S from "./MetabotChainOfThought.module.css";import { resultContextParts, toSearchModel } from "./utils";@@ -55,6 +55,19 @@ const ResultIcon = ({ result }: { result: SearchResultItem }) => {return <EntityIcon {...iconData} size={13} className={S.resultIcon} />;};+// Measures open in the explorer: their `modelToUrl` page is in Data Studio,+// which only admins and analysts can open.+const resultUrl = (result: SearchResultItem) =>+ result.type === "measure"+ ? exploreMeasure(result.id)+ : modelToUrl({+ id: result.id,+ model: toSearchModel(result.type),+ name: result.name,+ database_id: result.database_id,+ table_id: result.table_id,+ });+const SearchResultRow = ({result,index,@@ -65,12 +78,7 @@ const SearchResultRow = ({animate: boolean;}) => (<Link- to={modelToUrl({- id: result.id,- model: toSearchModel(result.type),- name: result.name,- database_id: result.database_id,- })}+ to={resultUrl(result)}className={cx(S.resultRow, animate && Animation.fadeIn)}style={animate ? { animationDelay: `${Math.min(index * 45, 360)}ms` } : undefineddiff --git a/resources/metabot/prompts/shared/prompt_snippets/data-sources.selmer b/resources/metabot/prompts/shared/prompt_snippets/data-sources.selmerindex b6a5c8af847..5af6368e8f9 100644--- a/resources/metabot/prompts/shared/prompt_snippets/data-sources.selmer+++ b/resources/metabot/prompts/shared/prompt_snippets/data-sources.selmer@@ -1,16 +1,6 @@-## Prefer curated entities over raw tables+## Prefer curated sources-Metrics are reusable calculations — same definition every team uses. Models are curated, analysis-ready datasets. Saved questions are previously-built answers — narrower than a model; reach for one when it directly answers the user's question, or use it as a starting point when its shape is close. Raw tables are the source. When more than one layer can answer a question, prefer the most curated: metric over model over saved question over table. Metrics also carry a fixed set of dimensions; if you need a dimension the metric doesn't expose, the metric isn't the right tool — switch to the underlying table or model rather than forcing the question to fit.--## A named measure is almost always a defined metric, measure, or segment — find it, don't recompute it--When the user names a business quantity — a **rate or ratio** ("open rate", "bounce rate", "churn rate", "conversion rate", "utilization rate"), an **aggregate with a name** ("average order value", "MRR", "active subscribers"), or a **named cohort/filter** ("hard bounces", "SUBSCRIBED customers", "high-value opportunities") — assume the organization has already defined it and reach for the definition rather than reconstructing it from raw columns:--- **Metrics** — a named aggregation that lives independently. Search for it (`entity_types: ["metric"]`, where your search tool supports that type); if one matches, reference it with `["metric", {}, "<portable_entity_id>"]` on its base table when building a notebook query. Don't rebuild it as `count` / `count-where` / a hand-rolled ratio.-- **Measures** — a named aggregation attached to a table. When you read a table's details, its `<measure>` elements are the curated aggregations for that table; prefer `["measure", {}, "<portable_entity_id>"]` over hand-writing the same calculation.-- **Segments** — a named, reusable filter attached to a table. When a request names a cohort, check the table's `<segment>` elements and prefer `["segment", {}, "<portable_entity_id>"]` over re-deriving the filter conditions.--Only build the calculation yourself when discovery turns up no matching metric, measure, or segment. (See the construct skills for the exact clause syntax — this opaque-id syntax is for notebook queries. When writing raw SQL there's no equivalent clause, so query the underlying table or an existing model that already encodes the same logic instead.) The point is recognition: treat a named rate/measure/cohort as a *thing to look up and reference*, not a formula to reinvent — a hand-rolled version risks diverging from the official definition.+Metrics are reusable calculations — same definition every team uses. Models are older saved datasets; treat one as a fallback, not a curated layer. Saved questions are previously-built answers; reach for one when it directly answers the user's question, or use it as a starting point when its shape is close. A table's `data_layer` is a quality preference (`final` over `internal` over `hidden`), not by itself an endorsement: use `is_curated` when present, or `data_authority` (`authoritative`), to identify a curated table. When more than one candidate can answer a question, prefer the most curated one. Search results are already ranked that way and, when available, expose `is_curated` plus a table's `data_layer` and `data_authority`. Metrics also carry a fixed set of dimensions; if you need a dimension the metric doesn't expose, the metric isn't the right tool — switch to its underlying table or model, preserving a source model's transformations when the metric is based on one, rather than forcing the question to fit.**A source the user names is authoritative.** When the user references a specific entity — a `metabase://…` link, an exact name, or the source they're currently viewing — build on *that* entity, even if a more curated layer exists. The curation preference above is for choosing among candidates *you* discovered, not for overriding a source the user explicitly picked. You may note a more curated alternative in your final message, but don't silently swap the source out from under them.diff --git a/resources/metabot/prompts/shared/prompt_snippets/discovery.selmer b/resources/metabot/prompts/shared/prompt_snippets/discovery.selmerindex b5e462fc55c..9eaf2e151ef 100644--- a/resources/metabot/prompts/shared/prompt_snippets/discovery.selmer+++ b/resources/metabot/prompts/shared/prompt_snippets/discovery.selmer@@ -7,7 +7,7 @@ You have two ways to find data, with different strengths:These play different roles, and the distinction matters:-- **Choosing which data source answers an analytical question → start with `search`.** When you don't know which database or collection holds the answer, topic search is the only way to surface the curated layer — models, and metrics when your search tool's scope includes them. (Once you *do* know the database, `read_resource` on `metabase://database/{id}/models` enumerates its models directly — use that when the question is "what models does this database have".) Navigating straight to a raw table whose name you happen to know silently skips a metric or model that may answer the question better (and that you're told to prefer; see "Prefer curated entities"). Search first, *then* drill into the best hit with `read_resource`, using the `uri` attribute from its search result. Recognizing a raw table's name is not a reason to skip the search.+- **Choosing which data source answers an analytical question → start with `search`.** When you don't know which database or collection holds the answer, topic search is the only way to reach curated content you cannot name; it ranks curated and Library items first, and each result says whether it is curated. (Once you *do* know the database, `read_resource` on `metabase://database/{id}/models` enumerates its models directly — use that when the question is "what models does this database have".) Navigating straight to a raw table whose name you happen to know silently skips curated content that may answer the question better (and that you're told to prefer; see "Prefer curated sources"). Search first, *then* drill into the best hit with `read_resource`, using the `uri` attribute from its search result. Recognizing a raw table's name is not a reason to skip the search.- **Enumerating a structure you can already name → navigate directly.** "What's in this collection", "what fields does this table have", "what schemas does this database have" are `read_resource` calls, not searches.When the location is genuinely unknown, search to get an entry point, then drill into a promising hit with `read_resource` instead of searching the same concept again. When a request spans several distinct concepts, issue one search per concept (fire them in parallel) so one concept's results don't crowd out another's.diff --git a/resources/metabot/prompts/shared/prompt_snippets/semantic-layer-discovery.selmer b/resources/metabot/prompts/shared/prompt_snippets/semantic-layer-discovery.selmernew file mode 100644index 00000000000..66cb6a8001e--- /dev/null+++ b/resources/metabot/prompts/shared/prompt_snippets/semantic-layer-discovery.selmer@@ -0,0 +1,9 @@+## A named measure is almost always a defined metric, measure, or segment — find it, don't recompute it++When the user names a business quantity — a **rate or ratio** ("open rate", "bounce rate", "churn rate", "conversion rate", "utilization rate"), an **aggregate with a name** ("average order value", "MRR", "active subscribers"), or a **named cohort/filter** ("hard bounces", "SUBSCRIBED customers", "high-value opportunities") — assume the organization has already defined it and reach for the definition rather than reconstructing it from raw columns:++- **Metrics** — a named aggregation that lives independently. Search for it (`entity_types: ["metric"]`); if one matches, reference it with `["metric", {}, "<portable_entity_id>"]` on its base table when building a notebook query. Don't rebuild it as `count` / `count-where` / a hand-rolled ratio.+- **Measures** — a named aggregation attached to a table. Search for it (`entity_types: ["measure"]`), or read the table's details: its `<measure>` elements are the curated aggregations for that table. Prefer `["measure", {}, "<portable_entity_id>"]` over hand-writing the same calculation.+- **Segments** — a named, reusable filter attached to a table. When a request names a cohort, search for it (`entity_types: ["segment"]`) or check the table's `<segment>` elements, and prefer `["segment", {}, "<portable_entity_id>"]` over re-deriving the filter conditions.++Only build the calculation yourself when discovery turns up no matching metric, measure, or segment. (See the construct skills for the exact clause syntax — this opaque-id syntax is for notebook queries. When writing raw SQL there's no equivalent clause, so query the underlying table or an existing model that already encodes the same logic instead.) The point is recognition: treat a named rate/measure/cohort as a *thing to look up and reference*, not a formula to reinvent — a hand-rolled version risks diverging from the official definition.diff --git a/resources/metabot/prompts/system/embedding-next.selmer b/resources/metabot/prompts/system/embedding-next.selmerindex c3c7a8dca26..7e015833398 100644--- a/resources/metabot/prompts/system/embedding-next.selmer+++ b/resources/metabot/prompts/system/embedding-next.selmer@@ -13,6 +13,7 @@ You are judged on whether the produced query matches the user's intent against t{% include "metabot/prompts/shared/results.selmer" %}{% include "metabot/prompts/shared/prompt_snippets/grounding.selmer" %}{% include "metabot/prompts/shared/prompt_snippets/data-sources.selmer" %}+{% include "metabot/prompts/shared/prompt_snippets/semantic-layer-discovery.selmer" %}{% include "metabot/prompts/shared/prompt_snippets/data-sources-curation-signals.selmer" %}{% include "metabot/prompts/shared/prompt_snippets/discovery.selmer" %}diff --git a/resources/metabot/prompts/system/internal.selmer b/resources/metabot/prompts/system/internal.selmerindex 7b047f02631..af224721ded 100644--- a/resources/metabot/prompts/system/internal.selmer+++ b/resources/metabot/prompts/system/internal.selmer@@ -12,6 +12,7 @@ A single objective: **the query you build should return the data the user asked{% include "metabot/prompts/shared/results.selmer" %}{% include "metabot/prompts/shared/prompt_snippets/grounding.selmer" %}{% include "metabot/prompts/shared/prompt_snippets/data-sources.selmer" %}+{% include "metabot/prompts/shared/prompt_snippets/semantic-layer-discovery.selmer" %}{% include "metabot/prompts/shared/prompt_snippets/data-sources-curation-signals.selmer" %}{% if has_notebook_query %}{% include "metabot/prompts/shared/prompt_snippets/mbql-shape.selmer" %}diff --git a/resources/metabot/prompts/system/natural-language-querying-fallback.selmer b/resources/metabot/prompts/system/natural-language-querying-fallback.selmerindex 4dcc81ec0b6..9714010b592 100644--- a/resources/metabot/prompts/system/natural-language-querying-fallback.selmer+++ b/resources/metabot/prompts/system/natural-language-querying-fallback.selmer@@ -13,6 +13,7 @@ You are judged on whether the produced query matches the user's intent against t{% include "metabot/prompts/shared/results.selmer" %}{% include "metabot/prompts/shared/prompt_snippets/grounding.selmer" %}{% include "metabot/prompts/shared/prompt_snippets/data-sources.selmer" %}+{% include "metabot/prompts/shared/prompt_snippets/semantic-layer-discovery.selmer" %}{% include "metabot/prompts/shared/prompt_snippets/data-sources-curation-signals.selmer" %}# Finding data: search for the right entity, then navigate indiff --git a/resources/metabot/prompts/system/slackbot.selmer b/resources/metabot/prompts/system/slackbot.selmerindex 26865590f5e..7450d89cda5 100644--- a/resources/metabot/prompts/system/slackbot.selmer+++ b/resources/metabot/prompts/system/slackbot.selmer@@ -7,6 +7,7 @@ Be direct and concise. Get users to insights efficiently while providing transpa{% include "metabot/prompts/shared/results.selmer" %}{% include "metabot/prompts/shared/prompt_snippets/grounding.selmer" %}{% include "metabot/prompts/shared/prompt_snippets/data-sources.selmer" %}+{% include "metabot/prompts/shared/prompt_snippets/semantic-layer-discovery.selmer" %}{% include "metabot/prompts/shared/prompt_snippets/data-sources-curation-signals.selmer" %}<slack_rules>@@ -232,7 +233,7 @@ When users ask business questions ("What's our revenue?", "Show customer growth"When users ask for data exploration ("Show me all orders", "List customers", "What fields are available?"):1. **Understand the scope** - Determine if they want raw data listing or analytical insights-2. **Find appropriate data source** - Look for models first (curated for analysis), then tables (raw data)+2. **Find appropriate data source** - Look for curated tables first, then other tables; treat models as a fallback (see "Prefer curated sources")3. **Start simple** - Use basic aggregations and groupings that provide business value4. **Validate field formats** - Before any filtering, check actual field values to understand their structurediff --git a/src/metabase/collections/models/collection.clj b/src/metabase/collections/models/collection.cljindex 1f7e1e307b4..14a94775c01 100644--- a/src/metabase/collections/models/collection.clj+++ b/src/metabase/collections/models/collection.clj@@ -245,6 +245,14 @@parse-long)](root-collection-type-by-id root-id))))+(defn table-root-collection-type+ "Return the `:type` of the top-level ancestor of the collection that holds a record's parent table.+ Reads `:table_collection_id`, `:table_collection_location`, `:table_collection_type` from the ingestion record map."+ [{:keys [table_collection_id table_collection_location table_collection_type]}]+ (root-collection-type {:collection_id table_collection_id+ :collection_location table_collection_location+ :collection_type table_collection_type}))+(def library-entity-id"The entity_id for the Library collection.""librarylibrarylibrary")diff --git a/src/metabase/measures/models/measure.clj b/src/metabase/measures/models/measure.cljindex ee527a88f24..318e30ecfb1 100644--- a/src/metabase/measures/models/measure.clj+++ b/src/metabase/measures/models/measure.clj@@ -4,6 +4,7 @@(:require[medley.core :as m][metabase.api.common :as api]+ [metabase.collections.models.collection :as collection][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.schema :as lib.schema]@@ -247,14 +248,34 @@:creator-id true:database-id :table.db_id:created-at true- :updated-at true}+ :updated-at true+ ;; A measure on a Library table inherits the table's library ranking boost and curated status.+ :root-collection-type {:fn collection/table-root-collection-type}}:search-terms [:name :description]- :render-terms {:table-id :table_id- :table_description :table.description- :table_name :table.name- :table_display_name :table.display_name- :table_schema :table.schema}- :joins {:table [:model/Table [:= :table.id :this.table_id]]}})+ :render-terms {:table-id :table_id+ :table_description :table.description+ :table_name :table.name+ :table_display_name :table.display_name+ :table_schema :table.schema+ ;; The parent table's collection, for [[collection/table-root-collection-type]].+ ;; Publishing is a user setting, so the published state and collection come from the table's+ ;; user settings when they are set, as in `metabase.warehouse-schema-overlay.core/table-query`.+ ;; Listing these terms also reindexes a measure when its table moves into or out of the Library.+ :table-collection-id [:case+ [:not= :settings.is_published nil] :settings.collection_id+ :else :table.collection_id]+ :table-is-published [:coalesce :settings.is_published :table.is_published]+ :table-collection-location :collection.location+ :table-collection-type :collection.type}+ ;; Only published tables count as library content, matching the table spec's own collection join.+ :joins {:table [:model/Table [:= :table.id :this.table_id]]+ :settings [:model/TableUserSettings [:= :settings.table_id :this.table_id]]+ :collection [:model/Collection+ [:and+ [:= [:coalesce :settings.is_published :table.is_published] true]+ [:= :collection.id [:case+ [:not= :settings.is_published nil] :settings.collection_id+ :else :table.collection_id]]]]}});;; ------------------------------------------------- Dimension Persistence --------------------------------------------------diff --git a/src/metabase/metabot/db.clj b/src/metabase/metabot/db.cljindex fa8dcd48f55..8fe9e4f2af1 100644--- a/src/metabase/metabot/db.clj+++ b/src/metabase/metabot/db.clj@@ -1072,3 +1072,13 @@"The ID, name, description, Table ID, and entity ID of the Segments with `ids`."[ids :- [:sequential ms/PositiveInt]](t2/select [:model/Segment :id :name :description :table_id :entity_id] :id [:in ids]))++(mu/defn measure-entity-ids+ "A map of Measure ID to entity ID for `ids`."+ [ids :- [:sequential ::lib.schema.id/measure]]+ (t2/select-pk->fn :entity_id :model/Measure :id [:in ids]))++(mu/defn segment-entity-ids+ "A map of Segment ID to entity ID for `ids`."+ [ids :- [:sequential ::lib.schema.id/segment]]+ (t2/select-pk->fn :entity_id :model/Segment :id [:in ids]))diff --git a/src/metabase/metabot/tools/search.clj b/src/metabase/metabot/tools/search.cljindex 5bd1e12e7ed..8d5c4a3bd28 100644--- a/src/metabase/metabot/tools/search.clj+++ b/src/metabase/metabot/tools/search.clj@@ -25,7 +25,16 @@(set! *warn-on-reflection* true)(def ^:private metabot-search-models- (sorted-set "card" "dashboard" "database" "dataset" "document" "metric" "table" "transform"))+ (sorted-set "card"+ "dashboard"+ "database"+ "dataset"+ "document"+ "measure"+ "metric"+ "segment"+ "table"+ "transform"))(defn- postprocess-search-result"Transform a single search result to match the appropriate entity-specific schema."@@ -78,6 +87,16 @@(m/assoc-some :curated curated:can_write can_write))+ ("measure" "segment")+ ;; Measures and segments belong to a table, not a collection, so they carry the base table from the index row.+ ;; Their `:portable_entity_id` comes later, from [[enrich-with-portable-entity-ids]].+ (-> common-fields+ (merge {:database_id (:database_id result)+ :base_table_id (:table_id result)+ :base_table_name (:table_name result)+ :base_table_schema (:table_schema result)})+ (m/assoc-some :curated curated))+"transform"(merge common-fields{:database_id (:database_id result)})@@ -120,25 +139,35 @@(m/assoc-some :database_engine engine)(m/assoc-some :database_name db-name))))))))+(def ^:private entity-id-lookups+ "Each pair holds the entity types that share an ID space and the function that looks up their entity IDs.+ The function takes a seq of IDs and returns a map of ID to entity ID."+ [[#{"metric" "model" "question"} metabot.db/card-entity-ids]+ [#{"measure"} metabot.db/measure-entity-ids]+ [#{"segment"} metabot.db/segment-entity-ids]])+(defn- enrich-with-portable-entity-ids- "Attach `:portable_entity_id` (the card's `entity_id` NanoID) to saved-question, model,- and metric search results so the LLM can use it verbatim as `source-card:` (for- questions/models) or inside a `[metric, {}, <entity_id>]` aggregation clause (for- metrics) without a follow-up `read_resource` round-trip."+ "Attach `:portable_entity_id` (the entity's `entity_id` NanoID) to saved-question, model, metric, measure and+ segment search results that do not have one yet.+ The LLM can then use it verbatim as `source-card:` (questions and models) or inside a+ `[metric|measure|segment, {}, <entity_id>]` clause without a follow-up `read_resource` round-trip."[results]- (let [carded-types #{"question" "model" "metric"}- card-ids (->> results- (filter #(carded-types (:type %)))- (keep :id)- distinct)- id->eid (when (seq card-ids)- (metabot.db/card-entity-ids card-ids))]+ ;; The search index doesn't carry `entity_id`, so each lookup is one query for all the types that share it.+ (let [type->id->eid (u/for-map [[types ids->eids] entity-id-lookups+ :let [ids (->> results+ (filter (comp types :type))+ (remove :portable_entity_id)+ (keep :id)+ distinct)]+ :when (seq ids)+ :let [id->eid (ids->eids ids)]+ type types]+ [type id->eid])](cond->> results- (seq id->eid) (mapv (fn [r]- (if-let [eid (and (carded-types (:type r))- (get id->eid (:id r)))]- (assoc r :portable_entity_id eid)- r))))))+ (seq type->id->eid) (mapv (fn [{:keys [id type] :as r}]+ (if-let [eid (get-in type->id->eid [type id])]+ (assoc r :portable_entity_id eid)+ r))))))(defn- enrich-with-metric-base-tables"Attach base-table info (`:base_table_id`, `:base_table_name`, `:base_table_schema`,@@ -189,6 +218,17 @@(assoc :base_table_portable_fk [database_name schema table-name]))result))))))+(defn- enrich-with-measure-segment-base-tables+ "Attach `:base_table_portable_fk` (`[database_name schema table]`) to measure and segment results.+ The agent queries a measure or segment against that table, as it does for a metric.+ Requires `:database_name` to already be set on each result, by [[enrich-with-database-engines]]."+ [results]+ (mapv (fn [{:keys [type database_name base_table_schema base_table_name] :as r}]+ (if (and (#{"measure" "segment"} type) database_name base_table_name)+ (assoc r :base_table_portable_fk [database_name base_table_schema base_table_name])+ r))+ results))+(defn- remove-unreadable-transforms"Remove transforms from search results that the user cannot read.This filters out transforms where the user doesn't have access to the source tables/database."@@ -283,7 +323,7 @@(reciprocal-rank-fusion result-lists))))(defn search- "Search for data sources (tables, models, cards, dashboards, metrics, transforms) in Metabase.+ "Search for data sources (tables, models, cards, dashboards, metrics, measures, segments, transforms) in Metabase.Abstracted from the API endpoint logic.Optional filter keys threaded straight into the search context: `created-by` (set of user ids),@@ -325,6 +365,8 @@;; replace it. Unconfined, the caller's collection-id applies.confined-id (when (or embedded-metabot? (= profile-id "nlq"))(:collection_id metabot))+ ;; Measures and segments belong to a table, not a collection, so a search scoped to a collection+ ;; never returns them, even though the search tools list them as entity types.collection-id (or confined-id collection-id)limit (or limit 50)ranked-fn (fn [search-string search-engine]@@ -413,6 +455,7 @@enrich-with-database-enginesenrich-with-portable-entity-idsenrich-with-metric-base-tables+ enrich-with-measure-segment-base-tables(validate-and-enrich-documents (boolean archived)))(vary-meta assoc :total total))))@@ -490,17 +533,6 @@:base_table_name (:name t):base_table_schema (:schema t)))))))-(defn- enrich-with-measure-segment-base-tables- "Assemble `:base_table_portable_fk [database_name schema table]` for measure/segment results once- `:database_name` is set (by [[enrich-with-database-engines]]), giving the agent the table to query a- measure/segment against — the same affordance metrics get from [[enrich-with-metric-base-tables]]."- [results]- (mapv (fn [{:keys [type database_name base_table_schema base_table_name] :as r}]- (if (and (#{"measure" "segment"} type) database_name base_table_name)- (assoc r :base_table_portable_fk [database_name base_table_schema base_table_name])- r))- results))-(defn ref-model->entity-type"Normalize an entity ref's `:model` string to the agent-facing entity type: plain cards are`\"question\"` everywhere the agent sees them (`read_resource` URIs, search results)."@@ -577,11 +609,13 @@(defn- search-result->item"Trim a search result to the fields the chain-of-thought results card renders.`:display` (a question's viz type) and `:moderated_status` let the client pick the- exact entity icon, matching the app's search/command-palette icons."+ exact entity icon, matching the app's search/command-palette icons.+ `:table_id` is the base table of a measure or segment, which the client needs to link to it."[r](-> (select-keys r [:id :type :name :display_name :database_id :database_schema :database_name])(m/assoc-some :display (some-> (:display r) name):moderated_status (:moderated_status r)+ :table_id (when (#{"measure" "segment"} (:type r)) (:base_table_id r)):collection (some-> (:collection r) (select-keys [:id :name])))))(defn- do-search@@ -620,12 +654,22 @@(def ^:private limit-schema[:maybe [:int {:min 1 :max max-search-limit :description limit-desc}]])+(def ^:private general-entity-types+ "The entity types the general and NLQ search tools accept."+ (sorted-set "dashboard"+ "document"+ "measure"+ "metric"+ "model"+ "question"+ "segment"+ "table"))+(def ^:private search-schema[:map {:closed true}[:semantic_queries {:optional true :feature :semantic-search} semantic-queries-schema][:keyword_queries {:optional true} keyword-queries-schema]- [:entity_types {:optional true}- (entity-types-schema "table" "model" "metric" "dashboard" "document" "question")]+ [:entity_types {:optional true} (apply entity-types-schema general-entity-types)][:limit {:optional true} limit-schema]])(defn- search-display@@ -645,9 +689,9 @@:scope scope/agent-search:title-fn search-display}search-tool- "Find tables, models, metrics, dashboards, documents, and saved questions by topic across the instance. Use it when you don't know where something lives; once you have a hit, drill into it with read_resource rather than searching the same concept again."+ "Find tables, models, metrics, measures, segments, dashboards, documents, and saved questions by topic across the instance. Use it when you don't know where something lives; once you have a hit, drill into it with read_resource rather than searching the same concept again."[args :- search-schema]- (do-search "search" (sorted-set "dashboard" "document" "metric" "model" "question" "table") {} args))+ (do-search "search" general-entity-types {} args))(def ^:private sql-search-schema[:map {:closed true}@@ -669,8 +713,7 @@[:map {:closed true}[:semantic_queries {:optional true :feature :semantic-search} semantic-queries-schema][:keyword_queries {:optional true} keyword-queries-schema]- [:entity_types {:optional true}- (entity-types-schema "table" "model" "metric" "question" "dashboard" "document")]+ [:entity_types {:optional true} (apply entity-types-schema general-entity-types)][:limit {:optional true} limit-schema]])(mu/defn ^{:tool-name "search"@@ -679,11 +722,10 @@nlq-search-tool"Find NLQ-queryable data sources by topic, or find dashboards and documents as save destinations."[{:keys [entity_types] :as args} :- nlq-search-schema]- (let [allowed-types (sorted-set "dashboard" "document" "metric" "model" "question" "table")- args (cond-> args- (not (seq entity_types))- (assoc :entity_types ["metric" "model" "question" "table"]))]- (do-search "NLQ search" allowed-types {:profile-id "nlq"} args)))+ (let [args (cond-> args+ (not (seq entity_types))+ (assoc :entity_types ["metric" "measure" "segment" "model" "question" "table"]))]+ (do-search "NLQ search" general-entity-types {:profile-id "nlq"} args)))(def ^:private transform-search-schema[:map {:closed true}diff --git a/src/metabase/metabot/tools/shared/instructions.clj b/src/metabase/metabot/tools/shared/instructions.cljindex c7c78b2f87b..c6598be94a9 100644--- a/src/metabase/metabot/tools/shared/instructions.clj+++ b/src/metabase/metabot/tools/shared/instructions.clj@@ -8,7 +8,7 @@"Search results ranked by relevance (highest first).When using results:-- Prioritize verified dashboards, questions, and metrics over raw data exploration when they match the user's request+- Prioritize verified dashboards, questions, and metrics over raw data exploration when they match the user's request; treat models as a fallback- Check available fields or metadata of promising results before recommending them to ensure they contain needed data- **Be proactive with clear matches**: When the top result(s) have names or descriptions that clearly match the user's request, use them to achieve the users goal first, then list alternatives at the end. Don't ask for confirmation when there's an obvious best choice.- **Collection metadata**: Results may include collection information (name, description, authority level) that can help you determine whether the result is relevant to the user's query. Use this context to assess relevance.diff --git a/src/metabase/search/config.clj b/src/metabase/search/config.cljindex 0ec2bdef1ef..743350d946d 100644--- a/src/metabase/search/config.clj+++ b/src/metabase/search/config.clj@@ -73,8 +73,15 @@(def curated-search-models"Search models that can carry a curation signal, so the `:curated?` filter restricts to these (andkeeps them consistent across the appdb and in-place engines). Includes `table`, which is exactly why- curated content stays visible where the older verified-only filter dropped it (BOT-1536)."- #{"card" "dataset" "metric" "dashboard" "table"})+ curated content stays visible where the older verified-only filter dropped it (BOT-1536).+ Measures and segments count as curated when their parent table is published into the Library."+ #{"card"+ "dashboard"+ "dataset"+ "measure"+ "metric"+ "segment"+ "table"})(def models-search-order"The order of this list influences the order of the results: items earlier in the@@ -103,8 +110,8 @@;; Curation badges act as tie-breakers by default; the :data-picker context boosts them (to 35 each).:official-collection 1:verified 1- ;; :library is a data-layer curation signal relevant only when picking a data source, so it is off by- ;; default; the :data-picker context opts in, at a curation-tier level that an exact match can overpower.+ ;; Library membership is a curation signal for choosing data sources, so :library is off by default.+ ;; The :data-picker and :metabot contexts turn it on, at a level an exact match can override.:library 0;; RRF is the "Reciprocal Rank Fusion" score used by the semantic search backend to blend semantic and keyword scores:rrf 500@@ -153,11 +160,51 @@;; - `custom-viz.cy.spec.ts`;; - `search-snowplow.cy.spec.js`:metabot- {:data-layer 33+ ;; Metabot searches for data sources, so it uses the data picker's library boost.+ ;; General search needs the boost too, because library-only retrieval falls back to it.+ {:library 80+ ;; An exact name match must win.+ ;; So :exact is larger than the library boost, the largest type boost and the tie-breakers combined.+ :exact 150+ ;; Popularity and recency only break ties within a tier.+ ;; Recency alone (0.4) stays below the narrowest tier gap (1).+ ;; All of them together (1.9) stay below the gap that keeps a metric over a final-layer table (2).+ :view-count 0.5+ :bookmarked 0.5+ :mine 0.5+ :recency 0.2+ :user-recency 0.2+ ;; Relevance has to be able to cross that gap, or any metric that matches at all outranks a better-matching table.+ ;; The scorer maps cosine distance d to 1 - d/2, so at double the default a 0.2 difference in distance (2) does it.+ :semantic-distance 20+ :data-layer 33:data-layer/final 1 ; ≈ 33+ ;; New tables use the internal layer by default, so an ordinary table gets this score even if nobody has vetted it.+ ;; That score ranks it above dashboards, questions, and models.:data-layer/internal 0.3 ; ≈ 10:data-layer/hidden 0.03 ; ≈ 1- }})+ ;; Semantic-layer content ranks above saved questions, and saved questions above models.+ ;; Tables are ranked mostly by :data-layer; their small type boost only keeps a table above a model.+ ;; A metric beats every table on an equal match: the semantic layer leads a final-layer table (33 + 1) by 2.+ ;; That is more than the tie-breakers add together, but small enough that a clearly better match wins.+ ;;+ ;; Every search model is listed, because [[models-search-order]] supplies a fallback boost for any left out.+ ;; Collections and actions are here because the Agent API and MCP search use this context and return them.+ :model 10+ :model/metric 3.6 ; = 36+ :model/measure 3.6 ; = 36+ :model/segment 3.6 ; = 36+ :model/dashboard 0.3 ; = 3+ :model/card 0.3 ; = 3+ :model/table 0.1 ; = 1+ :model/collection 0.1 ; = 1+ :model/dataset 0+ :model/document 0+ :model/transform 0+ :model/database 0+ :model/action 0+ :model/exploration 0+ :model/indexed-entity 0}})(def known-rankers"Scorer keys the weights API accepts as overrides: the union across [[static-default-weights]] and everydiff --git a/src/metabase/search/in_place/filter.clj b/src/metabase/search/in_place/filter.cljindex c8b1fd0d64a..7304a0d4918 100644--- a/src/metabase/search/in_place/filter.clj+++ b/src/metabase/search/in_place/filter.clj@@ -237,6 +237,18 @@(sql.helpers/where query (cond-> [:or authoritative]published-final (conj published-final)))))+(doseq [model ["measure" "segment"]]+ (defmethod build-optional-filter-query [:curated model]+ [_filter _model query curated?]+ (assert (true? curated?) "filter for non-curated content is not supported")+ ;; A measure or segment is curated when its parent table is published into the Library.+ ;; This engine has no index row to read the table's root collection type from, so it only checks that the+ ;; parent table is published, and only with the :library feature, like the table filter above.+ ;; The legacy query joins the parent table as `table`; see metabase.search.in-place.legacy/search-query-for-model.+ (sql.helpers/where query (if (premium-features/has-feature? :library)+ [:= :table.is_published true]+ false-clause))))+;; Created at filters(defn- date-range-filter-clausediff --git a/src/metabase/search/in_place/scoring.clj b/src/metabase/search/in_place/scoring.cljindex 0d55b12e562..3a02b9edfaa 100644--- a/src/metabase/search/in_place/scoring.clj+++ b/src/metabase/search/in_place/scoring.clj@@ -109,7 +109,20 @@Get the score for a result with `score-and-result`, and efficiently get the most relevant results with`top-results`.- Some of the scorers can be tweaked with configuration in [[metabase.search.config]]."+ Some of the scorers can be tweaked with configuration in [[metabase.search.config]].++ ## Divergence from the appdb engine++ This engine ranks every search the same way, so it ignores the per-context weights in+ [[metabase.search.config/static-context-weights]].+ In particular, the `:metabot` and `:data-picker` contexts get none of their ranking behavior here:+ - Entity types are ordered by [[metabase.search.config/models-search-order]], not by `:model/*` weights,+ so dashboards outrank metrics and models outrank tables.+ - Library membership is not scored.+ - A table's `data_layer` is not scored.+ - Official-collection and verified scores use their default weights, not the context's boosts.++ Results also omit `curated`."(:require[clojure.string :as str][java-time.api :as t]diff --git a/src/metabase/search/spec.clj b/src/metabase/search/spec.cljindex bdb26382c51..1cdd1794042 100644--- a/src/metabase/search/spec.clj+++ b/src/metabase/search/spec.clj@@ -90,7 +90,19 @@;; `:document` is the document model's prose-mirror body: it's indexed as searchable text (via;; ast->text) but the raw JSON should never be echoed back in the search response or bloat the index row.;; `:data_layer` also stays IN: Metabot surfaces it on table results so the LLM sees a table's data layer.- #{:pinned :view_count :last_viewed_at :native_query :dataset_query :document :exploration_id})+ ;; The `:table_collection_*` and `:table_is_published` render terms only feed the measure and segment specs'+ ;; `:root-collection-type` function and their reindex hooks, so they stay out of the response.+ #{:dataset_query+ :document+ :exploration_id+ :last_viewed_at+ :native_query+ :pinned+ :table_collection_id+ :table_collection_location+ :table_collection_type+ :table_is_published+ :view_count})(def attr-types"The abstract types of each attribute."diff --git a/src/metabase/segments/models/segment.clj b/src/metabase/segments/models/segment.cljindex 1bc9c181285..8f097a9bee2 100644--- a/src/metabase/segments/models/segment.clj+++ b/src/metabase/segments/models/segment.clj@@ -4,6 +4,7 @@(:require[medley.core :as m][metabase.api.common :as api]+ [metabase.collections.models.collection :as collection][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.schema :as lib.schema]@@ -264,11 +265,31 @@:database-id :table.db_id;; should probably change this, but will break legacy search tests:created-at false- :updated-at true}+ :updated-at true+ ;; A segment on a Library table inherits the table's library ranking boost and curated status.+ :root-collection-type {:fn collection/table-root-collection-type}}:search-terms [:name :description]- :render-terms {:table-id :table_id- :table_description :table.description- :table_name :table.name- :table_display_name :table.display_name- :table_schema :table.schema}- :joins {:table [:model/Table [:= :table.id :this.table_id]]}})+ :render-terms {:table-id :table_id+ :table_description :table.description+ :table_name :table.name+ :table_display_name :table.display_name+ :table_schema :table.schema+ ;; The parent table's collection, for [[collection/table-root-collection-type]].+ ;; Publishing is a user setting, so the published state and collection come from the table's+ ;; user settings when they are set, as in `metabase.warehouse-schema-overlay.core/table-query`.+ ;; Listing these terms also reindexes a segment when its table moves into or out of the Library.+ :table-collection-id [:case+ [:not= :settings.is_published nil] :settings.collection_id+ :else :table.collection_id]+ :table-is-published [:coalesce :settings.is_published :table.is_published]+ :table-collection-location :collection.location+ :table-collection-type :collection.type}+ ;; Only published tables count as library content, matching the table spec's own collection join.+ :joins {:table [:model/Table [:= :table.id :this.table_id]]+ :settings [:model/TableUserSettings [:= :settings.table_id :this.table_id]]+ :collection [:model/Collection+ [:and+ [:= [:coalesce :settings.is_published :table.is_published] true]+ [:= :collection.id [:case+ [:not= :settings.is_published nil] :settings.collection_id+ :else :table.collection_id]]]]}})diff --git a/test/metabase/mcp/v2/tools/search_test.clj b/test/metabase/mcp/v2/tools/search_test.cljindex 316a52b696a..9bbd3ca69f3 100644--- a/test/metabase/mcp/v2/tools/search_test.clj+++ b/test/metabase/mcp/v2/tools/search_test.clj@@ -4,6 +4,7 @@[clojure.test :refer :all][metabase.activity-feed.core :as activity-feed][metabase.activity-feed.models.recent-views :as recent-views]+ [metabase.collections.test-utils :as collections.tu][metabase.mcp.v2.message :as message][metabase.mcp.v2.registry :as registry][metabase.mcp.v2.test-util :as v2.tu]@@ -493,6 +494,26 @@(is (some #(and (= "table" (:type %)) (= table-id (:id %))) rows)(str "the table is in the results: " (pr-str rows))))))))))+(deftest library-table-ranks-first-test+ (testing "search ranks a table in the Library above a final-layer table outside it"+ (search.tu/with-appdb-search-if-available-without-fallback+ (collections.tu/with-library [{data :data}]+ ;; The final layer puts the outside table first unless the Library boost applies.+ (mt/with-temp [:model/Table {outside-id :id} {:db_id (mt/id)+ :name "mcpboost outside"+ :active true+ :data_layer :final}+ :model/Table {inside-id :id} {:db_id (mt/id)+ :name "mcpboost inside"+ :active true+ :is_published true+ :collection_id (:id data)}]+ (mt/with-current-user (mt/user->id :crowberto)+ (let [content (tools.search/search-tool {:term_queries ["mcpboost"] :type ["table"]}+ {:token-scopes #{"agent:content:read"}})]+ (is (= [inside-id outside-id]+ (->> content :content first :text v2.tu/strip-data-boundary json/decode+kw :data (map :id)))))))))))+(deftest collection-row-path-omits-unreadable-ancestors-test(testing "GHY-4137: a collection row builds its path from its own :location — that path mustalso omit unreadable ancestors"diff --git a/test/metabase/metabot/tools/search_test.clj b/test/metabase/metabot/tools/search_test.cljindex ca6050e3f6f..81ab35ed89d 100644--- a/test/metabase/metabot/tools/search_test.clj+++ b/test/metabase/metabot/tools/search_test.clj@@ -55,7 +55,13 @@:moderated_status "verified"}(#'search/search-result->item{:id 7 :type "question" :name "Revenue" :display :line- :moderated_status "verified"})))))+ :moderated_status "verified"}))))+ (testing "carries the base table of a measure or segment, which the client needs to link to it"+ (is (= [{:id 8 :type "measure" :name "Revenue" :database_id 2 :table_id 10}+ {:id 9 :type "segment" :name "Big orders" :database_id 2 :table_id 10}]+ (map #'search/search-result->item+ [{:id 8 :type "measure" :name "Revenue" :database_id 2 :base_table_id 10}+ {:id 9 :type "segment" :name "Big orders" :database_id 2 :base_table_id 10}])))))(deftest ^:parallel reciprocal-rank-fusion-test(testing "Basic RRF with single list"@@ -394,6 +400,30 @@:created_at "2024-01-06"}](is (= expected (#'search/postprocess-search-result result))))))+(deftest ^:parallel postprocess-search-result-measure-segment-test+ (testing "measure and segment results carry their base table from the index row, and no collection fields"+ (doseq [model ["measure" "segment"]]+ (is (= {:id 5+ :type model+ :name "Big Orders"+ :description "desc"+ :updated_at "2024-01-05"+ :created_at "2024-01-05"+ :database_id 2+ :base_table_id 10+ :base_table_name "ORDERS"+ :base_table_schema "PUBLIC"}+ (#'search/postprocess-search-result {:model model+ :id 5+ :name "Big Orders"+ :description "desc"+ :updated_at "2024-01-05"+ :created_at "2024-01-05"+ :database_id 2+ :table_id 10+ :table_name "ORDERS"+ :table_schema "PUBLIC"}))))))+(deftest search-native-query-test(mt/with-test-user :rasta(with-redefs [perms/impersonated-user? (fn [] false)@@ -423,13 +453,19 @@(with-redefs [perms/impersonated-user? (fn [] false)perms/sandboxed-user? (fn [] false)api/*current-user-id* 1]- (testing "nlq-search-tool with no entity_types searches only table/model/metric/question"+ (testing "nlq-search-tool with no entity_types searches only the queryable types"(let [captured (atom nil)](mt/with-dynamic-fn-redefs [search-core/ranked-results (fn [context](reset! captured (:models context))[])](search/nlq-search-tool {:keyword_queries ["x"]}))- (is (= #{"table" "dataset" "metric" "card"} @captured))+ (is (= #{"card"+ "dataset"+ "measure"+ "metric"+ "segment"+ "table"}+ @captured))(is (not (contains? @captured "dashboard")))(is (not (contains? @captured "transform")))(is (not (contains? @captured "database")))))@@ -454,6 +490,13 @@[])](search/nlq-search-tool {:keyword_queries ["x"] :entity_types ["metric"]}))(is (= #{"metric"} @captured))))+ (testing "agent-supplied measure and segment types reach the engine unchanged"+ (let [captured (atom nil)]+ (mt/with-dynamic-fn-redefs [search-core/ranked-results (fn [context]+ (reset! captured (:models context))+ [])]+ (search/search-tool {:keyword_queries ["x"] :entity_types ["measure" "segment"]}))+ (is (= #{"measure" "segment"} @captured))))(testing "NLQ search accepts document and dashboard destination types"(let [captured (atom nil)](mt/with-dynamic-fn-redefs [search-core/ranked-results (fn [context]@@ -660,6 +703,45 @@(is (some? dash-res) "expected the dashboard to appear in search results")(is (not (contains? dash-res :portable_entity_id))))))))))+(deftest measure-segment-search-test+ (testing "measure and segment hits carry a portable entity id and their base table's portable FK, as metrics do"+ (mt/with-test-user :crowberto+ (search.tu/with-temp-index-table+ (mt/with-temp [:model/Measure+ {measure-id :id measure-eid :entity_id}+ {:name "MeasureSearch Sample Revenue"+ :table_id (mt/id :orders)}++ :model/Segment+ {segment-id :id segment-eid :entity_id}+ {:name "MeasureSearch Sample Big Orders"+ :table_id (mt/id :orders)}]+ (let [db-name (t2/select-one-fn :name :model/Database :id (mt/id))+ by-key (into {} (map (juxt (juxt :type :id) identity))+ (search/search {:term-queries ["MeasureSearch Sample"]}))]+ (is (=? {:type "measure"+ :name "MeasureSearch Sample Revenue"+ :database_id (mt/id)+ :database_name db-name+ :portable_entity_id measure-eid+ :base_table_id (mt/id :orders)+ :base_table_portable_fk [db-name "PUBLIC" "ORDERS"]}+ (get by-key ["measure" measure-id])))+ (is (=? {:type "segment"+ :name "MeasureSearch Sample Big Orders"+ :database_id (mt/id)+ :database_name db-name+ :portable_entity_id segment-eid+ :base_table_id (mt/id :orders)+ :base_table_portable_fk [db-name "PUBLIC" "ORDERS"]}+ (get by-key ["segment" segment-id])))+ (testing "the XML the agent sees carries a read_resource uri and the portable ids"+ (let [xml (llm-shape/search-results->xml (vals by-key))]+ (is (str/includes? xml (str "<measure id=\"" measure-id "\" uri=\"metabase://measure/" measure-id "\"")))+ (is (str/includes? xml (str "<segment id=\"" segment-id "\" uri=\"metabase://segment/" segment-id "\"")))+ (is (str/includes? xml (str "portable_entity_id=\"" measure-eid "\"")))+ (is (str/includes? xml "base_table_fully_qualified_name=\"PUBLIC.ORDERS\""))))))))))+(deftest entity-refs->search-results-test(testing "hydrates {:model :id} refs (as stored by the semantic layer) into enriched search records"(mt/with-test-user :crowbertodiff --git a/test/metabase/search/appdb/scoring_test.clj b/test/metabase/search/appdb/scoring_test.cljindex 136593fcf58..cc0bf8a7a37 100644--- a/test/metabase/search/appdb/scoring_test.clj+++ b/test/metabase/search/appdb/scoring_test.clj@@ -5,6 +5,7 @@[clojure.string :as str][clojure.test :refer :all][metabase.app-db.core :as mdb]+ [metabase.collections.test-utils :as collections.tu][metabase.search.appdb.core :as search.appdb.core][metabase.search.appdb.index :as search.index][metabase.search.appdb.scoring :as scoring]@@ -380,6 +381,47 @@(is (= [false false true true true true true](map in-library? (with-weights {:library -1} (search-results* "card"))))))))))))+(deftest measure-segment-library-test+ ;; Index table first, before `with-temp` opens its transaction; see [[library-test]].+ (search.tu/with-temp-index-table+ (testing "measures and segments on a published Library table inherit its library boost and curated status"+ (collections.tu/with-library [{data :data}]+ (mt/with-temp [:model/Table lib-tbl {:name "lib_tbl" :is_published true :collection_id (:id data)}+ :model/Table plain-tbl {:name "plain_tbl"}+ :model/Measure _ {:name "foo measure lib" :table_id (:id lib-tbl)}+ :model/Measure _ {:name "foo measure plain" :table_id (:id plain-tbl)}+ :model/Segment _ {:name "foo segment lib" :table_id (:id lib-tbl)}+ :model/Segment _ {:name "foo segment plain" :table_id (:id plain-tbl)}]+ (let [results (search-results** "foo" {:context :metabot :models #{"measure" "segment"}})]+ (is (=? [{:name #".* lib", :curated true}+ {:name #".* lib", :curated true}+ {:name #".* plain", :curated false}+ {:name #".* plain", :curated false}]+ results))+ (testing "the ingestion-only table collection columns stay out of the response"+ (is (not-any? #(contains? % :table_collection_id) results)))+ (testing "the curated filter keeps the library ones"+ (is (= [true true]+ (map #(str/ends-with? (:name %) " lib")+ (search-results** "foo" {:context :metabot :models #{"measure" "segment"} :curated true})))))))))))++(deftest measure-segment-library-user-settings-test+ ;; Index table first, before `with-temp` opens its transaction; see [[library-test]].+ (search.tu/with-temp-index-table+ (testing "measures and segments follow a table published and unpublished through its user settings"+ (collections.tu/with-library [{data :data}]+ (mt/with-temp [:model/Table {table-id :id} {:name "tbl"}+ :model/Measure _ {:name "foo measure" :table_id table-id}+ :model/Segment _ {:name "foo segment" :table_id table-id}]+ (let [curated #(mapv :curated (search-results** "foo" {:context :metabot :models #{"measure" "segment"}}))]+ (is (= [[false false] [true true] [false false]]+ [(curated)+ (do (t2/insert! :model/TableUserSettings+ {:table_id table-id :is_published true :collection_id (:id data)})+ (curated))+ (do (t2/update! :model/TableUserSettings table-id {:is_published false :collection_id nil})+ (curated))]))))))))+(deftest ^:parallel data-layer-test(testing ":data-layer scorer reads the active per-tier weight via :data-layer/* params"(with-index-contents@@ -411,3 +453,114 @@:data-layer/hidden 0.03}(search-results* "foo table"))(map second)))))))++(deftest metabot-recency-weights-test+ (testing "with Metabot weights, recency does not lift a result into a higher tier"+ (let [user-id (mt/user->id :crowberto)+ now (Instant/now)+ long-ago (.minus now 365 ChronoUnit/DAYS)+ viewed (fn [model model-id] {:model model :model_id model-id :user_id user-id :timestamp now})+ ranked (fn [search-string]+ (map second (search-results* search-string :context :metabot :current-user-id user-id)))]+ ;; Create each index table outside `with-temp`'s transaction; see [[user-recency-test]].+ (testing "a recent model stays below a stale question and metric"+ (search.tu/with-temp-index-table+ (mt/with-temp [:model/Card {model-id :id} {}+ :model/Card {question-id :id} {}+ :model/Card {metric-id :id} {}+ :model/RecentViews _ (viewed "card" model-id)]+ (with-index-contents+ [{:model "dataset" :id model-id :name "foo model" :last_viewed_at now}+ {:model "card" :id question-id :name "foo question" :last_viewed_at long-ago}+ {:model "metric" :id metric-id :name "foo metric" :last_viewed_at long-ago}]+ (is (= [metric-id question-id model-id] (ranked "foo")))))))+ (testing "a recent final-layer table stays below a stale metric"+ (search.tu/with-temp-index-table+ (mt/with-temp [:model/Table {table-id :id} {}+ :model/Card {metric-id :id} {}+ :model/RecentViews _ (viewed "table" table-id)]+ (with-index-contents+ [{:model "table" :id table-id :name "foo table" :last_viewed_at now :data_layer "final"}+ {:model "metric" :id metric-id :name "foo metric" :last_viewed_at long-ago}]+ (is (= [metric-id table-id] (ranked "foo")))))))+ (testing "a recent model stays below a stale table without a data layer"+ (search.tu/with-temp-index-table+ (mt/with-temp [:model/Card {model-id :id} {}+ :model/Table {table-id :id} {}+ :model/RecentViews _ (viewed "card" model-id)]+ (with-index-contents+ [{:model "dataset" :id model-id :name "foo model" :last_viewed_at now}+ {:model "table" :id table-id :name "foo table" :last_viewed_at long-ago}]+ (is (= [table-id model-id] (ranked "foo"))))))))))++(deftest ^:parallel metabot-model-weights-test+ (testing "with Metabot weights, a metric outranks a final-layer table"+ (with-index-contents+ [{:model "table" :id 1 :name "foo table" :data_layer "final"}+ {:model "metric" :id 2 :name "foo metric"}]+ (is (= [2 1] (map second (search-results* "foo" :context :metabot))))))+ (testing "with Metabot weights, a metric outranks a heavily viewed final-layer table"+ (with-index-contents+ [{:model "table" :id 1 :name "foo table" :data_layer "final" :view_count 1000}+ {:model "metric" :id 2 :name "foo metric" :view_count 0}]+ (is (= [2 1] (map second (search-results* "foo" :context :metabot))))))+ (testing "with Metabot weights, a table outranks a model even without a data-layer score"+ (with-index-contents+ [{:model "dataset" :id 1 :name "foo model"}+ {:model "table" :id 2 :name "foo table"}]+ (is (= [2 1] (map second (search-results* "foo" :context :metabot))))))+ (testing "with Metabot weights, measures and segments rank with metrics, above questions and models"+ (with-index-contents+ [{:model "dataset" :id 1 :name "foo model"}+ {:model "card" :id 2 :name "foo question"}+ {:model "measure" :id 3 :name "foo measure"}+ {:model "segment" :id 4 :name "foo segment"}]+ (let [[semantic-layer others] (split-at 2 (map second (search-results* "foo" :context :metabot)))]+ (is (= [#{3 4} [2 1]]+ [(set semantic-layer) others])))))+ (testing "with Metabot weights, collections and actions stay below questions"+ (with-index-contents+ [{:model "collection" :id 1 :name "foo collection"}+ {:model "action" :id 2 :name "foo action"}+ {:model "card" :id 3 :name "foo question"}]+ (is (= 3 (second (first (search-results* "foo" :context :metabot))))))))++(deftest ^:parallel metabot-relevance-crosses-tier-test+ (with-index-contents+ [{:model "table" :id 1 :name "foo orders" :data_layer "final"}+ {:model "metric" :id 2 :name "bar metric" :description "mentions foo"}]+ (case (mdb/db-type)+ :postgres+ (testing "with Metabot weights, a final-layer table whose name matches outranks a metric that only mentions the term"+ (is (= [1 2] (map second (search-results* "foo" :context :metabot)))))+ :h2+ ;; TODO text ranking (probably in-memory+ nil)))++(deftest metabot-library-weight-test+ (testing "with Metabot weights, library membership outranks type weights"+ (search.tu/with-temp-index-table+ (mt/with-temp [:model/Collection lib {:name "lib" :type "library" :location "/"}]+ (with-index-contents+ [{:model "metric" :id 1 :name "foo metric"}+ {:model "dataset"+ :id 2+ :name "foo model"+ :collection_id (:id lib)+ :collection_location (:location lib)+ :collection_type "library"}]+ (is (= [2 1] (map second (search-results* "foo" :context :metabot)))))))))++(deftest metabot-exact-match-test+ (testing "with Metabot weights, an exact name match outranks a library metric that only contains the term"+ (search.tu/with-temp-index-table+ (mt/with-temp [:model/Collection lib {:name "lib" :type "library" :location "/"}]+ (with-index-contents+ [{:model "dataset" :id 1 :name "foo"}+ {:model "metric"+ :id 2+ :name "foo metric"+ :collection_id (:id lib)+ :collection_location (:location lib)+ :collection_type "library"}]+ (is (= [1 2] (map second (search-results* "foo" :context :metabot)))))))))diff --git a/test/metabase/search/filter_test.clj b/test/metabase/search/filter_test.cljindex 5e803dd5bf6..6d8faca0636 100644--- a/test/metabase/search/filter_test.clj+++ b/test/metabase/search/filter_test.clj@@ -2,6 +2,7 @@(:require[clojure.math.combinatorics :as math.combo][clojure.test :refer :all]+ [metabase.collections.test-utils :as collections.tu][metabase.models.resolution][metabase.search.config :as search.config][metabase.search.filter :as search.filter]@@ -176,8 +177,37 @@(is (=? [:or any? [:exists any?]]clause)))))+(deftest in-place-curated-measure-segment-filter-test+ (testing "the in-place curated filter for measures and segments follows the parent table's published state"+ (mt/with-premium-features #{:library}+ (doseq [model ["measure" "segment"]]+ (is (= [:= :table.is_published true]+ (:where (#'search.in-place.filter/build-optional-filter-query :curated model {} true))))))+ (testing "and matches nothing without the :library feature"+ (mt/with-premium-features #{}+ (doseq [model ["measure" "segment"]]+ (is (= [:= [:inline 0] [:inline 1]]+ (:where (#'search.in-place.filter/build-optional-filter-query :curated model {} true)))))))))+(deftest in-place-curated-table-filter-counts-authoritative-test(testing "the in-place curated table filter counts authoritative tables regardless of publish state,mirroring collections.curation/curated? (BOT-1570)"(let [where (:where (#'search.in-place.filter/build-optional-filter-query :curated "table" {} true))](is (some #{"authoritative"} (tree-seq coll? seq where))))))++(deftest in-place-curated-measure-segment-search-test+ (testing "the in-place curated search keeps only measures and segments whose parent table is in the Library"+ (search.tu/with-legacy-search+ (mt/with-premium-features #{:library}+ (collections.tu/with-library [{data :data}]+ (mt/with-temp [:model/Table lib-tbl {:name "lib_tbl" :is_published true :collection_id (:id data)}+ :model/Table plain-tbl {:name "plain_tbl"}+ :model/Measure _ {:name "inplace measure lib" :table_id (:id lib-tbl)}+ :model/Measure _ {:name "inplace measure plain" :table_id (:id plain-tbl)}+ :model/Segment _ {:name "inplace segment lib" :table_id (:id lib-tbl)}+ :model/Segment _ {:name "inplace segment plain" :table_id (:id plain-tbl)}]+ (is (= #{"inplace measure lib" "inplace segment lib"}+ (into #{} (map :name)+ (search.tu/search-results "inplace" {:search-engine "in-place"+ :models #{"measure" "segment"}+ :curated true}))))))))))diff --git a/test/metabase/search/spec_test.clj b/test/metabase/search/spec_test.cljindex 2205e95924c..31b560e8eba 100644--- a/test/metabase/search/spec_test.clj+++ b/test/metabase/search/spec_test.clj@@ -98,7 +98,7 @@(deftest ^:parallel search-model-hooks-test-2;; TODO replace real specs with frozen test ones once things have stabilized(is (= #:model{:Table #{{:search-model "segment",- :fields #{:description :schema :name :db_id :display_name}+ :fields #{:collection_id :db_id :description :display_name :is_published :name :schema}:where [:= :updated.id :this.table_id]}{:search-model "table",:fields@@ -118,15 +118,49 @@:where [:= :updated.id :this.id]}{:search-model "table":fields #{:authority_level :name :type :location}- :where [:and [:= :this.is_published true] [:= :updated.id :this.collection_id]]}}+ :where [:and [:= :this.is_published true] [:= :updated.id :this.collection_id]]}+ {:search-model "segment"+ :fields #{:location :type}+ :where [:and+ [:= [:coalesce :settings.is_published :table.is_published] true]+ [:= :updated.id [:case+ [:not= :settings.is_published nil] :settings.collection_id+ :else :table.collection_id]]]}}:TableUserSettings #{{:search-model "table":fields #{:updated_at}- :where [:= :updated.table_id :this.id]}}}+ :where [:= :updated.table_id :this.id]}+ {:search-model "segment"+ :fields #{:collection_id :is_published}+ :where [:= :updated.table_id :this.table_id]}}}(#'search.spec/merge-hooks[(#'search.spec/search-model-hooks (search.spec/spec "table"))(#'search.spec/search-model-hooks (search.spec/spec "segment"))(#'search.spec/search-model-hooks (search.spec/spec "collection"))]))))+(deftest ^:parallel measure-parent-table-hooks-test+ (testing "a measure is reindexed when its parent table, that table's user settings, or its collection changes"+ (is (= #:model{:Table #{{:search-model "measure"+ :fields #{:collection_id+ :db_id+ :description+ :display_name+ :is_published+ :name+ :schema}+ :where [:= :updated.id :this.table_id]}}+ :Collection #{{:search-model "measure"+ :fields #{:location :type}+ :where [:and+ [:= [:coalesce :settings.is_published :table.is_published] true]+ [:= :updated.id [:case+ [:not= :settings.is_published nil] :settings.collection_id+ :else :table.collection_id]]]}}+ :TableUserSettings #{{:search-model "measure"+ :fields #{:collection_id :is_published}+ :where [:= :updated.table_id :this.table_id]}}}+ (select-keys (#'search.spec/merge-hooks [(#'search.spec/search-model-hooks (search.spec/spec "measure"))])+ [:model/Table :model/Collection :model/TableUserSettings])))))+(deftest ^:parallel search-models-to-update-test(is (= #{}(search.spec/search-models-to-update (t2/instance :model/Database {}))))