The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Improve LLM token usage for add_reseach_group (#79556)
Nathan Voxland250868257eba
diff --git a/e2e/support/helpers/e2e-explorations-helpers.ts b/e2e/support/helpers/e2e-explorations-helpers.tsindex 9bbfd4acb31..b00737202d9 100644--- a/e2e/support/helpers/e2e-explorations-helpers.ts+++ b/e2e/support/helpers/e2e-explorations-helpers.ts@@ -101,6 +101,18 @@ export function beginResearch(): Cypress.Chainable<number> {});}+/**+ * A data part a tool emits alongside its result. Payloads too large to spend LLM+ * context on ride one of these instead of the tool's `:output` — see+ * `add_research_groups` and `research_plan_update` in+ * `metabase.metabot.tools.explorations`.+ */+export interface ExplorationDataPart {+ /** AI-SDK data type without the `data-` prefix, e.g. `research_plan_update`. */+ dataType: string;+ data: unknown;+}+/*** Shape of a single tool-call event the explorations agent emits over the AI-streaming protocol.*/@@ -109,6 +121,11 @@ export interface ExplorationToolCall {toolName: string;args?: Record<string, unknown>;result: unknown;+ dataParts?: ExplorationDataPart[];+}++function isKeyedPayload(data: unknown): data is Record<string, unknown> {+ return typeof data === "object" && data !== null && !Array.isArray(data);}/**@@ -118,7 +135,8 @@ export interface ExplorationToolCall {* event is a `data: {json}` SSE line, wrapped in the backend's lifecycle* (`start` → `start-step` → events → `finish-step` → `finish` → `[DONE]`).* Tool results are passed as objects — the client JSON-stringifies non-string- * outputs before handing them to consumers like `NewExplorationChat`.+ * outputs before handing them to consumers like `NewExplorationChat`. A tool's+ * `dataParts` are emitted right after its output, as the backend does.*/export function buildExplorationStreamingBody(toolCalls: ExplorationToolCall[],@@ -139,6 +157,19 @@ export function buildExplorationStreamingBody(toolCallId: tc.toolCallId,output: tc.result,});+ // Data parts follow their tool's output, each stamped with the originating+ // tool-call id, the way `expand-data-parts-xf` emits them on the backend.+ // Array payloads (e.g. todo_list) have no keyed slot, so they're left as-is.+ (tc.dataParts ?? []).forEach((dataPart, index) => {+ const { data } = dataPart;+ events.push({+ type: `data-${dataPart.dataType}`,+ id: `${tc.toolCallId}-data-${index}`,+ data: isKeyedPayload(data)+ ? { ...data, tool_call_id: tc.toolCallId }+ : data,+ });+ });}events.push({ type: "finish-step" });events.push({diff --git a/e2e/test/scenarios/explorations/explorations.cy.spec.ts b/e2e/test/scenarios/explorations/explorations.cy.spec.tsindex 553fe400fe6..720335f65c0 100644--- a/e2e/test/scenarios/explorations/explorations.cy.spec.ts+++ b/e2e/test/scenarios/explorations/explorations.cy.spec.ts@@ -296,11 +296,19 @@ describe("scenarios > explorations > new research > metabot flow", () => {{toolCallId: "groups-1",toolName: "add_research_groups",- result: {- metrics: [firstMetric],- dimension_groups: [interestingGroup],- groups: [{ metric_id: firstMetric.id }],- },+ // The tool result is only the LLM's summary of the edit; the picker+ // hydration rides a `research_plan_update` data part.+ result: `Added 1 group(s) to the research plan:\n- ${firstMetric.name}, by the automatically-selected dimensions`,+ dataParts: [+ {+ dataType: "research_plan_update",+ data: {+ metrics: [firstMetric],+ dimension_groups: [interestingGroup],+ groups: [{ metric_id: firstMetric.id }],+ },+ },+ ],},{toolCallId: "name-1",diff --git a/frontend/src/metabase/api/ai-streaming/schemas.ts b/frontend/src/metabase/api/ai-streaming/schemas.tsindex 0fbf0d3f9e6..e2fb5a8d2d7 100644--- a/frontend/src/metabase/api/ai-streaming/schemas.ts+++ b/frontend/src/metabase/api/ai-streaming/schemas.ts@@ -1,6 +1,7 @@import * as Yup from "yup";import type {+ AddResearchGroupsResponse,CardDisplayType,DatasetQuery,MetabotCodeEdit,@@ -47,6 +48,7 @@ export const knownDataPartTypes = ["data-search_results","data-tool_title","data-conversation-title",+ "data-research_plan_update",] as const satisfies readonly KnownDataPart["type"][];export type SearchResultItem = {@@ -120,6 +122,13 @@ export type ToolTitleData = {title: string;};+// The picker hydration for a Research plan edit. It rides a data part rather than the+// `add_research_groups` tool result because that result is also the agent's LLM context, and this+// payload grows with every metric the call references.+export type ResearchPlanUpdateData = AddResearchGroupsResponse & {+ tool_call_id?: string;+};+export type KnownDataPart =| { type: "data-navigate_to"; data: string }| { type: "data-state"; data: Record<string, unknown> }@@ -132,7 +141,8 @@ export type KnownDataPart =| { type: "data-static_viz"; data: StaticVizValue }| { type: "data-search_results"; data: SearchResultsData }| { type: "data-tool_title"; data: ToolTitleData }- | { type: "data-conversation-title"; data: string };+ | { type: "data-conversation-title"; data: string }+ | { type: "data-research_plan_update"; data: ResearchPlanUpdateData };export const isKnownDataPart = (part: {type: string;diff --git a/frontend/src/metabase/explorations/components/NewExplorationChat/NewExplorationChat.tsx b/frontend/src/metabase/explorations/components/NewExplorationChat/NewExplorationChat.tsxindex 77c77e3fca2..3ce6d1322c2 100644--- a/frontend/src/metabase/explorations/components/NewExplorationChat/NewExplorationChat.tsx+++ b/frontend/src/metabase/explorations/components/NewExplorationChat/NewExplorationChat.tsx@@ -21,20 +21,18 @@ import {useUserMetabotPermissions,} from "metabase/metabot/hooks";import type {+ MetabotAgentDataPartMessage,MetabotChatMessage,+ MetabotDataPart,MetabotDebugToolCallMessage,} from "metabase/metabot/state";import { Box, Flex, Stack, Text } from "metabase/ui";-import type {- AddResearchGroupsResponse,- RemoveFromResearchPlanResponse,-} from "metabase-types/api";+import type { RemoveFromResearchPlanResponse } from "metabase-types/api";import S from "./NewExplorationChat.module.css";export const EXPLORATIONS_AGENT_ID = "explorations";-const ADD_RESEARCH_GROUPS_TOOL = "add_research_groups";const REMOVE_FROM_RESEARCH_PLAN_TOOL = "remove_from_research_plan";const SET_RESEARCH_NAME_TOOL = "set_research_name";const SELECT_RESEARCH_TIMELINES_TOOL = "select_research_timelines";@@ -43,6 +41,12 @@ type MetabotToolCallMessageWithResult = MetabotDebugToolCallMessage & {result: string;};+// `add_research_groups` delivers its picker hydration on a data part rather than in its tool+// result, because that result is also the agent's LLM context.+type ResearchPlanUpdateMessage = MetabotAgentDataPartMessage & {+ part: Extract<MetabotDataPart, { type: "data-research_plan_update" }>;+};+export interface NewExplorationChatProps {selection: ExplorationSelection;}@@ -103,18 +107,16 @@ export function NewExplorationChat({ selection }: NewExplorationChatProps) {const [sendToast] = useToast();- const handleAddResearchGroupsToolCallMessages = useCallback(- (messages: MetabotToolCallMessageWithResult[]) => {+ const handleResearchPlanUpdateMessages = useCallback(+ (messages: ResearchPlanUpdateMessage[]) => {const trackMetricsEdited = once(() =>trackExplorationPlanEdited("agent", "metrics"),);try {for (const message of messages) {- // tool result shape verified by the backend- const { metrics, dimension_groups, groups } = JSON.parse(- message.result,- ) as AddResearchGroupsResponse;+ // data part shape verified by the backend+ const { metrics, dimension_groups, groups } = message.part.data;const metricsById = new Map(metrics.map((m) => [m.id, m] as const));const dimensionsById = indexDimensionsById(dimension_groups);@@ -249,8 +251,8 @@ export function NewExplorationChat({ selection }: NewExplorationChatProps) {processedMessageIdsRef.current.add(message.id);}- handleAddResearchGroupsToolCallMessages(- unprocessedMessages.filter(isAddResearchGroupsToolCallMessage),+ handleResearchPlanUpdateMessages(+ unprocessedMessages.filter(isResearchPlanUpdateMessage),);handleRemoveFromResearchPlanToolCallMessages(unprocessedMessages.filter(isRemoveFromResearchPlanToolCallMessage),@@ -263,7 +265,7 @@ export function NewExplorationChat({ selection }: NewExplorationChatProps) {);}, [isDoingScience,- handleAddResearchGroupsToolCallMessages,+ handleResearchPlanUpdateMessages,handleRemoveFromResearchPlanToolCallMessages,handleSetExplorationNameToolCallMessages,handleSelectExplorationTimelinesToolCallMessages,@@ -348,15 +350,13 @@ export function NewExplorationChat({ selection }: NewExplorationChatProps) {);}-function isAddResearchGroupsToolCallMessage(+function isResearchPlanUpdateMessage(message: MetabotChatMessage,-): message is MetabotToolCallMessageWithResult {+): message is ResearchPlanUpdateMessage {return (message.role === "agent" &&- message.type === "tool_call" &&- message.name === ADD_RESEARCH_GROUPS_TOOL &&- !message.is_error &&- !!message.result+ message.type === "data_part" &&+ message.part.type === "data-research_plan_update");}diff --git a/frontend/src/metabase/explorations/components/NewExplorationChat/NewExplorationChat.unit.spec.tsx b/frontend/src/metabase/explorations/components/NewExplorationChat/NewExplorationChat.unit.spec.tsxindex 8fc96ff35dc..4d2e79bcd0a 100644--- a/frontend/src/metabase/explorations/components/NewExplorationChat/NewExplorationChat.unit.spec.tsx+++ b/frontend/src/metabase/explorations/components/NewExplorationChat/NewExplorationChat.unit.spec.tsx@@ -11,6 +11,7 @@ import {import { makeMockSelection } from "metabase/explorations/test-utils";import { useMetabotAgent } from "metabase/metabot/hooks";import type {+ MetabotAgentDataPartMessage,MetabotChatMessage,MetabotDebugToolCallMessage,} from "metabase/metabot/state";@@ -148,13 +149,14 @@ const addResearchGroupsResponse: AddResearchGroupsResponse = {],};-const addResearchGroupsToolCallMessage: MetabotDebugToolCallMessage = {- id: "tool-call-2",+const researchPlanUpdateMessage: MetabotAgentDataPartMessage = {+ id: "data-part-2",role: "agent",- type: "tool_call",- name: "add_research_groups",- status: "ended",- result: JSON.stringify(addResearchGroupsResponse),+ type: "data_part",+ part: {+ type: "data-research_plan_update",+ data: addResearchGroupsResponse,+ },};const setNameToolCallMessage: MetabotDebugToolCallMessage = {@@ -302,7 +304,7 @@ describe("NewExplorationChat", () => {});});- it("adds groups from an add_research_groups tool call response", async () => {+ it("adds groups from a research_plan_update data part", async () => {const { selection, rerender } = setup();rerender({@@ -310,11 +312,7 @@ describe("NewExplorationChat", () => {isDoingScience: true,});rerender({- messages: [- userMessage,- searchToolCallMessage,- addResearchGroupsToolCallMessage,- ],+ messages: [userMessage, searchToolCallMessage, researchPlanUpdateMessage],isDoingScience: true,});@@ -324,7 +322,7 @@ describe("NewExplorationChat", () => {messages: [userMessage,searchToolCallMessage,- addResearchGroupsToolCallMessage,+ researchPlanUpdateMessage,agentMessage,],isDoingScience: false,@@ -370,19 +368,22 @@ describe("NewExplorationChat", () => {it("forwards replace_default_dimensions to addMetric", async () => {const { selection, rerender } = setup();- const message: MetabotDebugToolCallMessage = {- ...addResearchGroupsToolCallMessage,- id: "tool-call-replace",- result: JSON.stringify({- ...addResearchGroupsResponse,- groups: [- {- metric_id: metricRevenue.id,- dimension_ids: [revenueDateDimension.id],- replace_default_dimensions: true,- },- ],- }),+ const message: MetabotAgentDataPartMessage = {+ ...researchPlanUpdateMessage,+ id: "data-part-replace",+ part: {+ type: "data-research_plan_update",+ data: {+ ...addResearchGroupsResponse,+ groups: [+ {+ metric_id: metricRevenue.id,+ dimension_ids: [revenueDateDimension.id],+ replace_default_dimensions: true,+ },+ ],+ },+ },};rerender({ messages: [userMessage, message], isDoingScience: true });diff --git a/frontend/src/metabase/metabot/components/MetabotChat/MetabotAgentDataPartMessage.tsx b/frontend/src/metabase/metabot/components/MetabotChat/MetabotAgentDataPartMessage.tsxindex 44593d5c066..3b43938793d 100644--- a/frontend/src/metabase/metabot/components/MetabotChat/MetabotAgentDataPartMessage.tsx+++ b/frontend/src/metabase/metabot/components/MetabotChat/MetabotAgentDataPartMessage.tsx@@ -129,6 +129,9 @@ export const AgentDataPartMessage = ({.with({ part: { type: "data-static_viz" } }, ({ part }) =>debug ? <DataPartJsonCard type={part.type} value={part.data} /> : null,)+ .with({ part: { type: "data-research_plan_update" } }, ({ part }) =>+ debug ? <DataPartJsonCard type={part.type} value={part.data} /> : null,+ ).exhaustive((msg: unknown) => {console.warn("AgentDataPartMessage received an unexpected value:", msg);return null;diff --git a/frontend/src/metabase/metabot/components/MetabotChat/MetabotChatMessage.tsx b/frontend/src/metabase/metabot/components/MetabotChat/MetabotChatMessage.tsxindex 7f93ec882ba..082585682e8 100644--- a/frontend/src/metabase/metabot/components/MetabotChat/MetabotChatMessage.tsx+++ b/frontend/src/metabase/metabot/components/MetabotChat/MetabotChatMessage.tsx@@ -60,6 +60,7 @@ const isUserVisibleDataPart = (part: MetabotDataPart): boolean =>.with({ type: "data-entity_saved" }, () => true).with({ type: "data-adhoc_viz" }, () => false).with({ type: "data-static_viz" }, () => false)+ .with({ type: "data-research_plan_update" }, () => false).exhaustive();const isUserVisibleDataPartMessage = (diff --git a/frontend/src/metabase/metabot/state/actions.ts b/frontend/src/metabase/metabot/state/actions.tsindex edee82f132e..2dc73dfdba9 100644--- a/frontend/src/metabase/metabot/state/actions.ts+++ b/frontend/src/metabase/metabot/state/actions.ts@@ -532,6 +532,9 @@ export const sendAgentRequest = createAsyncThunk<.with({ type: "data-todo_list" }, (part) => {pushDataPart({ type: "data_part", part });})+ .with({ type: "data-research_plan_update" }, (part) => {+ pushDataPart({ type: "data_part", part });+ }).with({ type: "data-code_edit" }, (part) => {dispatch(addSuggestedCodeEdit({ ...part.data, active: true }));diff --git a/src/metabase/explorations/impl.clj b/src/metabase/explorations/impl.cljindex 0715d978c5d..fdaeb76d452 100644--- a/src/metabase/explorations/impl.clj+++ b/src/metabase/explorations/impl.clj@@ -502,24 +502,36 @@referenced metrics, and `:groups` echoes the validated specs for the FE to turn into pickerblocks."[{:keys [groups]}]- (let [all (mapv with-candidate-dimensions (hydrated-metrics {}))- metric-by-id (u/index-by :id all)]- (doseq [g groups]- (let [metric-id (:metric_id g)- metric (get metric-by-id metric-id)]- (when-not metric- (throw (ex-info (format "Unknown or inaccessible metric id %s" metric-id)- {:metric_id metric-id})))- (when (and (:replace_default_dimensions g) (empty? (:dimension_ids g)))- (throw (ex-info "replace_default_dimensions requires at least one dimension_id"- {:metric_id metric-id})))- (let [valid (set (map :id (:dimensions metric)))]- (doseq [d (:dimension_ids g)]- (when-not (contains? valid d)- (throw (ex-info (format "Dimension %s is not a candidate of metric %s" d metric-id)- {:metric_id metric-id :dimension_id d})))))))- (let [relevant (into #{} (map :metric_id) groups)- relevant-metrics (filterv #(contains? relevant (:id %)) all)]- {:metrics (mapv slim-metric relevant-metrics)- :dimension_groups (group-dimensions relevant-metrics)- :groups (vec groups)})))+ (lib-be/with-metadata-provider-cache+ (let [all (mapv with-candidate-dimensions (catalog-metrics {}))+ metric-by-id (u/index-by :id all)]+ ;; Metric-level checks run against the cheap catalog — they need no query build.+ (doseq [g groups]+ (let [metric-id (:metric_id g)]+ (when-not (contains? metric-by-id metric-id)+ (throw (ex-info (format "Unknown or inaccessible metric id %s" metric-id)+ {:metric_id metric-id})))+ (when (and (:replace_default_dimensions g) (empty? (:dimension_ids g)))+ (throw (ex-info "replace_default_dimensions requires at least one dimension_id"+ {:metric_id metric-id})))))+ (let [relevant (into #{} (map :metric_id) groups)+ ;; Only the referenced metrics pay [[resolve-metric-queries]]. `all` is already+ ;; candidate-filtered and resolution only drops dimensions further, so no second+ ;; [[with-candidate-dimensions]] pass is needed.+ relevant-metrics (->> all+ (filterv #(contains? relevant (:id %)))+ resolve-metric-queries)+ resolved-by-id (u/index-by :id relevant-metrics)]+ ;; Dimension ids are checked against the *resolved* metrics: a saved dimension whose+ ;; target no longer resolves is dropped by [[resolve-metric-queries]], and accepting it+ ;; would hand the FE a picker block for a dimension that isn't in the payload.+ (doseq [g groups+ :let [metric-id (:metric_id g)+ valid (set (map :id (:dimensions (get resolved-by-id metric-id))))]+ d (:dimension_ids g)]+ (when-not (contains? valid d)+ (throw (ex-info (format "Dimension %s is not a candidate of metric %s" d metric-id)+ {:metric_id metric-id :dimension_id d}))))+ {:metrics (mapv slim-metric relevant-metrics)+ :dimension_groups (group-dimensions relevant-metrics)+ :groups (vec groups)}))))diff --git a/src/metabase/metabot/agent/streaming.clj b/src/metabase/metabot/agent/streaming.cljindex 3772d726b6c..6ae068d923d 100644--- a/src/metabase/metabot/agent/streaming.clj+++ b/src/metabase/metabot/agent/streaming.clj@@ -23,6 +23,7 @@(def static-viz-type "AI-SDK data type for static visualizations." "static_viz")(def search-results-type "AI-SDK data type for a search tool's result list." "search_results")(def tool-title-type "AI-SDK data type for a tool call's settled display title." "tool_title")+(def research-plan-update-type "AI-SDK data type for a Research plan edit's picker hydration." "research_plan_update")(def ^:private ephemeral-data-types"Data types not written to MetabotMessage.data."@@ -158,6 +159,17 @@:data-type search-results-type:data value})+(defn research-plan-update-part+ "Data part carrying the picker hydration for a Research plan edit — the metrics and+ dimension groups the FE needs to turn the agent's chosen groups into picker blocks.+ It rides here rather than in the tool's `:output` because the LLM only needs to know+ the edit happened, while the payload is large and grows with the metrics a group+ pulls in."+ [value]+ {:type :data+ :data-type research-plan-update-type+ :data value})+(defn tool-title-part"Data part carrying a tool call's display title, derived from what the toolactually read/did; [[expand-data-parts-xf]] stamps on the tool-call id itdiff --git a/src/metabase/metabot/tools/explorations.clj b/src/metabase/metabot/tools/explorations.cljindex d44c1a94e1b..84804b845b8 100644--- a/src/metabase/metabot/tools/explorations.clj+++ b/src/metabase/metabot/tools/explorations.clj@@ -1,14 +1,18 @@(ns metabase.metabot.tools.explorations"Exploration-specific tool wrappers.- Every tool here returns `{:output <json-string>}`: the exploration chat FE applies plan edits- by parsing the tool result it sees on the `tool-output-available` stream event, and only a- result's `:output` string makes it onto the wire (see- `metabase.metabot.self.core/tool-output->wire-output`) — a bare map would reach the LLM but- stream to the client as an empty string, and the plan would silently never update."+ A tool result's `:output` is the one string that goes to *both* the LLM and the client,+ so the small plan edits the FE applies by parsing the tool result — `remove_from_research_plan`, `set_research_name`,+ `select_research_timelines` — return `{:output <json-string>}`; a bare map would reach the LLM+ but stream to the client as an empty string, and the plan would silently never update.++ `add_research_groups` is the exception: its picker hydration is far too large to spend LLM+ context on and grows with every metric the call references, so it rides a `research_plan_update`+ data part (FE only) while `:output` carries a plain-text summary."(:require[clojure.string :as str][metabase.explorations.core :as explorations]+ [metabase.metabot.agent.streaming :as streaming][metabase.metabot.scope :as scope][metabase.metabot.tmpl :as te][metabase.util.json :as json]@@ -111,6 +115,51 @@:else{:output (json/encode (explorations/research-candidates {:metric-ids metric_ids :q q}))}))+(defn- research-groups-payload+ "Validate `groups` and hydrate the FE picker payload for them, in the snake_case shape the+ client consumes."+ [groups]+ (explorations/exploration-data->api (explorations/research-groups {:groups groups})))++(def ^:private summary-max-names+ "How many dimension names one summary line spells out before falling back to a count. A metric+ can be sliced by a long list of candidate dimensions, and the summary is the whole of what the+ LLM sees of the result."+ 15)++(defn- summarize-names [names]+ (let [extra (- (count names) summary-max-names)]+ (if (pos? extra)+ (str (str/join ", " (take summary-max-names names)) " and " extra " more")+ (str/join ", " names))))++(defn- format-added-groups+ "Plain-text summary of the groups `add_research_groups` just applied — the whole of what the+ LLM sees of the result, since the picker hydration rides a data part instead. Names what the+ user now has in the plan so the agent can describe it without re-reading the catalog; ids are+ deliberately absent, as the next turn's plan snapshot carries+ the `block_id`s needed to edit these groups."+ [{:keys [metrics dimension_groups groups]}]+ (let [metric-name (into {} (map (juxt :id :name)) metrics)+ group-idx (into {} (for [[i g] (map-indexed vector dimension_groups)+ d (:dimensions g)]+ [(:id d) i]))+ group-names (mapv :name dimension_groups)+ ;; Nil for a dimension the payload doesn't group: groups are validated against the+ ;; unresolved catalog, so a dimension can pass validation and still be dropped from the+ ;; hydrated metrics because their query can't actually break out on it. The summary skips+ ;; what it can't name rather than failing the whole tool call.+ group-name (fn [dimension-id] (some-> (group-idx dimension-id) group-names))]+ (te/lines+ (str "Added " (count groups) " group(s) to the research plan:")+ (for [g groups]+ (let [dims (seq (keep group-name (:dimension_ids g)))]+ (str "- " (metric-name (:metric_id g))+ (cond+ (and dims (:replace_default_dimensions g)) (str ", by exactly: " (summarize-names dims))+ dims (str ", by: " (summarize-names dims) ", plus the automatic selection")+ :else ", by the automatically-selected dimensions")))))))+(def ^:private add-research-groups-schema[:map {:closed true}[:groups@@ -130,8 +179,9 @@list (no automatic ones), also pass `\"replace_default_dimensions\": true` - then`dimension_ids` must be non-empty."[{:keys [groups]} :- add-research-groups-schema]- {:output (json/encode (explorations/exploration-data->api- (explorations/research-groups {:groups groups})))})+ (let [payload (research-groups-payload groups)]+ {:output (format-added-groups payload)+ :data-parts [(streaming/research-plan-update-part payload)]}))(def ^:private remove-from-research-plan-schema[:map {:closed true}diff --git a/test/metabase/explorations/impl_test.clj b/test/metabase/explorations/impl_test.cljindex a6fd67508f9..a257bdd1379 100644--- a/test/metabase/explorations/impl_test.clj+++ b/test/metabase/explorations/impl_test.clj@@ -139,7 +139,8 @@(defmacro ^:private with-synthetic-metrics [& body]`(mt/with-dynamic-fn-redefs [explorations.impl/hydrated-metrics synthetic-hydrated- explorations.impl/index-metrics synthetic-hydrated]+ explorations.impl/index-metrics synthetic-hydrated+ explorations.impl/catalog-metrics synthetic-hydrated]~@body))(deftest research-metric-index-test@@ -361,7 +362,18 @@(testing "replace_default_dimensions with no dimension_ids"(is (thrown-with-msg? Exception #"replace_default_dimensions requires"(explorations.impl/research-groups- {:groups [{:metric_id 1 :replace_default_dimensions true}]}))))))+ {:groups [{:metric_id 1 :replace_default_dimensions true}]}))))+ (testing "dimension the catalog still lists but query resolution drops"+ ;; `plan-1` survives the cheap catalog but its target no longer resolves against the+ ;; metric's query, so it must be rejected rather than silently dropped from the payload.+ (mt/with-dynamic-fn-redefs [explorations.impl/resolve-metric-queries+ (fn [metrics]+ (mapv #(update % :dimensions+ (fn [ds] (filterv (comp #{"region-1"} :id) ds)))+ metrics))]+ (is (thrown-with-msg? Exception #"not a candidate of metric"+ (explorations.impl/research-groups+ {:groups [{:metric_id 1 :dimension_ids ["plan-1"]}]})))))));;; ------------------------------------------ exploration-data ------------------------------------------diff --git a/test/metabase/metabot/tools/explorations_test.clj b/test/metabase/metabot/tools/explorations_test.cljindex 756b3933cc7..745b1b957f6 100644--- a/test/metabase/metabot/tools/explorations_test.clj+++ b/test/metabase/metabot/tools/explorations_test.clj@@ -5,9 +5,11 @@[metabase.api-scope.core :as api-scope][metabase.metabot.agent.messages :as messages][metabase.metabot.agent.profiles :as profiles]+ [metabase.metabot.agent.streaming :as streaming][metabase.metabot.scope :as scope][metabase.metabot.self.core :as metabot.self][metabase.metabot.tools.explorations :as tools.explorations]+ [metabase.test :as mt][metabase.util.json :as json]))(def ^:private exploration-tool-vars@@ -57,6 +59,71 @@(is (str/includes? output "at most 20"))(is (str/includes? output "got 21")))))+;;; ------------------------------------ add_research_groups ------------------------------------++(defn- picker-payload+ "An `add_research_groups` result: `n-metrics` metrics, all sliceable by one dimension group."+ [n-metrics]+ {:metrics (vec (for [i (range n-metrics)]+ {:id i :name (str "Metric " i) :dimension_ids [(str "d" i)]}))+ :dimension_groups [{:name "Orders - Region"+ :dimensions (vec (for [i (range n-metrics)] {:id (str "d" i)}))}]+ :groups []})++(defn- wide-metric-payload+ "An `add_research_groups` result: one metric sliced by `n-dims` distinct dimension groups."+ [n-dims]+ (let [ids (vec (for [i (range n-dims)] (str "d" i)))]+ {:metrics [{:id 0 :name "Metric 0" :dimension_ids ids}]+ :dimension_groups (vec (for [i (range n-dims)]+ {:name (str "Group " i) :dimensions [{:id (str "d" i)}]}))+ :groups [{:metric_id 0 :dimension_ids ids}]}))++(deftest add-research-groups-keeps-the-payload-off-the-llm-wire-test+ (testing "the picker hydration rides a data part the FE consumes; the LLM only sees a summary"+ (let [payload (assoc (picker-payload 3)+ :groups [{:metric_id 0 :dimension_ids ["d0"]}+ {:metric_id 1 :dimension_ids ["d1"] :replace_default_dimensions true}+ {:metric_id 2}])]+ (mt/with-dynamic-fn-redefs [tools.explorations/research-groups-payload (fn [_] payload)]+ (let [{:keys [output data-parts]} (tools.explorations/add-research-groups-tool+ {:groups (:groups payload)})]+ (testing "the payload is on the data part, not in :output"+ (is (= [{:type :data :data-type "research_plan_update" :data payload}] data-parts)))+ (testing ":output is prose naming what the user got, not the payload"+ (is (= (str "Added 3 group(s) to the research plan:\n"+ "- Metric 0, by: Orders - Region, plus the automatic selection\n"+ "- Metric 1, by exactly: Orders - Region\n"+ "- Metric 2, by the automatically-selected dimensions")+ output))))))))++(deftest add-research-groups-summary-tolerates-unnamed-dimensions-test+ (testing (str "groups are validated against the unresolved catalog, so a dimension can pass "+ "validation and still be absent from the hydrated payload (its metric's query "+ "can't break out on it) — the summary skips what it can't name rather than "+ "throwing away the whole tool call")+ (let [payload (assoc (picker-payload 1)+ :groups [{:metric_id 0 :dimension_ids ["d0" "unresolved"]}])]+ (mt/with-dynamic-fn-redefs [tools.explorations/research-groups-payload (fn [_] payload)]+ (is (= (str "Added 1 group(s) to the research plan:\n"+ "- Metric 0, by: Orders - Region, plus the automatic selection")+ (:output (tools.explorations/add-research-groups-tool+ {:groups (:groups payload)}))))))))++(deftest add-research-groups-summary-is-bounded-test+ (testing "a metric can be sliced by a long dimension list, so the summary counts the tail+ instead of listing it — :output is the agent's context, not a report"+ (let [summary-for (fn [n-dims]+ (let [payload (wide-metric-payload n-dims)]+ (mt/with-dynamic-fn-redefs [tools.explorations/research-groups-payload (fn [_] payload)]+ (:output (tools.explorations/add-research-groups-tool+ {:groups (:groups payload)})))))+ fifty (summary-for 50)]+ (testing "the tail past the first `summary-max-names` members is counted, not listed"+ (is (str/includes? fifty "and 35 more")))+ (testing "so a 10x longer dimension list costs only the extra digits in that count"+ (is (> 5 (- (count (summary-for 500)) (count fifty))))))))+(deftest ^:parallel remove-from-research-plan-tool-test(testing "echoes the block ids the agent asked to remove (pure-echo; the FE applies them)"(is (= {:block_ids ["metric:42" "metric:43"] :members nil :timeline_ids nil}@@ -94,10 +161,28 @@events)](some-> (:output event) json/decode+kw)))+(deftest ^:parallel research-plan-update-part-reaches-the-wire-test+ (testing (str "add_research_groups' picker payload is the plan edit, and it now travels as a "+ "data part rather than in :output — if it never reached the client the plan would "+ "silently never update, the same failure the :output tools guard against below")+ (let [payload {:metrics [{:id 1}] :dimension_groups [] :groups []}+ events (into [] (comp streaming/expand-data-parts-xf (metabot.self/parts->aisdk-sse-xf))+ [{:type :start :id "s1"}+ {:type :tool-output :id "tc1"+ :result {:output "Added 1 group(s) to the research plan:"+ :data-parts [(streaming/research-plan-update-part payload)]}}])+ event (some #(when (str/includes? % "\"data-research_plan_update\"")+ (json/decode+kw (str/replace-first % "data: " "")))+ events)]+ (is (some? event) "a data-research_plan_update event is streamed")+ (is (= [{:id 1}] (get-in event [:data :metrics])))+ (testing "stamped with the tool call it came from, so the client can attribute it"+ (is (= "tc1" (get-in event [:data :tool_call_id])))))))+(deftest ^:parallel plan-tool-results-reach-the-wire-test- (testing (str "the exploration chat FE applies plan edits by parsing the streamed tool result; "- "only a result's :output string makes it onto tool-output-available, so a bare-map "- "result would stream as \"\" and the plan would silently never update")+ (testing (str "the exploration chat FE applies the smaller plan edits by parsing the streamed "+ "tool result; only a result's :output string makes it onto tool-output-available, "+ "so a bare-map result would stream as \"\" and the plan would silently never update")(testing "set_research_name"(is (= {:name "Quarterly revenue"}(streamed-tool-output