Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Improve LLM token usage for add_reseach_group (#79556)

Nathan Voxland250868257eba
diff --git a/e2e/support/helpers/e2e-explorations-helpers.ts b/e2e/support/helpers/e2e-explorations-helpers.ts
index 9bbfd4acb31..b00737202d9 100644
--- a/e2e/support/helpers/e2e-explorations-helpers.ts
+++ b/e2e/support/helpers/e2e-explorations-helpers.ts
@@ -101,6 +101,18 @@ export function beginResearch(): Cypress.Chainable<number> {
});
}
+/**
+ * A data part a tool emits alongside its result. Payloads too large to spend LLM
+ * context on ride one of these instead of the tool's `:output` — see
+ * `add_research_groups` and `research_plan_update` in
+ * `metabase.metabot.tools.explorations`.
+ */
+export interface ExplorationDataPart {
+ /** AI-SDK data type without the `data-` prefix, e.g. `research_plan_update`. */
+ dataType: string;
+ data: unknown;
+}
+
/**
* Shape of a single tool-call event the explorations agent emits over the AI-streaming protocol.
*/
@@ -109,6 +121,11 @@ export interface ExplorationToolCall {
toolName: string;
args?: Record<string, unknown>;
result: unknown;
+ dataParts?: ExplorationDataPart[];
+}
+
+function isKeyedPayload(data: unknown): data is Record<string, unknown> {
+ return typeof data === "object" && data !== null && !Array.isArray(data);
}
/**
@@ -118,7 +135,8 @@ export interface ExplorationToolCall {
* event is a `data: {json}` SSE line, wrapped in the backend's lifecycle
* (`start` → `start-step` → events → `finish-step` → `finish` → `[DONE]`).
* Tool results are passed as objects — the client JSON-stringifies non-string
- * outputs before handing them to consumers like `NewExplorationChat`.
+ * outputs before handing them to consumers like `NewExplorationChat`. A tool's
+ * `dataParts` are emitted right after its output, as the backend does.
*/
export function buildExplorationStreamingBody(
toolCalls: ExplorationToolCall[],
@@ -139,6 +157,19 @@ export function buildExplorationStreamingBody(
toolCallId: tc.toolCallId,
output: tc.result,
});
+ // Data parts follow their tool's output, each stamped with the originating
+ // tool-call id, the way `expand-data-parts-xf` emits them on the backend.
+ // Array payloads (e.g. todo_list) have no keyed slot, so they're left as-is.
+ (tc.dataParts ?? []).forEach((dataPart, index) => {
+ const { data } = dataPart;
+ events.push({
+ type: `data-${dataPart.dataType}`,
+ id: `${tc.toolCallId}-data-${index}`,
+ data: isKeyedPayload(data)
+ ? { ...data, tool_call_id: tc.toolCallId }
+ : data,
+ });
+ });
}
events.push({ type: "finish-step" });
events.push({
diff --git a/e2e/test/scenarios/explorations/explorations.cy.spec.ts b/e2e/test/scenarios/explorations/explorations.cy.spec.ts
index 553fe400fe6..720335f65c0 100644
--- a/e2e/test/scenarios/explorations/explorations.cy.spec.ts
+++ b/e2e/test/scenarios/explorations/explorations.cy.spec.ts
@@ -296,11 +296,19 @@ describe("scenarios > explorations > new research > metabot flow", () => {
{
toolCallId: "groups-1",
toolName: "add_research_groups",
- result: {
- metrics: [firstMetric],
- dimension_groups: [interestingGroup],
- groups: [{ metric_id: firstMetric.id }],
- },
+ // The tool result is only the LLM's summary of the edit; the picker
+ // hydration rides a `research_plan_update` data part.
+ result: `Added 1 group(s) to the research plan:\n- ${firstMetric.name}, by the automatically-selected dimensions`,
+ dataParts: [
+ {
+ dataType: "research_plan_update",
+ data: {
+ metrics: [firstMetric],
+ dimension_groups: [interestingGroup],
+ groups: [{ metric_id: firstMetric.id }],
+ },
+ },
+ ],
},
{
toolCallId: "name-1",
diff --git a/frontend/src/metabase/api/ai-streaming/schemas.ts b/frontend/src/metabase/api/ai-streaming/schemas.ts
index 0fbf0d3f9e6..e2fb5a8d2d7 100644
--- a/frontend/src/metabase/api/ai-streaming/schemas.ts
+++ b/frontend/src/metabase/api/ai-streaming/schemas.ts
@@ -1,6 +1,7 @@
import * as Yup from "yup";
import type {
+ AddResearchGroupsResponse,
CardDisplayType,
DatasetQuery,
MetabotCodeEdit,
@@ -47,6 +48,7 @@ export const knownDataPartTypes = [
"data-search_results",
"data-tool_title",
"data-conversation-title",
+ "data-research_plan_update",
] as const satisfies readonly KnownDataPart["type"][];
export type SearchResultItem = {
@@ -120,6 +122,13 @@ export type ToolTitleData = {
title: string;
};
+// The picker hydration for a Research plan edit. It rides a data part rather than the
+// `add_research_groups` tool result because that result is also the agent's LLM context, and this
+// payload grows with every metric the call references.
+export type ResearchPlanUpdateData = AddResearchGroupsResponse & {
+ tool_call_id?: string;
+};
+
export type KnownDataPart =
| { type: "data-navigate_to"; data: string }
| { type: "data-state"; data: Record<string, unknown> }
@@ -132,7 +141,8 @@ export type KnownDataPart =
| { type: "data-static_viz"; data: StaticVizValue }
| { type: "data-search_results"; data: SearchResultsData }
| { type: "data-tool_title"; data: ToolTitleData }
- | { type: "data-conversation-title"; data: string };
+ | { type: "data-conversation-title"; data: string }
+ | { type: "data-research_plan_update"; data: ResearchPlanUpdateData };
export const isKnownDataPart = (part: {
type: string;
diff --git a/frontend/src/metabase/explorations/components/NewExplorationChat/NewExplorationChat.tsx b/frontend/src/metabase/explorations/components/NewExplorationChat/NewExplorationChat.tsx
index 77c77e3fca2..3ce6d1322c2 100644
--- a/frontend/src/metabase/explorations/components/NewExplorationChat/NewExplorationChat.tsx
+++ b/frontend/src/metabase/explorations/components/NewExplorationChat/NewExplorationChat.tsx
@@ -21,20 +21,18 @@ import {
useUserMetabotPermissions,
} from "metabase/metabot/hooks";
import type {
+ MetabotAgentDataPartMessage,
MetabotChatMessage,
+ MetabotDataPart,
MetabotDebugToolCallMessage,
} from "metabase/metabot/state";
import { Box, Flex, Stack, Text } from "metabase/ui";
-import type {
- AddResearchGroupsResponse,
- RemoveFromResearchPlanResponse,
-} from "metabase-types/api";
+import type { RemoveFromResearchPlanResponse } from "metabase-types/api";
import S from "./NewExplorationChat.module.css";
export const EXPLORATIONS_AGENT_ID = "explorations";
-const ADD_RESEARCH_GROUPS_TOOL = "add_research_groups";
const REMOVE_FROM_RESEARCH_PLAN_TOOL = "remove_from_research_plan";
const SET_RESEARCH_NAME_TOOL = "set_research_name";
const SELECT_RESEARCH_TIMELINES_TOOL = "select_research_timelines";
@@ -43,6 +41,12 @@ type MetabotToolCallMessageWithResult = MetabotDebugToolCallMessage & {
result: string;
};
+// `add_research_groups` delivers its picker hydration on a data part rather than in its tool
+// result, because that result is also the agent's LLM context.
+type ResearchPlanUpdateMessage = MetabotAgentDataPartMessage & {
+ part: Extract<MetabotDataPart, { type: "data-research_plan_update" }>;
+};
+
export interface NewExplorationChatProps {
selection: ExplorationSelection;
}
@@ -103,18 +107,16 @@ export function NewExplorationChat({ selection }: NewExplorationChatProps) {
const [sendToast] = useToast();
- const handleAddResearchGroupsToolCallMessages = useCallback(
- (messages: MetabotToolCallMessageWithResult[]) => {
+ const handleResearchPlanUpdateMessages = useCallback(
+ (messages: ResearchPlanUpdateMessage[]) => {
const trackMetricsEdited = once(() =>
trackExplorationPlanEdited("agent", "metrics"),
);
try {
for (const message of messages) {
- // tool result shape verified by the backend
- const { metrics, dimension_groups, groups } = JSON.parse(
- message.result,
- ) as AddResearchGroupsResponse;
+ // data part shape verified by the backend
+ const { metrics, dimension_groups, groups } = message.part.data;
const metricsById = new Map(metrics.map((m) => [m.id, m] as const));
const dimensionsById = indexDimensionsById(dimension_groups);
@@ -249,8 +251,8 @@ export function NewExplorationChat({ selection }: NewExplorationChatProps) {
processedMessageIdsRef.current.add(message.id);
}
- handleAddResearchGroupsToolCallMessages(
- unprocessedMessages.filter(isAddResearchGroupsToolCallMessage),
+ handleResearchPlanUpdateMessages(
+ unprocessedMessages.filter(isResearchPlanUpdateMessage),
);
handleRemoveFromResearchPlanToolCallMessages(
unprocessedMessages.filter(isRemoveFromResearchPlanToolCallMessage),
@@ -263,7 +265,7 @@ export function NewExplorationChat({ selection }: NewExplorationChatProps) {
);
}, [
isDoingScience,
- handleAddResearchGroupsToolCallMessages,
+ handleResearchPlanUpdateMessages,
handleRemoveFromResearchPlanToolCallMessages,
handleSetExplorationNameToolCallMessages,
handleSelectExplorationTimelinesToolCallMessages,
@@ -348,15 +350,13 @@ export function NewExplorationChat({ selection }: NewExplorationChatProps) {
);
}
-function isAddResearchGroupsToolCallMessage(
+function isResearchPlanUpdateMessage(
message: MetabotChatMessage,
-): message is MetabotToolCallMessageWithResult {
+): message is ResearchPlanUpdateMessage {
return (
message.role === "agent" &&
- message.type === "tool_call" &&
- message.name === ADD_RESEARCH_GROUPS_TOOL &&
- !message.is_error &&
- !!message.result
+ message.type === "data_part" &&
+ message.part.type === "data-research_plan_update"
);
}
diff --git a/frontend/src/metabase/explorations/components/NewExplorationChat/NewExplorationChat.unit.spec.tsx b/frontend/src/metabase/explorations/components/NewExplorationChat/NewExplorationChat.unit.spec.tsx
index 8fc96ff35dc..4d2e79bcd0a 100644
--- a/frontend/src/metabase/explorations/components/NewExplorationChat/NewExplorationChat.unit.spec.tsx
+++ b/frontend/src/metabase/explorations/components/NewExplorationChat/NewExplorationChat.unit.spec.tsx
@@ -11,6 +11,7 @@ import {
import { makeMockSelection } from "metabase/explorations/test-utils";
import { useMetabotAgent } from "metabase/metabot/hooks";
import type {
+ MetabotAgentDataPartMessage,
MetabotChatMessage,
MetabotDebugToolCallMessage,
} from "metabase/metabot/state";
@@ -148,13 +149,14 @@ const addResearchGroupsResponse: AddResearchGroupsResponse = {
],
};
-const addResearchGroupsToolCallMessage: MetabotDebugToolCallMessage = {
- id: "tool-call-2",
+const researchPlanUpdateMessage: MetabotAgentDataPartMessage = {
+ id: "data-part-2",
role: "agent",
- type: "tool_call",
- name: "add_research_groups",
- status: "ended",
- result: JSON.stringify(addResearchGroupsResponse),
+ type: "data_part",
+ part: {
+ type: "data-research_plan_update",
+ data: addResearchGroupsResponse,
+ },
};
const setNameToolCallMessage: MetabotDebugToolCallMessage = {
@@ -302,7 +304,7 @@ describe("NewExplorationChat", () => {
});
});
- it("adds groups from an add_research_groups tool call response", async () => {
+ it("adds groups from a research_plan_update data part", async () => {
const { selection, rerender } = setup();
rerender({
@@ -310,11 +312,7 @@ describe("NewExplorationChat", () => {
isDoingScience: true,
});
rerender({
- messages: [
- userMessage,
- searchToolCallMessage,
- addResearchGroupsToolCallMessage,
- ],
+ messages: [userMessage, searchToolCallMessage, researchPlanUpdateMessage],
isDoingScience: true,
});
@@ -324,7 +322,7 @@ describe("NewExplorationChat", () => {
messages: [
userMessage,
searchToolCallMessage,
- addResearchGroupsToolCallMessage,
+ researchPlanUpdateMessage,
agentMessage,
],
isDoingScience: false,
@@ -370,19 +368,22 @@ describe("NewExplorationChat", () => {
it("forwards replace_default_dimensions to addMetric", async () => {
const { selection, rerender } = setup();
- const message: MetabotDebugToolCallMessage = {
- ...addResearchGroupsToolCallMessage,
- id: "tool-call-replace",
- result: JSON.stringify({
- ...addResearchGroupsResponse,
- groups: [
- {
- metric_id: metricRevenue.id,
- dimension_ids: [revenueDateDimension.id],
- replace_default_dimensions: true,
- },
- ],
- }),
+ const message: MetabotAgentDataPartMessage = {
+ ...researchPlanUpdateMessage,
+ id: "data-part-replace",
+ part: {
+ type: "data-research_plan_update",
+ data: {
+ ...addResearchGroupsResponse,
+ groups: [
+ {
+ metric_id: metricRevenue.id,
+ dimension_ids: [revenueDateDimension.id],
+ replace_default_dimensions: true,
+ },
+ ],
+ },
+ },
};
rerender({ messages: [userMessage, message], isDoingScience: true });
diff --git a/frontend/src/metabase/metabot/components/MetabotChat/MetabotAgentDataPartMessage.tsx b/frontend/src/metabase/metabot/components/MetabotChat/MetabotAgentDataPartMessage.tsx
index 44593d5c066..3b43938793d 100644
--- a/frontend/src/metabase/metabot/components/MetabotChat/MetabotAgentDataPartMessage.tsx
+++ b/frontend/src/metabase/metabot/components/MetabotChat/MetabotAgentDataPartMessage.tsx
@@ -129,6 +129,9 @@ export const AgentDataPartMessage = ({
.with({ part: { type: "data-static_viz" } }, ({ part }) =>
debug ? <DataPartJsonCard type={part.type} value={part.data} /> : null,
)
+ .with({ part: { type: "data-research_plan_update" } }, ({ part }) =>
+ debug ? <DataPartJsonCard type={part.type} value={part.data} /> : null,
+ )
.exhaustive((msg: unknown) => {
console.warn("AgentDataPartMessage received an unexpected value:", msg);
return null;
diff --git a/frontend/src/metabase/metabot/components/MetabotChat/MetabotChatMessage.tsx b/frontend/src/metabase/metabot/components/MetabotChat/MetabotChatMessage.tsx
index 7f93ec882ba..082585682e8 100644
--- a/frontend/src/metabase/metabot/components/MetabotChat/MetabotChatMessage.tsx
+++ b/frontend/src/metabase/metabot/components/MetabotChat/MetabotChatMessage.tsx
@@ -60,6 +60,7 @@ const isUserVisibleDataPart = (part: MetabotDataPart): boolean =>
.with({ type: "data-entity_saved" }, () => true)
.with({ type: "data-adhoc_viz" }, () => false)
.with({ type: "data-static_viz" }, () => false)
+ .with({ type: "data-research_plan_update" }, () => false)
.exhaustive();
const isUserVisibleDataPartMessage = (
diff --git a/frontend/src/metabase/metabot/state/actions.ts b/frontend/src/metabase/metabot/state/actions.ts
index edee82f132e..2dc73dfdba9 100644
--- a/frontend/src/metabase/metabot/state/actions.ts
+++ b/frontend/src/metabase/metabot/state/actions.ts
@@ -532,6 +532,9 @@ export const sendAgentRequest = createAsyncThunk<
.with({ type: "data-todo_list" }, (part) => {
pushDataPart({ type: "data_part", part });
})
+ .with({ type: "data-research_plan_update" }, (part) => {
+ pushDataPart({ type: "data_part", part });
+ })
.with({ type: "data-code_edit" }, (part) => {
dispatch(addSuggestedCodeEdit({ ...part.data, active: true }));
diff --git a/src/metabase/explorations/impl.clj b/src/metabase/explorations/impl.clj
index 0715d978c5d..fdaeb76d452 100644
--- a/src/metabase/explorations/impl.clj
+++ b/src/metabase/explorations/impl.clj
@@ -502,24 +502,36 @@
referenced metrics, and `:groups` echoes the validated specs for the FE to turn into picker
blocks."
[{:keys [groups]}]
- (let [all (mapv with-candidate-dimensions (hydrated-metrics {}))
- metric-by-id (u/index-by :id all)]
- (doseq [g groups]
- (let [metric-id (:metric_id g)
- metric (get metric-by-id metric-id)]
- (when-not metric
- (throw (ex-info (format "Unknown or inaccessible metric id %s" metric-id)
- {:metric_id metric-id})))
- (when (and (:replace_default_dimensions g) (empty? (:dimension_ids g)))
- (throw (ex-info "replace_default_dimensions requires at least one dimension_id"
- {:metric_id metric-id})))
- (let [valid (set (map :id (:dimensions metric)))]
- (doseq [d (:dimension_ids g)]
- (when-not (contains? valid d)
- (throw (ex-info (format "Dimension %s is not a candidate of metric %s" d metric-id)
- {:metric_id metric-id :dimension_id d})))))))
- (let [relevant (into #{} (map :metric_id) groups)
- relevant-metrics (filterv #(contains? relevant (:id %)) all)]
- {:metrics (mapv slim-metric relevant-metrics)
- :dimension_groups (group-dimensions relevant-metrics)
- :groups (vec groups)})))
+ (lib-be/with-metadata-provider-cache
+ (let [all (mapv with-candidate-dimensions (catalog-metrics {}))
+ metric-by-id (u/index-by :id all)]
+ ;; Metric-level checks run against the cheap catalog — they need no query build.
+ (doseq [g groups]
+ (let [metric-id (:metric_id g)]
+ (when-not (contains? metric-by-id metric-id)
+ (throw (ex-info (format "Unknown or inaccessible metric id %s" metric-id)
+ {:metric_id metric-id})))
+ (when (and (:replace_default_dimensions g) (empty? (:dimension_ids g)))
+ (throw (ex-info "replace_default_dimensions requires at least one dimension_id"
+ {:metric_id metric-id})))))
+ (let [relevant (into #{} (map :metric_id) groups)
+ ;; Only the referenced metrics pay [[resolve-metric-queries]]. `all` is already
+ ;; candidate-filtered and resolution only drops dimensions further, so no second
+ ;; [[with-candidate-dimensions]] pass is needed.
+ relevant-metrics (->> all
+ (filterv #(contains? relevant (:id %)))
+ resolve-metric-queries)
+ resolved-by-id (u/index-by :id relevant-metrics)]
+ ;; Dimension ids are checked against the *resolved* metrics: a saved dimension whose
+ ;; target no longer resolves is dropped by [[resolve-metric-queries]], and accepting it
+ ;; would hand the FE a picker block for a dimension that isn't in the payload.
+ (doseq [g groups
+ :let [metric-id (:metric_id g)
+ valid (set (map :id (:dimensions (get resolved-by-id metric-id))))]
+ d (:dimension_ids g)]
+ (when-not (contains? valid d)
+ (throw (ex-info (format "Dimension %s is not a candidate of metric %s" d metric-id)
+ {:metric_id metric-id :dimension_id d}))))
+ {:metrics (mapv slim-metric relevant-metrics)
+ :dimension_groups (group-dimensions relevant-metrics)
+ :groups (vec groups)}))))
diff --git a/src/metabase/metabot/agent/streaming.clj b/src/metabase/metabot/agent/streaming.clj
index 3772d726b6c..6ae068d923d 100644
--- a/src/metabase/metabot/agent/streaming.clj
+++ b/src/metabase/metabot/agent/streaming.clj
@@ -23,6 +23,7 @@
(def static-viz-type "AI-SDK data type for static visualizations." "static_viz")
(def search-results-type "AI-SDK data type for a search tool's result list." "search_results")
(def tool-title-type "AI-SDK data type for a tool call's settled display title." "tool_title")
+(def research-plan-update-type "AI-SDK data type for a Research plan edit's picker hydration." "research_plan_update")
(def ^:private ephemeral-data-types
"Data types not written to MetabotMessage.data."
@@ -158,6 +159,17 @@
:data-type search-results-type
:data value})
+(defn research-plan-update-part
+ "Data part carrying the picker hydration for a Research plan edit — the metrics and
+ dimension groups the FE needs to turn the agent's chosen groups into picker blocks.
+ It rides here rather than in the tool's `:output` because the LLM only needs to know
+ the edit happened, while the payload is large and grows with the metrics a group
+ pulls in."
+ [value]
+ {:type :data
+ :data-type research-plan-update-type
+ :data value})
+
(defn tool-title-part
"Data part carrying a tool call's display title, derived from what the tool
actually read/did; [[expand-data-parts-xf]] stamps on the tool-call id it
diff --git a/src/metabase/metabot/tools/explorations.clj b/src/metabase/metabot/tools/explorations.clj
index d44c1a94e1b..84804b845b8 100644
--- a/src/metabase/metabot/tools/explorations.clj
+++ b/src/metabase/metabot/tools/explorations.clj
@@ -1,14 +1,18 @@
(ns metabase.metabot.tools.explorations
"Exploration-specific tool wrappers.
- Every tool here returns `{:output <json-string>}`: the exploration chat FE applies plan edits
- by parsing the tool result it sees on the `tool-output-available` stream event, and only a
- result's `:output` string makes it onto the wire (see
- `metabase.metabot.self.core/tool-output->wire-output`) — a bare map would reach the LLM but
- stream to the client as an empty string, and the plan would silently never update."
+ A tool result's `:output` is the one string that goes to *both* the LLM and the client,
+ so the small plan edits the FE applies by parsing the tool result — `remove_from_research_plan`, `set_research_name`,
+ `select_research_timelines` — return `{:output <json-string>}`; a bare map would reach the LLM
+ but stream to the client as an empty string, and the plan would silently never update.
+
+ `add_research_groups` is the exception: its picker hydration is far too large to spend LLM
+ context on and grows with every metric the call references, so it rides a `research_plan_update`
+ data part (FE only) while `:output` carries a plain-text summary."
(:require
[clojure.string :as str]
[metabase.explorations.core :as explorations]
+ [metabase.metabot.agent.streaming :as streaming]
[metabase.metabot.scope :as scope]
[metabase.metabot.tmpl :as te]
[metabase.util.json :as json]
@@ -111,6 +115,51 @@
:else
{:output (json/encode (explorations/research-candidates {:metric-ids metric_ids :q q}))}))
+(defn- research-groups-payload
+ "Validate `groups` and hydrate the FE picker payload for them, in the snake_case shape the
+ client consumes."
+ [groups]
+ (explorations/exploration-data->api (explorations/research-groups {:groups groups})))
+
+(def ^:private summary-max-names
+ "How many dimension names one summary line spells out before falling back to a count. A metric
+ can be sliced by a long list of candidate dimensions, and the summary is the whole of what the
+ LLM sees of the result."
+ 15)
+
+(defn- summarize-names [names]
+ (let [extra (- (count names) summary-max-names)]
+ (if (pos? extra)
+ (str (str/join ", " (take summary-max-names names)) " and " extra " more")
+ (str/join ", " names))))
+
+(defn- format-added-groups
+ "Plain-text summary of the groups `add_research_groups` just applied — the whole of what the
+ LLM sees of the result, since the picker hydration rides a data part instead. Names what the
+ user now has in the plan so the agent can describe it without re-reading the catalog; ids are
+ deliberately absent, as the next turn's plan snapshot carries
+ the `block_id`s needed to edit these groups."
+ [{:keys [metrics dimension_groups groups]}]
+ (let [metric-name (into {} (map (juxt :id :name)) metrics)
+ group-idx (into {} (for [[i g] (map-indexed vector dimension_groups)
+ d (:dimensions g)]
+ [(:id d) i]))
+ group-names (mapv :name dimension_groups)
+ ;; Nil for a dimension the payload doesn't group: groups are validated against the
+ ;; unresolved catalog, so a dimension can pass validation and still be dropped from the
+ ;; hydrated metrics because their query can't actually break out on it. The summary skips
+ ;; what it can't name rather than failing the whole tool call.
+ group-name (fn [dimension-id] (some-> (group-idx dimension-id) group-names))]
+ (te/lines
+ (str "Added " (count groups) " group(s) to the research plan:")
+ (for [g groups]
+ (let [dims (seq (keep group-name (:dimension_ids g)))]
+ (str "- " (metric-name (:metric_id g))
+ (cond
+ (and dims (:replace_default_dimensions g)) (str ", by exactly: " (summarize-names dims))
+ dims (str ", by: " (summarize-names dims) ", plus the automatic selection")
+ :else ", by the automatically-selected dimensions")))))))
+
(def ^:private add-research-groups-schema
[:map {:closed true}
[:groups
@@ -130,8 +179,9 @@
list (no automatic ones), also pass `\"replace_default_dimensions\": true` - then
`dimension_ids` must be non-empty."
[{:keys [groups]} :- add-research-groups-schema]
- {:output (json/encode (explorations/exploration-data->api
- (explorations/research-groups {:groups groups})))})
+ (let [payload (research-groups-payload groups)]
+ {:output (format-added-groups payload)
+ :data-parts [(streaming/research-plan-update-part payload)]}))
(def ^:private remove-from-research-plan-schema
[:map {:closed true}
diff --git a/test/metabase/explorations/impl_test.clj b/test/metabase/explorations/impl_test.clj
index a6fd67508f9..a257bdd1379 100644
--- a/test/metabase/explorations/impl_test.clj
+++ b/test/metabase/explorations/impl_test.clj
@@ -139,7 +139,8 @@
(defmacro ^:private with-synthetic-metrics [& body]
`(mt/with-dynamic-fn-redefs [explorations.impl/hydrated-metrics synthetic-hydrated
- explorations.impl/index-metrics synthetic-hydrated]
+ explorations.impl/index-metrics synthetic-hydrated
+ explorations.impl/catalog-metrics synthetic-hydrated]
~@body))
(deftest research-metric-index-test
@@ -361,7 +362,18 @@
(testing "replace_default_dimensions with no dimension_ids"
(is (thrown-with-msg? Exception #"replace_default_dimensions requires"
(explorations.impl/research-groups
- {:groups [{:metric_id 1 :replace_default_dimensions true}]}))))))
+ {:groups [{:metric_id 1 :replace_default_dimensions true}]}))))
+ (testing "dimension the catalog still lists but query resolution drops"
+ ;; `plan-1` survives the cheap catalog but its target no longer resolves against the
+ ;; metric's query, so it must be rejected rather than silently dropped from the payload.
+ (mt/with-dynamic-fn-redefs [explorations.impl/resolve-metric-queries
+ (fn [metrics]
+ (mapv #(update % :dimensions
+ (fn [ds] (filterv (comp #{"region-1"} :id) ds)))
+ metrics))]
+ (is (thrown-with-msg? Exception #"not a candidate of metric"
+ (explorations.impl/research-groups
+ {:groups [{:metric_id 1 :dimension_ids ["plan-1"]}]})))))))
;;; ------------------------------------------ exploration-data ------------------------------------------
diff --git a/test/metabase/metabot/tools/explorations_test.clj b/test/metabase/metabot/tools/explorations_test.clj
index 756b3933cc7..745b1b957f6 100644
--- a/test/metabase/metabot/tools/explorations_test.clj
+++ b/test/metabase/metabot/tools/explorations_test.clj
@@ -5,9 +5,11 @@
[metabase.api-scope.core :as api-scope]
[metabase.metabot.agent.messages :as messages]
[metabase.metabot.agent.profiles :as profiles]
+ [metabase.metabot.agent.streaming :as streaming]
[metabase.metabot.scope :as scope]
[metabase.metabot.self.core :as metabot.self]
[metabase.metabot.tools.explorations :as tools.explorations]
+ [metabase.test :as mt]
[metabase.util.json :as json]))
(def ^:private exploration-tool-vars
@@ -57,6 +59,71 @@
(is (str/includes? output "at most 20"))
(is (str/includes? output "got 21")))))
+;;; ------------------------------------ add_research_groups ------------------------------------
+
+(defn- picker-payload
+ "An `add_research_groups` result: `n-metrics` metrics, all sliceable by one dimension group."
+ [n-metrics]
+ {:metrics (vec (for [i (range n-metrics)]
+ {:id i :name (str "Metric " i) :dimension_ids [(str "d" i)]}))
+ :dimension_groups [{:name "Orders - Region"
+ :dimensions (vec (for [i (range n-metrics)] {:id (str "d" i)}))}]
+ :groups []})
+
+(defn- wide-metric-payload
+ "An `add_research_groups` result: one metric sliced by `n-dims` distinct dimension groups."
+ [n-dims]
+ (let [ids (vec (for [i (range n-dims)] (str "d" i)))]
+ {:metrics [{:id 0 :name "Metric 0" :dimension_ids ids}]
+ :dimension_groups (vec (for [i (range n-dims)]
+ {:name (str "Group " i) :dimensions [{:id (str "d" i)}]}))
+ :groups [{:metric_id 0 :dimension_ids ids}]}))
+
+(deftest add-research-groups-keeps-the-payload-off-the-llm-wire-test
+ (testing "the picker hydration rides a data part the FE consumes; the LLM only sees a summary"
+ (let [payload (assoc (picker-payload 3)
+ :groups [{:metric_id 0 :dimension_ids ["d0"]}
+ {:metric_id 1 :dimension_ids ["d1"] :replace_default_dimensions true}
+ {:metric_id 2}])]
+ (mt/with-dynamic-fn-redefs [tools.explorations/research-groups-payload (fn [_] payload)]
+ (let [{:keys [output data-parts]} (tools.explorations/add-research-groups-tool
+ {:groups (:groups payload)})]
+ (testing "the payload is on the data part, not in :output"
+ (is (= [{:type :data :data-type "research_plan_update" :data payload}] data-parts)))
+ (testing ":output is prose naming what the user got, not the payload"
+ (is (= (str "Added 3 group(s) to the research plan:\n"
+ "- Metric 0, by: Orders - Region, plus the automatic selection\n"
+ "- Metric 1, by exactly: Orders - Region\n"
+ "- Metric 2, by the automatically-selected dimensions")
+ output))))))))
+
+(deftest add-research-groups-summary-tolerates-unnamed-dimensions-test
+ (testing (str "groups are validated against the unresolved catalog, so a dimension can pass "
+ "validation and still be absent from the hydrated payload (its metric's query "
+ "can't break out on it) — the summary skips what it can't name rather than "
+ "throwing away the whole tool call")
+ (let [payload (assoc (picker-payload 1)
+ :groups [{:metric_id 0 :dimension_ids ["d0" "unresolved"]}])]
+ (mt/with-dynamic-fn-redefs [tools.explorations/research-groups-payload (fn [_] payload)]
+ (is (= (str "Added 1 group(s) to the research plan:\n"
+ "- Metric 0, by: Orders - Region, plus the automatic selection")
+ (:output (tools.explorations/add-research-groups-tool
+ {:groups (:groups payload)}))))))))
+
+(deftest add-research-groups-summary-is-bounded-test
+ (testing "a metric can be sliced by a long dimension list, so the summary counts the tail
+ instead of listing it — :output is the agent's context, not a report"
+ (let [summary-for (fn [n-dims]
+ (let [payload (wide-metric-payload n-dims)]
+ (mt/with-dynamic-fn-redefs [tools.explorations/research-groups-payload (fn [_] payload)]
+ (:output (tools.explorations/add-research-groups-tool
+ {:groups (:groups payload)})))))
+ fifty (summary-for 50)]
+ (testing "the tail past the first `summary-max-names` members is counted, not listed"
+ (is (str/includes? fifty "and 35 more")))
+ (testing "so a 10x longer dimension list costs only the extra digits in that count"
+ (is (> 5 (- (count (summary-for 500)) (count fifty))))))))
+
(deftest ^:parallel remove-from-research-plan-tool-test
(testing "echoes the block ids the agent asked to remove (pure-echo; the FE applies them)"
(is (= {:block_ids ["metric:42" "metric:43"] :members nil :timeline_ids nil}
@@ -94,10 +161,28 @@
events)]
(some-> (:output event) json/decode+kw)))
+(deftest ^:parallel research-plan-update-part-reaches-the-wire-test
+ (testing (str "add_research_groups' picker payload is the plan edit, and it now travels as a "
+ "data part rather than in :output — if it never reached the client the plan would "
+ "silently never update, the same failure the :output tools guard against below")
+ (let [payload {:metrics [{:id 1}] :dimension_groups [] :groups []}
+ events (into [] (comp streaming/expand-data-parts-xf (metabot.self/parts->aisdk-sse-xf))
+ [{:type :start :id "s1"}
+ {:type :tool-output :id "tc1"
+ :result {:output "Added 1 group(s) to the research plan:"
+ :data-parts [(streaming/research-plan-update-part payload)]}}])
+ event (some #(when (str/includes? % "\"data-research_plan_update\"")
+ (json/decode+kw (str/replace-first % "data: " "")))
+ events)]
+ (is (some? event) "a data-research_plan_update event is streamed")
+ (is (= [{:id 1}] (get-in event [:data :metrics])))
+ (testing "stamped with the tool call it came from, so the client can attribute it"
+ (is (= "tc1" (get-in event [:data :tool_call_id])))))))
+
(deftest ^:parallel plan-tool-results-reach-the-wire-test
- (testing (str "the exploration chat FE applies plan edits by parsing the streamed tool result; "
- "only a result's :output string makes it onto tool-output-available, so a bare-map "
- "result would stream as \"\" and the plan would silently never update")
+ (testing (str "the exploration chat FE applies the smaller plan edits by parsing the streamed "
+ "tool result; only a result's :output string makes it onto tool-output-available, "
+ "so a bare-map result would stream as \"\" and the plan would silently never update")
(testing "set_research_name"
(is (= {:name "Quarterly revenue"}
(streamed-tool-output