Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[Metabot] Context window improvements (#81133)

Sloan Spargerd273dc56b6a9
diff --git a/enterprise/frontend/src/embedding-sdk-ee/metabot/MetabotChatHistory.unit.spec.tsx b/enterprise/frontend/src/embedding-sdk-ee/metabot/MetabotChatHistory.unit.spec.tsx
index 22ef4bce815..3693bcaa3c1 100644
--- a/enterprise/frontend/src/embedding-sdk-ee/metabot/MetabotChatHistory.unit.spec.tsx
+++ b/enterprise/frontend/src/embedding-sdk-ee/metabot/MetabotChatHistory.unit.spec.tsx
@@ -28,11 +28,22 @@ describe("MetabotChatHistory", () => {
const { store } = setup({
ui: <MetabotChatHistory />,
metabotInitialState: assocIn(
- makeVisibleState([
- { id: "1", role: "user", type: "text", message: "hi" },
- ]),
- ["conversations", testConversationId("omnibot"), "lastTokenUsage"],
- { contextTokens: 200, contextWindowTokens: 200 },
+ assocIn(
+ makeVisibleState([
+ { id: "1", role: "user", type: "text", message: "hi" },
+ { id: "2", role: "agent", type: "text", message: "hello" },
+ ]),
+ [
+ "conversations",
+ testConversationId("omnibot"),
+ "messages",
+ 1,
+ "contextTokens",
+ ],
+ 200,
+ ),
+ ["conversations", testConversationId("omnibot"), "contextWindowTokens"],
+ 200,
),
});
diff --git a/frontend/src/metabase/metabot/api.ts b/frontend/src/metabase/metabot/api.ts
index b8b9a581c4e..c672144878c 100644
--- a/frontend/src/metabase/metabot/api.ts
+++ b/frontend/src/metabase/metabot/api.ts
@@ -31,6 +31,7 @@ export type MetabotConversationDetail = {
forked_from_conversation_id: string | null;
state?: MetabotStateContext;
messages: MetabotMessage[];
+ context_window_tokens?: number;
};
export const metabotApi = Api.injectEndpoints({
diff --git a/frontend/src/metabase/metabot/components/MetabotChat/MetabotChat.tsx b/frontend/src/metabase/metabot/components/MetabotChat/MetabotChat.tsx
index 9c3911ce9c3..c70b0f66d30 100644
--- a/frontend/src/metabase/metabot/components/MetabotChat/MetabotChat.tsx
+++ b/frontend/src/metabase/metabot/components/MetabotChat/MetabotChat.tsx
@@ -208,37 +208,40 @@ export const MetabotChat = ({
onNewChat={onNewConversation}
/>
)}
- <Paper
- className={cx(
- Styles.inputContainer,
- metabot.isDoingScience && Styles.inputContainerLoading,
- )}
- >
- <MetabotChatEditor
- ref={metabot.promptInputRef}
- value={metabot.prompt}
- autoFocus
- isResponding={metabot.isDoingScience}
- placeholder={t`How can I help? Type @ to mention items.`}
- onChange={metabot.setPrompt}
- onSubmit={() => metabot.submitInput(metabot.prompt)}
- onStop={metabot.cancelRequest}
- suggestionConfig={{
- suggestionModels: config.suggestionModels,
- }}
- />
- </Paper>
+ {!metabot.isContextWindowFull && (
+ <Paper
+ className={cx(
+ Styles.inputContainer,
+ metabot.isDoingScience && Styles.inputContainerLoading,
+ )}
+ >
+ <MetabotChatEditor
+ ref={metabot.promptInputRef}
+ value={metabot.prompt}
+ autoFocus
+ isResponding={metabot.isDoingScience}
+ placeholder={t`How can I help? Type @ to mention items.`}
+ onChange={metabot.setPrompt}
+ onSubmit={() => metabot.submitInput(metabot.prompt)}
+ onStop={metabot.cancelRequest}
+ suggestionConfig={{
+ suggestionModels: config.suggestionModels,
+ }}
+ />
+ </Paper>
+ )}
</Box>
<Box className={Styles.footerRow}>
<Text fz="sm" c="text-secondary" ta="center">
{t`${metabotName} isn't perfect. Double-check results.`}
</Text>
- {metabot.contextWindowPercentUsage > 50 && (
- <MetabotContextUsageRing
- className={Styles.contextUsage}
- percentUsage={metabot.contextWindowPercentUsage}
- />
- )}
+ {metabot.contextWindowPercentUsage > 50 &&
+ !metabot.isContextWindowFull && (
+ <MetabotContextUsageRing
+ className={Styles.contextUsage}
+ percentUsage={metabot.contextWindowPercentUsage}
+ />
+ )}
</Box>
</Box>
)}
diff --git a/frontend/src/metabase/metabot/components/MetabotConversationPage/MetabotConversationPage.tsx b/frontend/src/metabase/metabot/components/MetabotConversationPage/MetabotConversationPage.tsx
index aed9c1c89ee..b37761c7839 100644
--- a/frontend/src/metabase/metabot/components/MetabotConversationPage/MetabotConversationPage.tsx
+++ b/frontend/src/metabase/metabot/components/MetabotConversationPage/MetabotConversationPage.tsx
@@ -60,6 +60,7 @@ export const MetabotConversationPage = () => {
title: conversation.title ?? undefined,
forkedFromConversationId:
conversation.forked_from_conversation_id ?? undefined,
+ contextWindowTokens: conversation.context_window_tokens,
messages: conversation.messages,
state: conversation.state,
activeToolCalls: [],
diff --git a/frontend/src/metabase/metabot/state/actions.ts b/frontend/src/metabase/metabot/state/actions.ts
index 12d8d4c8a40..1c798ce9b0c 100644
--- a/frontend/src/metabase/metabot/state/actions.ts
+++ b/frontend/src/metabase/metabot/state/actions.ts
@@ -950,6 +950,7 @@ export const fetchConversationSnapshot = createAsyncThunk(
title: detail.title ?? undefined,
forkedFromConversationId:
detail.forked_from_conversation_id ?? undefined,
+ contextWindowTokens: detail.context_window_tokens,
messages: detail.messages,
state: detail.state,
activeToolCalls: [],
@@ -1008,6 +1009,7 @@ export const forkConversation = createAsyncThunk(
title: conversation.title ?? undefined,
forkedFromConversationId:
conversation.forked_from_conversation_id ?? undefined,
+ contextWindowTokens: conversation.context_window_tokens,
messages: conversation.messages,
state: conversation.state,
activeToolCalls: [],
diff --git a/frontend/src/metabase/metabot/state/reducer.ts b/frontend/src/metabase/metabot/state/reducer.ts
index 7e219bfde87..10b9ba2a1ed 100644
--- a/frontend/src/metabase/metabot/state/reducer.ts
+++ b/frontend/src/metabase/metabot/state/reducer.ts
@@ -441,6 +441,7 @@ export const metabot = createSlice({
conversationId: string;
title?: string;
forkedFromConversationId?: string;
+ contextWindowTokens?: number;
}>,
) => {
const {
@@ -450,6 +451,7 @@ export const metabot = createSlice({
conversationId,
title,
forkedFromConversationId,
+ contextWindowTokens,
} = action.payload;
const convo =
@@ -463,7 +465,7 @@ export const metabot = createSlice({
convo.activeToolCalls = activeToolCalls ?? [];
convo.title = title;
convo.forkedFromConversationId = forkedFromConversationId;
- convo.lastTokenUsage = undefined;
+ convo.contextWindowTokens = contextWindowTokens;
convo.isProcessing = hasInProgressMessage(messages);
if (convo.isProcessing) {
ensureChain(convo); // resuming mid-response
@@ -502,25 +504,31 @@ export const metabot = createSlice({
convo.state = { ...action.payload.state };
}
+ closeChain(convo);
+ const message = openAgentMessage(convo);
+
const metadata = action.payload?.processedResponse.messageMetadata;
- if (metadata?.contextTokens && metadata.contextWindowTokens) {
- convo.lastTokenUsage = {
- contextTokens: metadata.contextTokens,
- contextWindowTokens: metadata.contextWindowTokens,
- };
- }
+ const contextUsage =
+ metadata?.contextTokens && metadata.contextWindowTokens
+ ? {
+ contextTokens: metadata.contextTokens,
+ contextWindowTokens: metadata.contextWindowTokens,
+ }
+ : undefined;
+ message.contextTokens = contextUsage?.contextTokens;
+ convo.contextWindowTokens =
+ contextUsage?.contextWindowTokens ?? convo.contextWindowTokens;
const finishReason = action.payload?.processedResponse.finishReason;
const isResumableFinishReason =
finishReason && finishReason !== "stop" && finishReason !== "error";
- closeChain(convo);
- openAgentMessage(convo).status = isResumableFinishReason
+ message.status = isResumableFinishReason
? {
type: "incomplete",
finishReason,
contextWindowFull:
finishReason === "length" &&
- isContextWindowFull(convo.lastTokenUsage),
+ isContextWindowFull(contextUsage),
}
: { type: "done" };
diff --git a/frontend/src/metabase/metabot/state/selectors.ts b/frontend/src/metabase/metabot/state/selectors.ts
index 75517505145..3943f374e96 100644
--- a/frontend/src/metabase/metabot/state/selectors.ts
+++ b/frontend/src/metabase/metabot/state/selectors.ts
@@ -17,7 +17,11 @@ import {
isContextWindowFull,
} from "../utils/context-usage";
-import type { MetabotAgentId, MetabotMessage } from "./types";
+import type {
+ MetabotAgentId,
+ MetabotContextUsage,
+ MetabotMessage,
+} from "./types";
import { hasInProgressMessage, isGeneratedCardPart, isTextPart } from "./utils";
/*
@@ -253,15 +257,28 @@ export const getConversationChart = createSelector(
export type MetabotLongChatNoticeVariant = "warning" | "full";
+export const getContextUsage = createSelector(
+ [getMessages, getConversation],
+ (messages, convo): MetabotContextUsage | undefined => {
+ const contextTokens = messages.findLast(
+ (m) => m.role === "agent" && m.contextTokens,
+ )?.contextTokens;
+ const { contextWindowTokens } = convo;
+ return contextTokens && contextWindowTokens
+ ? { contextTokens, contextWindowTokens }
+ : undefined;
+ },
+);
+
export const getContextUsagePercent = createSelector(
- [getConversation],
- (convo): number => getContextWindowPercentUsage(convo.lastTokenUsage),
+ getContextUsage,
+ getContextWindowPercentUsage,
);
export const getLongChatNotice = createSelector(
- [getConversation, getContextUsagePercent],
- (convo, percentUsage): MetabotLongChatNoticeVariant | undefined => {
- if (isContextWindowFull(convo.lastTokenUsage)) {
+ [getContextUsage, getContextUsagePercent],
+ (contextUsage, percentUsage): MetabotLongChatNoticeVariant | undefined => {
+ if (isContextWindowFull(contextUsage)) {
return "full";
}
return percentUsage >= CONTEXT_WINDOW_WARNING_PERCENT
diff --git a/frontend/src/metabase/metabot/state/selectors.unit.spec.ts b/frontend/src/metabase/metabot/state/selectors.unit.spec.ts
index 70a97189cad..253603ae722 100644
--- a/frontend/src/metabase/metabot/state/selectors.unit.spec.ts
+++ b/frontend/src/metabase/metabot/state/selectors.unit.spec.ts
@@ -19,13 +19,12 @@ import {
getUserPromptMessage,
} from "./index";
+const CONTEXT_WINDOW = 200000;
+
function setup(
messages: MetabotMessage[],
convoState?: Partial<
- Pick<
- MetabotConversationState,
- "conversationId" | "lastTokenUsage" | "profileOverride"
- >
+ Pick<MetabotConversationState, "conversationId" | "profileOverride">
>,
): State {
setupEnterprisePlugins();
@@ -38,10 +37,15 @@ function setup(
["conversations", conversationId, "messages"],
messages,
);
+ const withWindow = assocIn(
+ withMessages,
+ ["conversations", conversationId, "contextWindowTokens"],
+ CONTEXT_WINDOW,
+ );
const withConvoState = Object.entries(convoState ?? {}).reduce(
(acc, [key, value]) =>
assocIn(acc, ["conversations", conversationId, key], value),
- withMessages,
+ withWindow,
);
return createMockState({ metabot: withConvoState });
@@ -152,32 +156,30 @@ describe("metabot selectors", () => {
role: "user",
parts: [{ id: "1", role: "user", type: "text", message: "hi" }],
});
- const CONTEXT_WINDOW = 200000;
- const usage = (contextTokens: number) => ({
- lastTokenUsage: { contextTokens, contextWindowTokens: CONTEXT_WINDOW },
- });
+ const usage = (contextTokens: number) =>
+ createMockMetabotMessage({ contextTokens });
it("warns when context tokens reach the warning percent of the window", () => {
- const state = setup(
- [shortMessage],
+ const state = setup([
+ shortMessage,
usage((CONTEXT_WINDOW * CONTEXT_WINDOW_WARNING_PERCENT) / 100),
- );
+ ]);
expect(getLongChatNotice(state, "omnibot")).toBe("warning");
});
it("still only warns just short of the window — the whole window is usable", () => {
- const state = setup([shortMessage], usage(CONTEXT_WINDOW - 1));
+ const state = setup([shortMessage, usage(CONTEXT_WINDOW - 1)]);
expect(getLongChatNotice(state, "omnibot")).toBe("warning");
});
it("reports full once context tokens consume the whole window", () => {
- const state = setup([shortMessage], usage(CONTEXT_WINDOW));
+ const state = setup([shortMessage, usage(CONTEXT_WINDOW)]);
expect(getLongChatNotice(state, "omnibot")).toBe("full");
});
it("shows nothing below the warning percent or when no usage has been observed", () => {
expect(
- getLongChatNotice(setup([shortMessage], usage(10000)), "omnibot"),
+ getLongChatNotice(setup([shortMessage, usage(10000)]), "omnibot"),
).toBeUndefined();
expect(
getLongChatNotice(setup([shortMessage]), "omnibot"),
@@ -186,7 +188,7 @@ describe("metabot selectors", () => {
describe("getContextUsagePercent", () => {
it("reports the last observed usage as a share of the window, 0-100", () => {
- const state = setup([shortMessage], usage(CONTEXT_WINDOW / 4));
+ const state = setup([shortMessage, usage(CONTEXT_WINDOW / 4)]);
expect(getContextUsagePercent(state, "omnibot")).toBe(25);
});
@@ -197,7 +199,7 @@ describe("metabot selectors", () => {
});
it("caps at 100 when the window is overrun", () => {
- const state = setup([shortMessage], usage(CONTEXT_WINDOW * 2));
+ const state = setup([shortMessage, usage(CONTEXT_WINDOW * 2)]);
expect(getContextUsagePercent(state, "omnibot")).toBe(100);
});
});
diff --git a/frontend/src/metabase/metabot/state/types.ts b/frontend/src/metabase/metabot/state/types.ts
index e12d8b53502..6d5193ee99b 100644
--- a/frontend/src/metabase/metabot/state/types.ts
+++ b/frontend/src/metabase/metabot/state/types.ts
@@ -120,12 +120,18 @@ export type MetabotMessagePart =
| MetabotDebugToolCallMessage
| MetabotAgentChainOfThoughtMessage;
+export type MetabotContextUsage = {
+ contextTokens: number;
+ contextWindowTokens: number;
+};
+
export type MetabotMessage = {
id: string;
externalId?: string;
role: "user" | "agent";
parts: MetabotMessagePart[];
status: MetabotMessageStatus;
+ contextTokens?: number;
};
export type MetabotToolCall = {
@@ -155,11 +161,6 @@ export type MetabotReactionsState = {
suggestedTransforms: MetabotSuggestedTransform[];
};
-export type MetabotContextUsage = {
- contextTokens: number;
- contextWindowTokens: number;
-};
-
export interface MetabotConversationState {
conversationId: string;
title: string | undefined;
@@ -170,7 +171,7 @@ export interface MetabotConversationState {
state: MetabotStateContext;
stateBeforeTurn?: MetabotStateContext;
activeToolCalls: MetabotToolCall[];
- lastTokenUsage?: MetabotContextUsage;
+ contextWindowTokens?: number;
profileOverride: MetabotProfileId | undefined;
experimental: {
developerMessage: string;
diff --git a/frontend/src/metabase/metabot/tests/reducer.unit.spec.ts b/frontend/src/metabase/metabot/tests/reducer.unit.spec.ts
index 306c4a46644..35141f57e19 100644
--- a/frontend/src/metabase/metabot/tests/reducer.unit.spec.ts
+++ b/frontend/src/metabase/metabot/tests/reducer.unit.spec.ts
@@ -759,14 +759,11 @@ describe("metabot reducer", () => {
]);
});
- it("releases the active chain id and context usage when a snapshot replaces the conversation", () => {
+ it("drops the streamed messages when a snapshot replaces the conversation", () => {
const store = createTestStore({
conversations: {
...createTestMetabotState().conversations,
- [conversationId]: createConversation({
- conversationId,
- lastTokenUsage: { contextTokens: 950, contextWindowTokens: 1000 },
- }),
+ [conversationId]: createConversation({ conversationId }),
},
});
startRequestlessAgentTurn(store, conversationId);
@@ -779,7 +776,6 @@ describe("metabot reducer", () => {
);
expect(getConvo(store)?.messages).toEqual([]);
- expect(getConvo(store)?.lastTokenUsage).toBeUndefined();
});
});
});
diff --git a/frontend/src/metabase/metabot/tests/retry.unit.spec.tsx b/frontend/src/metabase/metabot/tests/retry.unit.spec.tsx
index 1f9c658a480..cbf497442aa 100644
--- a/frontend/src/metabase/metabot/tests/retry.unit.spec.tsx
+++ b/frontend/src/metabase/metabot/tests/retry.unit.spec.tsx
@@ -23,6 +23,7 @@ import {
createPauses,
createTestMetabotState,
enterChatMessage,
+ expectContextUsage,
lastChatMessage,
lastReqBody,
mockAgentEndpoint,
@@ -38,12 +39,28 @@ const emptyContext = {
capabilities: [],
};
-const turnEvents = (opts: { messageId: string; text: string }): SSEEvent[] => [
- { type: "start", messageId: opts.messageId },
+const CONTEXT_WINDOW = 1000;
+
+const turnEvents = ({
+ messageId,
+ text,
+ contextTokens,
+}: {
+ messageId: string;
+ text: string;
+ contextTokens?: number;
+}): SSEEvent[] => [
+ { type: "start", messageId },
{ type: "text-start", id: "t1" },
- { type: "text-delta", id: "t1", delta: opts.text },
+ { type: "text-delta", id: "t1", delta: text },
{ type: "text-end", id: "t1" },
- { type: "finish", finishReason: "stop" },
+ {
+ type: "finish",
+ finishReason: "stop",
+ ...(contextTokens != null && {
+ messageMetadata: { contextTokens, contextWindowTokens: CONTEXT_WINDOW },
+ }),
+ },
];
describe("metabot > retry", () => {
@@ -281,6 +298,43 @@ describe("metabot > retry", () => {
expect(getConvoReqState()).toEqual({ todos: [{ id: "a" }] });
});
+ it("should rewind the context window usage to the retried turn", async () => {
+ setup();
+
+ mockAgentEndpoint({
+ events: turnEvents({
+ messageId: "msg_1",
+ text: "first reply",
+ contextTokens: 520,
+ }),
+ });
+ await enterChatMessage("first prompt");
+ expect(await screen.findByText("first reply")).toBeInTheDocument();
+ await expectContextUsage(52);
+
+ mockAgentEndpoint({
+ events: turnEvents({
+ messageId: "msg_2",
+ text: "second reply",
+ contextTokens: 640,
+ }),
+ });
+ await enterChatMessage("second prompt");
+ expect(await screen.findByText("second reply")).toBeInTheDocument();
+ await expectContextUsage(64);
+
+ // the regenerated turn reports no usage of its own, so 52% can only come
+ // from the rewind
+ mockAgentEndpoint({
+ events: turnEvents({ messageId: "msg_3", text: "regenerated reply" }),
+ });
+ await userEvent.click(
+ await screen.findByTestId("metabot-chat-message-retry"),
+ );
+ expect(await screen.findByText("regenerated reply")).toBeInTheDocument();
+ await expectContextUsage(52);
+ });
+
it("should stamp the user message with the id it sent", async () => {
const { store } = setup();
const spy = mockAgentEndpoint({
diff --git a/frontend/src/metabase/metabot/tests/ui.unit.spec.tsx b/frontend/src/metabase/metabot/tests/ui.unit.spec.tsx
index c42aa8e1599..789ab7b419c 100644
--- a/frontend/src/metabase/metabot/tests/ui.unit.spec.tsx
+++ b/frontend/src/metabase/metabot/tests/ui.unit.spec.tsx
@@ -35,6 +35,7 @@ import {
createPauses,
createTestMetabotState,
enterChatMessage,
+ expectContextUsage,
hideMetabot,
input,
lastReqBody,
@@ -376,10 +377,10 @@ describe("metabot > ui", () => {
expect(
within(notice).queryByTestId("metabot-long-chat-dismiss"),
).not.toBeInTheDocument();
- expect(screen.getByTestId("metabot-chat-input")).toBeInTheDocument();
+ expect(screen.queryByTestId("metabot-chat-input")).not.toBeInTheDocument();
expect(
- screen.getByTestId("metabot-context-usage-ring"),
- ).toBeInTheDocument();
+ screen.queryByTestId("metabot-context-usage-ring"),
+ ).not.toBeInTheDocument();
await userEvent.click(
within(notice).getByTestId("metabot-long-chat-new-chat"),
@@ -665,14 +666,14 @@ describe("metabot > ui", () => {
});
};
- const selectPastConversation = async () => {
+ const selectPastConversation = async (title = "Orders by month") => {
await userEvent.click(
await screen.findByTestId("metabot-conversation-history"),
);
const list = await screen.findByTestId(
"metabot-conversation-history-list",
);
- await userEvent.click(await within(list).findByText("Orders by month"));
+ await userEvent.click(await within(list).findByText(title));
};
it("loads a past conversation into the chat when a history item is clicked", async () => {
@@ -695,6 +696,57 @@ describe("metabot > ui", () => {
});
});
+ it("renders the context usage of a loaded conversation", async () => {
+ const CONTEXT_WINDOW = 1000;
+ const used = [
+ {
+ id: "22222222-2222-2222-2222-222222222222",
+ title: "Half full",
+ contextTokens: 520,
+ },
+ {
+ id: "33333333-3333-3333-3333-333333333333",
+ title: "Brimming",
+ contextTokens: CONTEXT_WINDOW,
+ },
+ ];
+
+ used.forEach(({ id, title, contextTokens }) =>
+ setupGetMetabotConversationEndpoint(
+ createMockMetabotConversationDetail({
+ conversation_id: id,
+ title,
+ context_window_tokens: CONTEXT_WINDOW,
+ messages: [
+ createMockMetabotTextMessage("user", "How many orders?"),
+ createMockMetabotTextMessage("agent", "There are 42 orders.", {
+ contextTokens,
+ }),
+ ],
+ }),
+ ),
+ );
+ setup({
+ conversations: used.map(({ id, title }) =>
+ createMockMetabotConversation({ conversation_id: id, title }),
+ ),
+ });
+
+ await selectPastConversation("Half full");
+ await expectContextUsage(52);
+ expect(
+ screen.queryByTestId("metabot-long-chat-notice"),
+ ).not.toBeInTheDocument();
+
+ await selectPastConversation("Brimming");
+ expect(
+ await screen.findByTestId("metabot-long-chat-notice"),
+ ).toHaveTextContent(/This chat has reached the/);
+ expect(
+ screen.queryByTestId("metabot-context-usage-ring"),
+ ).not.toBeInTheDocument();
+ });
+
it("positions a loaded conversation before the next frame", async () => {
jest.spyOn(window, "requestAnimationFrame").mockReturnValue(1);
jest
diff --git a/frontend/src/metabase/metabot/tests/utils.tsx b/frontend/src/metabase/metabot/tests/utils.tsx
index 7a9cf557516..ff93aee8c60 100644
--- a/frontend/src/metabase/metabot/tests/utils.tsx
+++ b/frontend/src/metabase/metabot/tests/utils.tsx
@@ -273,6 +273,14 @@ export const assertConversation = async (
});
};
+export const expectContextUsage = (percent: number) =>
+ waitFor(() =>
+ expect(screen.getByTestId("metabot-context-usage-ring")).toHaveAttribute(
+ "aria-label",
+ `${percent}% of the context window used`,
+ ),
+ );
+
export const lastReqBody = async (
agentSpy: ReturnType<typeof mockAgentEndpoint>,
) => {
diff --git a/resources/migrations/064/20260826_metabot_message_context_tokens.yaml b/resources/migrations/064/20260826_metabot_message_context_tokens.yaml
new file mode 100644
index 00000000000..c354c3e64ad
--- /dev/null
+++ b/resources/migrations/064/20260826_metabot_message_context_tokens.yaml
@@ -0,0 +1,21 @@
+# Add context_tokens to metabot_message to help calculate remaining space in the context window
+
+databaseChangeLog:
+ - changeSet:
+ id: v64.2026-08-26T00:00:00
+ author: sloansparger
+ comment: Add context_tokens to metabot_message
+ changes:
+ - addColumn:
+ tableName: metabot_message
+ columns:
+ - column:
+ name: context_tokens
+ remarks: Prompt + completion tokens of the turn's final LLM call
+ type: integer
+ constraints:
+ nullable: true
+ rollback:
+ - dropColumn:
+ tableName: metabot_message
+ columnName: context_tokens
diff --git a/src/metabase/metabot/api/conversations.clj b/src/metabase/metabot/api/conversations.clj
index 3eadf267efd..0e9167896b9 100644
--- a/src/metabase/metabot/api/conversations.clj
+++ b/src/metabase/metabot/api/conversations.clj
@@ -16,6 +16,8 @@
[metabase.metabot.db :as metabot.db]
[metabase.metabot.persistence :as metabot.persistence]
[metabase.metabot.schema :as metabot.schema]
+ [metabase.metabot.self :as metabot.self]
+ [metabase.metabot.settings :as metabot.settings]
[metabase.queries.core :as queries]
[metabase.query-permissions.core :as query-perms]
[metabase.request.core :as request]
@@ -60,7 +62,8 @@
[:map
[:card_id ms/PositiveInt]
[:chart_id [:maybe :string]]]]]
- [:messages [:sequential ::metabot.schema/client-message]]])
+ [:messages [:sequential ::metabot.schema/client-message]]
+ [:context_window_tokens {:optional true} :int]])
(def ^:private ConversationTitleResponse
[:map
@@ -154,13 +157,22 @@
(let [conversation (api/read-check :model/MetabotConversation id)]
(conversation-title/title-status id (:title conversation))))
+(defn- with-context-window
+ "Attach the window each message's `contextTokens` should be read against. It comes
+ from the model currently serving requests rather than the one that served the
+ turn, because the client uses it to judge whether the *next* message will fit."
+ [detail]
+ (let [window (metabot.self/context-window-tokens (metabot.settings/llm-metabot-provider))]
+ (cond-> detail
+ (and detail window) (assoc :context_window_tokens window))))
+
(api.macros/defendpoint :get "/:id" :- ConversationDetail
"Return a single conversation with its flattened chat messages.
Accessible to any participant in the conversation or to any superuser."
[{:keys [id]} :- ConversationIdParams]
(api/read-check :model/MetabotConversation id)
- (metabot.persistence/conversation-detail id))
+ (with-context-window (metabot.persistence/conversation-detail id)))
(api.macros/defendpoint :post "/:id/fork" :- ConversationDetail
"Fork a conversation at an assistant message, returning a brand-new conversation
@@ -179,7 +191,7 @@
(let [new-conversation-id (metabot.persistence/fork-conversation! id message_id api/*current-user-id*)]
(api/check-400 (some? new-conversation-id)
(tru "Can only fork from a completed Metabot response."))
- (metabot.persistence/conversation-detail new-conversation-id))))
+ (with-context-window (metabot.persistence/conversation-detail new-conversation-id)))))
(api.macros/defendpoint :post "/:id/saved-entity" :- SaveEntityResponse
"Save a Metabot-generated chart from this conversation as a card, stamping the
diff --git a/src/metabase/metabot/persistence.clj b/src/metabase/metabot/persistence.clj
index d391d1dec59..16ac9be14b0 100644
--- a/src/metabase/metabot/persistence.clj
+++ b/src/metabase/metabot/persistence.clj
@@ -128,6 +128,19 @@
{}
parts))
+(defn extract-context-tokens
+ "Prompt + completion tokens of the turn's final LLM call — the conversation's size
+ once the turn finished. Nil when the turn observed no usage."
+ [parts]
+ (let [usages (filterv #(= :usage (:type %)) parts)]
+ (when-let [{:keys [model usage]} (peek usages)]
+ (let [prev (->> (pop usages)
+ (filter #(= model (:model %)))
+ last
+ :usage)]
+ (+ (- (:promptTokens usage 0) (:promptTokens prev 0))
+ (- (:completionTokens usage 0) (:completionTokens prev 0)))))))
+
(defn throwable->error-payload
"Coerce a `Throwable` into the same JSON-encodable map shape a streamed
`:error` part carries, so a turn that fails by *throwing* persists in the
@@ -396,14 +409,15 @@
{:profile-id (or profile-id "unknown")}
(u/string-byte-count (json/encode content)))
(metabot.db/update-message! assistant-msg-id
- (cond-> {:data content
- :data_version schema.v2/current-data-version
- :usage usage
- :total_tokens (->> (vals usage)
- (map #(+ (:prompt %) (:completion %)))
- (reduce + 0))
- :finished (boolean finished?)
- :error (safe-encode-error error)}
+ (cond-> {:data content
+ :data_version schema.v2/current-data-version
+ :usage usage
+ :total_tokens (->> (vals usage)
+ (map #(+ (:prompt %) (:completion %)))
+ (reduce + 0))
+ :context_tokens (extract-context-tokens parts)
+ :finished (boolean finished?)
+ :error (safe-encode-error error)}
turn-state (assoc :state turn-state)
slack-msg-id (assoc :slack_msg_id slack-msg-id)
channel-id (assoc :channel_id channel-id)))
@@ -681,7 +695,8 @@
[]
(message->parts row))
:status status}
- (:external_id row) (assoc :externalId (:external_id row)))))
+ (:external_id row) (assoc :externalId (:external_id row))
+ (:context_tokens row) (assoc :contextTokens (:context_tokens row)))))
(defn messages->client-messages
"Convert a seq of `MetabotMessage` model instances into their client shape, one
@@ -753,7 +768,8 @@
double-counts tokens in the analytics views. `forked_from_message_id` records
the source row so the copied prefix can be told apart from messages added after
the fork."
- [new-conversation-id user-id {:keys [id data data_version role profile_id ai_proxied finished error state]}]
+ [new-conversation-id user-id {:keys [id data data_version role profile_id ai_proxied finished error state
+ context_tokens]}]
(cond-> {:conversation_id new-conversation-id
:data data
:data_version data_version
@@ -762,6 +778,7 @@
:external_id (str (random-uuid))
:total_tokens 0
:usage nil
+ :context_tokens context_tokens
:ai_proxied (boolean ai_proxied)
:user_id user-id
:forked_from_message_id id}
diff --git a/src/metabase/metabot/schema.clj b/src/metabase/metabot/schema.clj
index f1d6ad92768..455283103eb 100644
--- a/src/metabase/metabot/schema.clj
+++ b/src/metabase/metabot/schema.clj
@@ -97,8 +97,11 @@
(mr/def ::client-message
"One persisted message as the client models it."
[:map
- [:id :string]
- [:externalId {:optional true} :string]
- [:role [:enum "user" "agent"]]
- [:parts [:sequential ::client-message-part]]
- [:status ::client-message-status]])
+ [:id :string]
+ [:externalId {:optional true} :string]
+ [:role [:enum "user" "agent"]]
+ [:parts [:sequential ::client-message-part]]
+ [:status ::client-message-status]
+ ;; how much of the window the conversation occupied once this message was
+ ;; produced; absent on all user rows and legacy agent records
+ [:contextTokens {:optional true} :int]])
diff --git a/test/metabase/metabot/api/conversations_test.clj b/test/metabase/metabot/api/conversations_test.clj
index 002d2be8d36..ead0d35697c 100644
--- a/test/metabase/metabot/api/conversations_test.clj
+++ b/test/metabase/metabot/api/conversations_test.clj
@@ -468,6 +468,7 @@
:model/MetabotMessage _a1 {:conversation_id convo-id :user_id user-id :role "assistant"
:external_id a1 :finished true :total_tokens 100
:usage {"gpt" {:prompt 60 :completion 40}}
+ :context_tokens 60
:data [{:type "text" :text "hello"}]
:created_at (seconds-ago 50)}
:model/MetabotMessage _u2 {:conversation_id convo-id :user_id user-id :role "user"
@@ -504,7 +505,10 @@
(is (every? #(t/after? (t/instant (:created_at %)) (t/instant orig-a1-at)) rows))))
(testing "token usage is zeroed so forks don't double-count in analytics"
(is (every? #(zero? (:total_tokens %)) rows))
- (is (every? #(nil? (:usage %)) rows))))
+ (is (every? #(nil? (:usage %)) rows)))
+ (testing "context size carries over so the fork is as full as the thread it copied"
+ (is (= [nil 60] (mapv :context_tokens rows)))
+ (is (= [nil 60] (mapv :contextTokens (:messages response))))))
(testing "the original conversation is untouched"
(is (= 4 (count (metabot.persistence/live-messages convo-id)))))
(finally
diff --git a/test/metabase/metabot/persistence_test.clj b/test/metabase/metabot/persistence_test.clj
index 82f9a6266f8..079702db5ff 100644
--- a/test/metabase/metabot/persistence_test.clj
+++ b/test/metabase/metabot/persistence_test.clj
@@ -478,6 +478,41 @@
(is (= [{:type "text" :text "Hello" :state "done"}] (:data row)))
(is (= 15 (:total_tokens row)))))))))
+(deftest finalize-assistant-turn-stores-context-tokens-test
+ (testing "finalize-assistant-turn! stores the final call's context size"
+ (with-rasta-tx
+ (let [conversation-id (str (random-uuid))
+ {:keys [assistant-msg-id]} (metabot-persistence/start-turn!
+ conversation-id "internal"
+ {:role "user" :content "hi"})]
+ (metabot-persistence/finalize-assistant-turn!
+ assistant-msg-id
+ ;; usage snapshots are cumulative, so the final call alone is 150 prompt + 10 completion
+ [{:type :usage :model "claude" :usage {:promptTokens 100 :completionTokens 20}}
+ {:type :text :text "Hello"}
+ {:type :usage :model "claude" :usage {:promptTokens 250 :completionTokens 30}}])
+ (is (=? {:context_tokens 160 :total_tokens 280}
+ (t2/select-one :model/MetabotMessage assistant-msg-id))))))
+ (testing "a turn that observed no usage stores no context size"
+ (with-rasta-tx
+ (let [conversation-id (str (random-uuid))
+ {:keys [assistant-msg-id]} (metabot-persistence/start-turn!
+ conversation-id "internal"
+ {:role "user" :content "hi"})]
+ (metabot-persistence/finalize-assistant-turn!
+ assistant-msg-id [{:type :text :text "Hello"}])
+ (is (nil? (:context_tokens (t2/select-one :model/MetabotMessage assistant-msg-id))))))))
+
+(deftest ^:parallel messages->client-messages-context-tokens-test
+ (testing "a message reports the context it occupied, and omits it when the turn observed no usage"
+ (is (= 520 (:contextTokens (client-message {:role :assistant :data []
+ :context_tokens 520}))))
+ (testing "no usage observed"
+ (is (not (contains? (client-message {:role :assistant :data []}) :contextTokens))))
+ (testing "user rows never carry it"
+ (is (not (contains? (client-message {:role :user :data [{:type "text" :text "hi"}]})
+ :contextTokens))))))
+
(deftest finalize-assistant-turn-passes-through-aborted-and-errored-test
(testing "finalize-assistant-turn! preserves :finished? false and JSON-encodes :error"
(with-rasta-tx