The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[Metabot] Context window improvements (#81133)
Sloan Spargerd273dc56b6a9
diff --git a/enterprise/frontend/src/embedding-sdk-ee/metabot/MetabotChatHistory.unit.spec.tsx b/enterprise/frontend/src/embedding-sdk-ee/metabot/MetabotChatHistory.unit.spec.tsxindex 22ef4bce815..3693bcaa3c1 100644--- a/enterprise/frontend/src/embedding-sdk-ee/metabot/MetabotChatHistory.unit.spec.tsx+++ b/enterprise/frontend/src/embedding-sdk-ee/metabot/MetabotChatHistory.unit.spec.tsx@@ -28,11 +28,22 @@ describe("MetabotChatHistory", () => {const { store } = setup({ui: <MetabotChatHistory />,metabotInitialState: assocIn(- makeVisibleState([- { id: "1", role: "user", type: "text", message: "hi" },- ]),- ["conversations", testConversationId("omnibot"), "lastTokenUsage"],- { contextTokens: 200, contextWindowTokens: 200 },+ assocIn(+ makeVisibleState([+ { id: "1", role: "user", type: "text", message: "hi" },+ { id: "2", role: "agent", type: "text", message: "hello" },+ ]),+ [+ "conversations",+ testConversationId("omnibot"),+ "messages",+ 1,+ "contextTokens",+ ],+ 200,+ ),+ ["conversations", testConversationId("omnibot"), "contextWindowTokens"],+ 200,),});diff --git a/frontend/src/metabase/metabot/api.ts b/frontend/src/metabase/metabot/api.tsindex b8b9a581c4e..c672144878c 100644--- a/frontend/src/metabase/metabot/api.ts+++ b/frontend/src/metabase/metabot/api.ts@@ -31,6 +31,7 @@ export type MetabotConversationDetail = {forked_from_conversation_id: string | null;state?: MetabotStateContext;messages: MetabotMessage[];+ context_window_tokens?: number;};export const metabotApi = Api.injectEndpoints({diff --git a/frontend/src/metabase/metabot/components/MetabotChat/MetabotChat.tsx b/frontend/src/metabase/metabot/components/MetabotChat/MetabotChat.tsxindex 9c3911ce9c3..c70b0f66d30 100644--- a/frontend/src/metabase/metabot/components/MetabotChat/MetabotChat.tsx+++ b/frontend/src/metabase/metabot/components/MetabotChat/MetabotChat.tsx@@ -208,37 +208,40 @@ export const MetabotChat = ({onNewChat={onNewConversation}/>)}- <Paper- className={cx(- Styles.inputContainer,- metabot.isDoingScience && Styles.inputContainerLoading,- )}- >- <MetabotChatEditor- ref={metabot.promptInputRef}- value={metabot.prompt}- autoFocus- isResponding={metabot.isDoingScience}- placeholder={t`How can I help? Type @ to mention items.`}- onChange={metabot.setPrompt}- onSubmit={() => metabot.submitInput(metabot.prompt)}- onStop={metabot.cancelRequest}- suggestionConfig={{- suggestionModels: config.suggestionModels,- }}- />- </Paper>+ {!metabot.isContextWindowFull && (+ <Paper+ className={cx(+ Styles.inputContainer,+ metabot.isDoingScience && Styles.inputContainerLoading,+ )}+ >+ <MetabotChatEditor+ ref={metabot.promptInputRef}+ value={metabot.prompt}+ autoFocus+ isResponding={metabot.isDoingScience}+ placeholder={t`How can I help? Type @ to mention items.`}+ onChange={metabot.setPrompt}+ onSubmit={() => metabot.submitInput(metabot.prompt)}+ onStop={metabot.cancelRequest}+ suggestionConfig={{+ suggestionModels: config.suggestionModels,+ }}+ />+ </Paper>+ )}</Box><Box className={Styles.footerRow}><Text fz="sm" c="text-secondary" ta="center">{t`${metabotName} isn't perfect. Double-check results.`}</Text>- {metabot.contextWindowPercentUsage > 50 && (- <MetabotContextUsageRing- className={Styles.contextUsage}- percentUsage={metabot.contextWindowPercentUsage}- />- )}+ {metabot.contextWindowPercentUsage > 50 &&+ !metabot.isContextWindowFull && (+ <MetabotContextUsageRing+ className={Styles.contextUsage}+ percentUsage={metabot.contextWindowPercentUsage}+ />+ )}</Box></Box>)}diff --git a/frontend/src/metabase/metabot/components/MetabotConversationPage/MetabotConversationPage.tsx b/frontend/src/metabase/metabot/components/MetabotConversationPage/MetabotConversationPage.tsxindex aed9c1c89ee..b37761c7839 100644--- a/frontend/src/metabase/metabot/components/MetabotConversationPage/MetabotConversationPage.tsx+++ b/frontend/src/metabase/metabot/components/MetabotConversationPage/MetabotConversationPage.tsx@@ -60,6 +60,7 @@ export const MetabotConversationPage = () => {title: conversation.title ?? undefined,forkedFromConversationId:conversation.forked_from_conversation_id ?? undefined,+ contextWindowTokens: conversation.context_window_tokens,messages: conversation.messages,state: conversation.state,activeToolCalls: [],diff --git a/frontend/src/metabase/metabot/state/actions.ts b/frontend/src/metabase/metabot/state/actions.tsindex 12d8d4c8a40..1c798ce9b0c 100644--- a/frontend/src/metabase/metabot/state/actions.ts+++ b/frontend/src/metabase/metabot/state/actions.ts@@ -950,6 +950,7 @@ export const fetchConversationSnapshot = createAsyncThunk(title: detail.title ?? undefined,forkedFromConversationId:detail.forked_from_conversation_id ?? undefined,+ contextWindowTokens: detail.context_window_tokens,messages: detail.messages,state: detail.state,activeToolCalls: [],@@ -1008,6 +1009,7 @@ export const forkConversation = createAsyncThunk(title: conversation.title ?? undefined,forkedFromConversationId:conversation.forked_from_conversation_id ?? undefined,+ contextWindowTokens: conversation.context_window_tokens,messages: conversation.messages,state: conversation.state,activeToolCalls: [],diff --git a/frontend/src/metabase/metabot/state/reducer.ts b/frontend/src/metabase/metabot/state/reducer.tsindex 7e219bfde87..10b9ba2a1ed 100644--- a/frontend/src/metabase/metabot/state/reducer.ts+++ b/frontend/src/metabase/metabot/state/reducer.ts@@ -441,6 +441,7 @@ export const metabot = createSlice({conversationId: string;title?: string;forkedFromConversationId?: string;+ contextWindowTokens?: number;}>,) => {const {@@ -450,6 +451,7 @@ export const metabot = createSlice({conversationId,title,forkedFromConversationId,+ contextWindowTokens,} = action.payload;const convo =@@ -463,7 +465,7 @@ export const metabot = createSlice({convo.activeToolCalls = activeToolCalls ?? [];convo.title = title;convo.forkedFromConversationId = forkedFromConversationId;- convo.lastTokenUsage = undefined;+ convo.contextWindowTokens = contextWindowTokens;convo.isProcessing = hasInProgressMessage(messages);if (convo.isProcessing) {ensureChain(convo); // resuming mid-response@@ -502,25 +504,31 @@ export const metabot = createSlice({convo.state = { ...action.payload.state };}+ closeChain(convo);+ const message = openAgentMessage(convo);+const metadata = action.payload?.processedResponse.messageMetadata;- if (metadata?.contextTokens && metadata.contextWindowTokens) {- convo.lastTokenUsage = {- contextTokens: metadata.contextTokens,- contextWindowTokens: metadata.contextWindowTokens,- };- }+ const contextUsage =+ metadata?.contextTokens && metadata.contextWindowTokens+ ? {+ contextTokens: metadata.contextTokens,+ contextWindowTokens: metadata.contextWindowTokens,+ }+ : undefined;+ message.contextTokens = contextUsage?.contextTokens;+ convo.contextWindowTokens =+ contextUsage?.contextWindowTokens ?? convo.contextWindowTokens;const finishReason = action.payload?.processedResponse.finishReason;const isResumableFinishReason =finishReason && finishReason !== "stop" && finishReason !== "error";- closeChain(convo);- openAgentMessage(convo).status = isResumableFinishReason+ message.status = isResumableFinishReason? {type: "incomplete",finishReason,contextWindowFull:finishReason === "length" &&- isContextWindowFull(convo.lastTokenUsage),+ isContextWindowFull(contextUsage),}: { type: "done" };diff --git a/frontend/src/metabase/metabot/state/selectors.ts b/frontend/src/metabase/metabot/state/selectors.tsindex 75517505145..3943f374e96 100644--- a/frontend/src/metabase/metabot/state/selectors.ts+++ b/frontend/src/metabase/metabot/state/selectors.ts@@ -17,7 +17,11 @@ import {isContextWindowFull,} from "../utils/context-usage";-import type { MetabotAgentId, MetabotMessage } from "./types";+import type {+ MetabotAgentId,+ MetabotContextUsage,+ MetabotMessage,+} from "./types";import { hasInProgressMessage, isGeneratedCardPart, isTextPart } from "./utils";/*@@ -253,15 +257,28 @@ export const getConversationChart = createSelector(export type MetabotLongChatNoticeVariant = "warning" | "full";+export const getContextUsage = createSelector(+ [getMessages, getConversation],+ (messages, convo): MetabotContextUsage | undefined => {+ const contextTokens = messages.findLast(+ (m) => m.role === "agent" && m.contextTokens,+ )?.contextTokens;+ const { contextWindowTokens } = convo;+ return contextTokens && contextWindowTokens+ ? { contextTokens, contextWindowTokens }+ : undefined;+ },+);+export const getContextUsagePercent = createSelector(- [getConversation],- (convo): number => getContextWindowPercentUsage(convo.lastTokenUsage),+ getContextUsage,+ getContextWindowPercentUsage,);export const getLongChatNotice = createSelector(- [getConversation, getContextUsagePercent],- (convo, percentUsage): MetabotLongChatNoticeVariant | undefined => {- if (isContextWindowFull(convo.lastTokenUsage)) {+ [getContextUsage, getContextUsagePercent],+ (contextUsage, percentUsage): MetabotLongChatNoticeVariant | undefined => {+ if (isContextWindowFull(contextUsage)) {return "full";}return percentUsage >= CONTEXT_WINDOW_WARNING_PERCENTdiff --git a/frontend/src/metabase/metabot/state/selectors.unit.spec.ts b/frontend/src/metabase/metabot/state/selectors.unit.spec.tsindex 70a97189cad..253603ae722 100644--- a/frontend/src/metabase/metabot/state/selectors.unit.spec.ts+++ b/frontend/src/metabase/metabot/state/selectors.unit.spec.ts@@ -19,13 +19,12 @@ import {getUserPromptMessage,} from "./index";+const CONTEXT_WINDOW = 200000;+function setup(messages: MetabotMessage[],convoState?: Partial<- Pick<- MetabotConversationState,- "conversationId" | "lastTokenUsage" | "profileOverride"- >+ Pick<MetabotConversationState, "conversationId" | "profileOverride">>,): State {setupEnterprisePlugins();@@ -38,10 +37,15 @@ function setup(["conversations", conversationId, "messages"],messages,);+ const withWindow = assocIn(+ withMessages,+ ["conversations", conversationId, "contextWindowTokens"],+ CONTEXT_WINDOW,+ );const withConvoState = Object.entries(convoState ?? {}).reduce((acc, [key, value]) =>assocIn(acc, ["conversations", conversationId, key], value),- withMessages,+ withWindow,);return createMockState({ metabot: withConvoState });@@ -152,32 +156,30 @@ describe("metabot selectors", () => {role: "user",parts: [{ id: "1", role: "user", type: "text", message: "hi" }],});- const CONTEXT_WINDOW = 200000;- const usage = (contextTokens: number) => ({- lastTokenUsage: { contextTokens, contextWindowTokens: CONTEXT_WINDOW },- });+ const usage = (contextTokens: number) =>+ createMockMetabotMessage({ contextTokens });it("warns when context tokens reach the warning percent of the window", () => {- const state = setup(- [shortMessage],+ const state = setup([+ shortMessage,usage((CONTEXT_WINDOW * CONTEXT_WINDOW_WARNING_PERCENT) / 100),- );+ ]);expect(getLongChatNotice(state, "omnibot")).toBe("warning");});it("still only warns just short of the window — the whole window is usable", () => {- const state = setup([shortMessage], usage(CONTEXT_WINDOW - 1));+ const state = setup([shortMessage, usage(CONTEXT_WINDOW - 1)]);expect(getLongChatNotice(state, "omnibot")).toBe("warning");});it("reports full once context tokens consume the whole window", () => {- const state = setup([shortMessage], usage(CONTEXT_WINDOW));+ const state = setup([shortMessage, usage(CONTEXT_WINDOW)]);expect(getLongChatNotice(state, "omnibot")).toBe("full");});it("shows nothing below the warning percent or when no usage has been observed", () => {expect(- getLongChatNotice(setup([shortMessage], usage(10000)), "omnibot"),+ getLongChatNotice(setup([shortMessage, usage(10000)]), "omnibot"),).toBeUndefined();expect(getLongChatNotice(setup([shortMessage]), "omnibot"),@@ -186,7 +188,7 @@ describe("metabot selectors", () => {describe("getContextUsagePercent", () => {it("reports the last observed usage as a share of the window, 0-100", () => {- const state = setup([shortMessage], usage(CONTEXT_WINDOW / 4));+ const state = setup([shortMessage, usage(CONTEXT_WINDOW / 4)]);expect(getContextUsagePercent(state, "omnibot")).toBe(25);});@@ -197,7 +199,7 @@ describe("metabot selectors", () => {});it("caps at 100 when the window is overrun", () => {- const state = setup([shortMessage], usage(CONTEXT_WINDOW * 2));+ const state = setup([shortMessage, usage(CONTEXT_WINDOW * 2)]);expect(getContextUsagePercent(state, "omnibot")).toBe(100);});});diff --git a/frontend/src/metabase/metabot/state/types.ts b/frontend/src/metabase/metabot/state/types.tsindex e12d8b53502..6d5193ee99b 100644--- a/frontend/src/metabase/metabot/state/types.ts+++ b/frontend/src/metabase/metabot/state/types.ts@@ -120,12 +120,18 @@ export type MetabotMessagePart =| MetabotDebugToolCallMessage| MetabotAgentChainOfThoughtMessage;+export type MetabotContextUsage = {+ contextTokens: number;+ contextWindowTokens: number;+};+export type MetabotMessage = {id: string;externalId?: string;role: "user" | "agent";parts: MetabotMessagePart[];status: MetabotMessageStatus;+ contextTokens?: number;};export type MetabotToolCall = {@@ -155,11 +161,6 @@ export type MetabotReactionsState = {suggestedTransforms: MetabotSuggestedTransform[];};-export type MetabotContextUsage = {- contextTokens: number;- contextWindowTokens: number;-};-export interface MetabotConversationState {conversationId: string;title: string | undefined;@@ -170,7 +171,7 @@ export interface MetabotConversationState {state: MetabotStateContext;stateBeforeTurn?: MetabotStateContext;activeToolCalls: MetabotToolCall[];- lastTokenUsage?: MetabotContextUsage;+ contextWindowTokens?: number;profileOverride: MetabotProfileId | undefined;experimental: {developerMessage: string;diff --git a/frontend/src/metabase/metabot/tests/reducer.unit.spec.ts b/frontend/src/metabase/metabot/tests/reducer.unit.spec.tsindex 306c4a46644..35141f57e19 100644--- a/frontend/src/metabase/metabot/tests/reducer.unit.spec.ts+++ b/frontend/src/metabase/metabot/tests/reducer.unit.spec.ts@@ -759,14 +759,11 @@ describe("metabot reducer", () => {]);});- it("releases the active chain id and context usage when a snapshot replaces the conversation", () => {+ it("drops the streamed messages when a snapshot replaces the conversation", () => {const store = createTestStore({conversations: {...createTestMetabotState().conversations,- [conversationId]: createConversation({- conversationId,- lastTokenUsage: { contextTokens: 950, contextWindowTokens: 1000 },- }),+ [conversationId]: createConversation({ conversationId }),},});startRequestlessAgentTurn(store, conversationId);@@ -779,7 +776,6 @@ describe("metabot reducer", () => {);expect(getConvo(store)?.messages).toEqual([]);- expect(getConvo(store)?.lastTokenUsage).toBeUndefined();});});});diff --git a/frontend/src/metabase/metabot/tests/retry.unit.spec.tsx b/frontend/src/metabase/metabot/tests/retry.unit.spec.tsxindex 1f9c658a480..cbf497442aa 100644--- a/frontend/src/metabase/metabot/tests/retry.unit.spec.tsx+++ b/frontend/src/metabase/metabot/tests/retry.unit.spec.tsx@@ -23,6 +23,7 @@ import {createPauses,createTestMetabotState,enterChatMessage,+ expectContextUsage,lastChatMessage,lastReqBody,mockAgentEndpoint,@@ -38,12 +39,28 @@ const emptyContext = {capabilities: [],};-const turnEvents = (opts: { messageId: string; text: string }): SSEEvent[] => [- { type: "start", messageId: opts.messageId },+const CONTEXT_WINDOW = 1000;++const turnEvents = ({+ messageId,+ text,+ contextTokens,+}: {+ messageId: string;+ text: string;+ contextTokens?: number;+}): SSEEvent[] => [+ { type: "start", messageId },{ type: "text-start", id: "t1" },- { type: "text-delta", id: "t1", delta: opts.text },+ { type: "text-delta", id: "t1", delta: text },{ type: "text-end", id: "t1" },- { type: "finish", finishReason: "stop" },+ {+ type: "finish",+ finishReason: "stop",+ ...(contextTokens != null && {+ messageMetadata: { contextTokens, contextWindowTokens: CONTEXT_WINDOW },+ }),+ },];describe("metabot > retry", () => {@@ -281,6 +298,43 @@ describe("metabot > retry", () => {expect(getConvoReqState()).toEqual({ todos: [{ id: "a" }] });});+ it("should rewind the context window usage to the retried turn", async () => {+ setup();++ mockAgentEndpoint({+ events: turnEvents({+ messageId: "msg_1",+ text: "first reply",+ contextTokens: 520,+ }),+ });+ await enterChatMessage("first prompt");+ expect(await screen.findByText("first reply")).toBeInTheDocument();+ await expectContextUsage(52);++ mockAgentEndpoint({+ events: turnEvents({+ messageId: "msg_2",+ text: "second reply",+ contextTokens: 640,+ }),+ });+ await enterChatMessage("second prompt");+ expect(await screen.findByText("second reply")).toBeInTheDocument();+ await expectContextUsage(64);++ // the regenerated turn reports no usage of its own, so 52% can only come+ // from the rewind+ mockAgentEndpoint({+ events: turnEvents({ messageId: "msg_3", text: "regenerated reply" }),+ });+ await userEvent.click(+ await screen.findByTestId("metabot-chat-message-retry"),+ );+ expect(await screen.findByText("regenerated reply")).toBeInTheDocument();+ await expectContextUsage(52);+ });+it("should stamp the user message with the id it sent", async () => {const { store } = setup();const spy = mockAgentEndpoint({diff --git a/frontend/src/metabase/metabot/tests/ui.unit.spec.tsx b/frontend/src/metabase/metabot/tests/ui.unit.spec.tsxindex c42aa8e1599..789ab7b419c 100644--- a/frontend/src/metabase/metabot/tests/ui.unit.spec.tsx+++ b/frontend/src/metabase/metabot/tests/ui.unit.spec.tsx@@ -35,6 +35,7 @@ import {createPauses,createTestMetabotState,enterChatMessage,+ expectContextUsage,hideMetabot,input,lastReqBody,@@ -376,10 +377,10 @@ describe("metabot > ui", () => {expect(within(notice).queryByTestId("metabot-long-chat-dismiss"),).not.toBeInTheDocument();- expect(screen.getByTestId("metabot-chat-input")).toBeInTheDocument();+ expect(screen.queryByTestId("metabot-chat-input")).not.toBeInTheDocument();expect(- screen.getByTestId("metabot-context-usage-ring"),- ).toBeInTheDocument();+ screen.queryByTestId("metabot-context-usage-ring"),+ ).not.toBeInTheDocument();await userEvent.click(within(notice).getByTestId("metabot-long-chat-new-chat"),@@ -665,14 +666,14 @@ describe("metabot > ui", () => {});};- const selectPastConversation = async () => {+ const selectPastConversation = async (title = "Orders by month") => {await userEvent.click(await screen.findByTestId("metabot-conversation-history"),);const list = await screen.findByTestId("metabot-conversation-history-list",);- await userEvent.click(await within(list).findByText("Orders by month"));+ await userEvent.click(await within(list).findByText(title));};it("loads a past conversation into the chat when a history item is clicked", async () => {@@ -695,6 +696,57 @@ describe("metabot > ui", () => {});});+ it("renders the context usage of a loaded conversation", async () => {+ const CONTEXT_WINDOW = 1000;+ const used = [+ {+ id: "22222222-2222-2222-2222-222222222222",+ title: "Half full",+ contextTokens: 520,+ },+ {+ id: "33333333-3333-3333-3333-333333333333",+ title: "Brimming",+ contextTokens: CONTEXT_WINDOW,+ },+ ];++ used.forEach(({ id, title, contextTokens }) =>+ setupGetMetabotConversationEndpoint(+ createMockMetabotConversationDetail({+ conversation_id: id,+ title,+ context_window_tokens: CONTEXT_WINDOW,+ messages: [+ createMockMetabotTextMessage("user", "How many orders?"),+ createMockMetabotTextMessage("agent", "There are 42 orders.", {+ contextTokens,+ }),+ ],+ }),+ ),+ );+ setup({+ conversations: used.map(({ id, title }) =>+ createMockMetabotConversation({ conversation_id: id, title }),+ ),+ });++ await selectPastConversation("Half full");+ await expectContextUsage(52);+ expect(+ screen.queryByTestId("metabot-long-chat-notice"),+ ).not.toBeInTheDocument();++ await selectPastConversation("Brimming");+ expect(+ await screen.findByTestId("metabot-long-chat-notice"),+ ).toHaveTextContent(/This chat has reached the/);+ expect(+ screen.queryByTestId("metabot-context-usage-ring"),+ ).not.toBeInTheDocument();+ });+it("positions a loaded conversation before the next frame", async () => {jest.spyOn(window, "requestAnimationFrame").mockReturnValue(1);jestdiff --git a/frontend/src/metabase/metabot/tests/utils.tsx b/frontend/src/metabase/metabot/tests/utils.tsxindex 7a9cf557516..ff93aee8c60 100644--- a/frontend/src/metabase/metabot/tests/utils.tsx+++ b/frontend/src/metabase/metabot/tests/utils.tsx@@ -273,6 +273,14 @@ export const assertConversation = async (});};+export const expectContextUsage = (percent: number) =>+ waitFor(() =>+ expect(screen.getByTestId("metabot-context-usage-ring")).toHaveAttribute(+ "aria-label",+ `${percent}% of the context window used`,+ ),+ );+export const lastReqBody = async (agentSpy: ReturnType<typeof mockAgentEndpoint>,) => {diff --git a/resources/migrations/064/20260826_metabot_message_context_tokens.yaml b/resources/migrations/064/20260826_metabot_message_context_tokens.yamlnew file mode 100644index 00000000000..c354c3e64ad--- /dev/null+++ b/resources/migrations/064/20260826_metabot_message_context_tokens.yaml@@ -0,0 +1,21 @@+# Add context_tokens to metabot_message to help calculate remaining space in the context window++databaseChangeLog:+ - changeSet:+ id: v64.2026-08-26T00:00:00+ author: sloansparger+ comment: Add context_tokens to metabot_message+ changes:+ - addColumn:+ tableName: metabot_message+ columns:+ - column:+ name: context_tokens+ remarks: Prompt + completion tokens of the turn's final LLM call+ type: integer+ constraints:+ nullable: true+ rollback:+ - dropColumn:+ tableName: metabot_message+ columnName: context_tokensdiff --git a/src/metabase/metabot/api/conversations.clj b/src/metabase/metabot/api/conversations.cljindex 3eadf267efd..0e9167896b9 100644--- a/src/metabase/metabot/api/conversations.clj+++ b/src/metabase/metabot/api/conversations.clj@@ -16,6 +16,8 @@[metabase.metabot.db :as metabot.db][metabase.metabot.persistence :as metabot.persistence][metabase.metabot.schema :as metabot.schema]+ [metabase.metabot.self :as metabot.self]+ [metabase.metabot.settings :as metabot.settings][metabase.queries.core :as queries][metabase.query-permissions.core :as query-perms][metabase.request.core :as request]@@ -60,7 +62,8 @@[:map[:card_id ms/PositiveInt][:chart_id [:maybe :string]]]]]- [:messages [:sequential ::metabot.schema/client-message]]])+ [:messages [:sequential ::metabot.schema/client-message]]+ [:context_window_tokens {:optional true} :int]])(def ^:private ConversationTitleResponse[:map@@ -154,13 +157,22 @@(let [conversation (api/read-check :model/MetabotConversation id)](conversation-title/title-status id (:title conversation))))+(defn- with-context-window+ "Attach the window each message's `contextTokens` should be read against. It comes+ from the model currently serving requests rather than the one that served the+ turn, because the client uses it to judge whether the *next* message will fit."+ [detail]+ (let [window (metabot.self/context-window-tokens (metabot.settings/llm-metabot-provider))]+ (cond-> detail+ (and detail window) (assoc :context_window_tokens window))))+(api.macros/defendpoint :get "/:id" :- ConversationDetail"Return a single conversation with its flattened chat messages.Accessible to any participant in the conversation or to any superuser."[{:keys [id]} :- ConversationIdParams](api/read-check :model/MetabotConversation id)- (metabot.persistence/conversation-detail id))+ (with-context-window (metabot.persistence/conversation-detail id)))(api.macros/defendpoint :post "/:id/fork" :- ConversationDetail"Fork a conversation at an assistant message, returning a brand-new conversation@@ -179,7 +191,7 @@(let [new-conversation-id (metabot.persistence/fork-conversation! id message_id api/*current-user-id*)](api/check-400 (some? new-conversation-id)(tru "Can only fork from a completed Metabot response."))- (metabot.persistence/conversation-detail new-conversation-id))))+ (with-context-window (metabot.persistence/conversation-detail new-conversation-id)))))(api.macros/defendpoint :post "/:id/saved-entity" :- SaveEntityResponse"Save a Metabot-generated chart from this conversation as a card, stamping thediff --git a/src/metabase/metabot/persistence.clj b/src/metabase/metabot/persistence.cljindex d391d1dec59..16ac9be14b0 100644--- a/src/metabase/metabot/persistence.clj+++ b/src/metabase/metabot/persistence.clj@@ -128,6 +128,19 @@{}parts))+(defn extract-context-tokens+ "Prompt + completion tokens of the turn's final LLM call — the conversation's size+ once the turn finished. Nil when the turn observed no usage."+ [parts]+ (let [usages (filterv #(= :usage (:type %)) parts)]+ (when-let [{:keys [model usage]} (peek usages)]+ (let [prev (->> (pop usages)+ (filter #(= model (:model %)))+ last+ :usage)]+ (+ (- (:promptTokens usage 0) (:promptTokens prev 0))+ (- (:completionTokens usage 0) (:completionTokens prev 0)))))))+(defn throwable->error-payload"Coerce a `Throwable` into the same JSON-encodable map shape a streamed`:error` part carries, so a turn that fails by *throwing* persists in the@@ -396,14 +409,15 @@{:profile-id (or profile-id "unknown")}(u/string-byte-count (json/encode content)))(metabot.db/update-message! assistant-msg-id- (cond-> {:data content- :data_version schema.v2/current-data-version- :usage usage- :total_tokens (->> (vals usage)- (map #(+ (:prompt %) (:completion %)))- (reduce + 0))- :finished (boolean finished?)- :error (safe-encode-error error)}+ (cond-> {:data content+ :data_version schema.v2/current-data-version+ :usage usage+ :total_tokens (->> (vals usage)+ (map #(+ (:prompt %) (:completion %)))+ (reduce + 0))+ :context_tokens (extract-context-tokens parts)+ :finished (boolean finished?)+ :error (safe-encode-error error)}turn-state (assoc :state turn-state)slack-msg-id (assoc :slack_msg_id slack-msg-id)channel-id (assoc :channel_id channel-id)))@@ -681,7 +695,8 @@[](message->parts row)):status status}- (:external_id row) (assoc :externalId (:external_id row)))))+ (:external_id row) (assoc :externalId (:external_id row))+ (:context_tokens row) (assoc :contextTokens (:context_tokens row)))))(defn messages->client-messages"Convert a seq of `MetabotMessage` model instances into their client shape, one@@ -753,7 +768,8 @@double-counts tokens in the analytics views. `forked_from_message_id` recordsthe source row so the copied prefix can be told apart from messages added afterthe fork."- [new-conversation-id user-id {:keys [id data data_version role profile_id ai_proxied finished error state]}]+ [new-conversation-id user-id {:keys [id data data_version role profile_id ai_proxied finished error state+ context_tokens]}](cond-> {:conversation_id new-conversation-id:data data:data_version data_version@@ -762,6 +778,7 @@:external_id (str (random-uuid)):total_tokens 0:usage nil+ :context_tokens context_tokens:ai_proxied (boolean ai_proxied):user_id user-id:forked_from_message_id id}diff --git a/src/metabase/metabot/schema.clj b/src/metabase/metabot/schema.cljindex f1d6ad92768..455283103eb 100644--- a/src/metabase/metabot/schema.clj+++ b/src/metabase/metabot/schema.clj@@ -97,8 +97,11 @@(mr/def ::client-message"One persisted message as the client models it."[:map- [:id :string]- [:externalId {:optional true} :string]- [:role [:enum "user" "agent"]]- [:parts [:sequential ::client-message-part]]- [:status ::client-message-status]])+ [:id :string]+ [:externalId {:optional true} :string]+ [:role [:enum "user" "agent"]]+ [:parts [:sequential ::client-message-part]]+ [:status ::client-message-status]+ ;; how much of the window the conversation occupied once this message was+ ;; produced; absent on all user rows and legacy agent records+ [:contextTokens {:optional true} :int]])diff --git a/test/metabase/metabot/api/conversations_test.clj b/test/metabase/metabot/api/conversations_test.cljindex 002d2be8d36..ead0d35697c 100644--- a/test/metabase/metabot/api/conversations_test.clj+++ b/test/metabase/metabot/api/conversations_test.clj@@ -468,6 +468,7 @@:model/MetabotMessage _a1 {:conversation_id convo-id :user_id user-id :role "assistant":external_id a1 :finished true :total_tokens 100:usage {"gpt" {:prompt 60 :completion 40}}+ :context_tokens 60:data [{:type "text" :text "hello"}]:created_at (seconds-ago 50)}:model/MetabotMessage _u2 {:conversation_id convo-id :user_id user-id :role "user"@@ -504,7 +505,10 @@(is (every? #(t/after? (t/instant (:created_at %)) (t/instant orig-a1-at)) rows))))(testing "token usage is zeroed so forks don't double-count in analytics"(is (every? #(zero? (:total_tokens %)) rows))- (is (every? #(nil? (:usage %)) rows))))+ (is (every? #(nil? (:usage %)) rows)))+ (testing "context size carries over so the fork is as full as the thread it copied"+ (is (= [nil 60] (mapv :context_tokens rows)))+ (is (= [nil 60] (mapv :contextTokens (:messages response))))))(testing "the original conversation is untouched"(is (= 4 (count (metabot.persistence/live-messages convo-id)))))(finallydiff --git a/test/metabase/metabot/persistence_test.clj b/test/metabase/metabot/persistence_test.cljindex 82f9a6266f8..079702db5ff 100644--- a/test/metabase/metabot/persistence_test.clj+++ b/test/metabase/metabot/persistence_test.clj@@ -478,6 +478,41 @@(is (= [{:type "text" :text "Hello" :state "done"}] (:data row)))(is (= 15 (:total_tokens row)))))))))+(deftest finalize-assistant-turn-stores-context-tokens-test+ (testing "finalize-assistant-turn! stores the final call's context size"+ (with-rasta-tx+ (let [conversation-id (str (random-uuid))+ {:keys [assistant-msg-id]} (metabot-persistence/start-turn!+ conversation-id "internal"+ {:role "user" :content "hi"})]+ (metabot-persistence/finalize-assistant-turn!+ assistant-msg-id+ ;; usage snapshots are cumulative, so the final call alone is 150 prompt + 10 completion+ [{:type :usage :model "claude" :usage {:promptTokens 100 :completionTokens 20}}+ {:type :text :text "Hello"}+ {:type :usage :model "claude" :usage {:promptTokens 250 :completionTokens 30}}])+ (is (=? {:context_tokens 160 :total_tokens 280}+ (t2/select-one :model/MetabotMessage assistant-msg-id))))))+ (testing "a turn that observed no usage stores no context size"+ (with-rasta-tx+ (let [conversation-id (str (random-uuid))+ {:keys [assistant-msg-id]} (metabot-persistence/start-turn!+ conversation-id "internal"+ {:role "user" :content "hi"})]+ (metabot-persistence/finalize-assistant-turn!+ assistant-msg-id [{:type :text :text "Hello"}])+ (is (nil? (:context_tokens (t2/select-one :model/MetabotMessage assistant-msg-id))))))))++(deftest ^:parallel messages->client-messages-context-tokens-test+ (testing "a message reports the context it occupied, and omits it when the turn observed no usage"+ (is (= 520 (:contextTokens (client-message {:role :assistant :data []+ :context_tokens 520}))))+ (testing "no usage observed"+ (is (not (contains? (client-message {:role :assistant :data []}) :contextTokens))))+ (testing "user rows never carry it"+ (is (not (contains? (client-message {:role :user :data [{:type "text" :text "hi"}]})+ :contextTokens))))))+(deftest finalize-assistant-turn-passes-through-aborted-and-errored-test(testing "finalize-assistant-turn! preserves :finished? false and JSON-encodes :error"(with-rasta-tx