Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[Metabot] Handle `length` finish reason (#79335)

Sloan Sparger7514efdd2a1e
diff --git a/frontend/src/metabase/metabot/components/MetabotChat/MetabotChat.tsx b/frontend/src/metabase/metabot/components/MetabotChat/MetabotChat.tsx
index a2e487742f0..b2a99b24fc9 100644
--- a/frontend/src/metabase/metabot/components/MetabotChat/MetabotChat.tsx
+++ b/frontend/src/metabase/metabot/components/MetabotChat/MetabotChat.tsx
@@ -172,6 +172,11 @@ export const MetabotChat = ({
onRetryMessage={
config.preventRetryMessage ? undefined : metabot.retryMessage
}
+ onContinueMessage={() =>
+ metabot.submitInput(
+ t`Your last response was cut off. Pick up exactly where you left off. Don't repeat anything you already wrote.`,
+ )
+ }
onRefreshConversation={() => {
metabot.setPrompt("");
metabot.loadConversation(metabot.conversationId);
diff --git a/frontend/src/metabase/metabot/components/MetabotChat/MetabotChatMessage.tsx b/frontend/src/metabase/metabot/components/MetabotChat/MetabotChatMessage.tsx
index d80cb909271..d889e52471a 100644
--- a/frontend/src/metabase/metabot/components/MetabotChat/MetabotChatMessage.tsx
+++ b/frontend/src/metabase/metabot/components/MetabotChat/MetabotChatMessage.tsx
@@ -15,6 +15,7 @@ import {
type MetabotAgentTextChatMessage,
type MetabotAgentTurnError,
type MetabotAgentTurnErroredMessage,
+ type MetabotAgentTurnIncompleteMessage,
type MetabotChatMessage,
type MetabotDataPart,
type MetabotDebugToolCallMessage,
@@ -79,6 +80,7 @@ const isUserVisibleMessage = (message: MetabotChatMessage): boolean =>
.with({ type: "tool_call" }, () => false)
.with({ type: "chain_of_thought" }, () => true)
.with({ type: "turn_aborted" }, () => true)
+ .with({ type: "turn_incomplete" }, () => true)
.with({ type: "turn_errored" }, () => true)
.with({ type: "turn_in_progress" }, () => false)
.exhaustive();
@@ -191,6 +193,7 @@ interface AgentMessageProps extends Omit<BaseMessageProps, "message"> {
readonly: boolean;
conversationId: string;
onRetry?: (messageId: string) => void;
+ onContinue?: () => void;
onRefreshConversation?: () => void;
getCopyText: () => string;
setFeedbackMessage?: (data: { messageId: string; positive: boolean }) => void;
@@ -212,6 +215,7 @@ export const AgentMessage = ({
conversationId,
getCopyText,
onRetry,
+ onContinue,
onRefreshConversation,
setFeedbackMessage,
submittedFeedback,
@@ -267,6 +271,12 @@ export const AgentMessage = ({
.with({ type: "turn_aborted" }, (m) => (
<AbortedTurnAlert messageId={m.id} debug={debug} onRetry={onRetry} />
))
+ .with({ type: "turn_incomplete" }, (m) => (
+ <IncompleteTurnAlert
+ finishReason={m.finishReason}
+ onContinue={onContinue}
+ />
+ ))
.with({ type: "turn_errored" }, (m) => (
<AgentErroredTurnAlert
message={m}
@@ -476,6 +486,50 @@ const AbortedTurnAlert = ({
);
};
+const IncompleteTurnAlert = ({
+ finishReason,
+ onContinue,
+}: {
+ finishReason: MetabotAgentTurnIncompleteMessage["finishReason"];
+ onContinue?: () => void;
+}) => {
+ const metabotName = useSetting("metabot-name");
+ const { message, continuable } = match(finishReason)
+ .with("length", () => ({
+ message: t`Response from ${metabotName} was cut off because it hit the maximum length`,
+ continuable: true,
+ }))
+ .with("content-filter", () => ({
+ message: t`Response from ${metabotName} was stopped by a content filter`,
+ continuable: false,
+ }))
+ .with("tool-calls", "other", () => ({
+ message: t`Response from ${metabotName} stopped before it finished`,
+ continuable: false,
+ }))
+ .exhaustive();
+ const canContinue = continuable && !!onContinue;
+ return (
+ <AgentTurnAlert
+ variant="info"
+ message={message}
+ cta={
+ canContinue ? (
+ <Button
+ variant="default"
+ size="compact-xs"
+ fz="xs"
+ onClick={onContinue}
+ data-testid="metabot-chat-message-continue"
+ >
+ {t`Continue`}
+ </Button>
+ ) : null
+ }
+ />
+ );
+};
+
export const getFullAgentReply = (
messages: MetabotChatMessage[],
messageId: string,
@@ -507,6 +561,7 @@ export const getFullAgentReply = (
export const Messages = ({
messages,
onRetryMessage,
+ onContinueMessage,
onRefreshConversation,
isDoingScience,
supportsReasoning = true,
@@ -521,6 +576,7 @@ export const Messages = ({
}: {
messages: MetabotChatMessage[];
onRetryMessage?: (messageId: string) => void;
+ onContinueMessage?: () => void;
onRefreshConversation?: () => void;
isDoingScience: boolean;
supportsReasoning?: boolean;
@@ -624,6 +680,7 @@ export const Messages = ({
readonly={readonly}
conversationId={conversationId}
onRetry={isLastUserMessage ? onRetryMessage : undefined}
+ onContinue={isLastUserMessage ? onContinueMessage : undefined}
onRefreshConversation={onRefreshConversation}
getCopyText={() => getAgentReplyCopyText(message.id)}
setFeedbackMessage={(data) =>
diff --git a/frontend/src/metabase/metabot/state/reducer-utils.ts b/frontend/src/metabase/metabot/state/reducer-utils.ts
index 00b48e2870b..d4a971a8dda 100644
--- a/frontend/src/metabase/metabot/state/reducer-utils.ts
+++ b/frontend/src/metabase/metabot/state/reducer-utils.ts
@@ -14,6 +14,7 @@ import type {
MetabotAgentId,
MetabotAgentTurnDisplayError,
MetabotAgentTurnError,
+ MetabotAgentTurnIncompleteMessage,
MetabotConverstationState,
MetabotDebugToolCallMessage,
MetabotSearchResults,
@@ -363,6 +364,19 @@ export const appendAgentTurnAborted = (
});
};
+export const appendAgentTurnIncomplete = (
+ convo: WritableDraft<MetabotConverstationState>,
+ finishReason: MetabotAgentTurnIncompleteMessage["finishReason"],
+) => {
+ convo.messages.push({
+ id: createMessageId(),
+ role: "agent",
+ type: "turn_incomplete",
+ finishReason,
+ externalId: convo.pendingMessageExternalId,
+ });
+};
+
export const appendAgentTurnErrored = (
convo: WritableDraft<MetabotConverstationState>,
error: MetabotAgentTurnError,
diff --git a/frontend/src/metabase/metabot/state/reducer.ts b/frontend/src/metabase/metabot/state/reducer.ts
index da01e443024..b4d473e4308 100644
--- a/frontend/src/metabase/metabot/state/reducer.ts
+++ b/frontend/src/metabase/metabot/state/reducer.ts
@@ -20,6 +20,7 @@ import {
addChainTool,
appendAgentTurnAborted,
appendAgentTurnErrored,
+ appendAgentTurnIncomplete,
appendChainReasoning,
closeChain,
convoReducer,
@@ -461,6 +462,14 @@ export const metabot = createSlice({
if (action.payload?.state) {
convo.state = { ...action.payload.state };
}
+
+ const finishReason = action.payload?.processedResponse.finishReason;
+ const isResumableFinishReason =
+ finishReason && finishReason !== "stop" && finishReason !== "error";
+ if (isResumableFinishReason) {
+ appendAgentTurnIncomplete(convo, finishReason);
+ }
+
convo.activeToolCalls = [];
closeChain(convo);
convo.isProcessing = false;
diff --git a/frontend/src/metabase/metabot/state/types.ts b/frontend/src/metabase/metabot/state/types.ts
index e9fca9dadb5..b5950a590ff 100644
--- a/frontend/src/metabase/metabot/state/types.ts
+++ b/frontend/src/metabase/metabot/state/types.ts
@@ -2,6 +2,7 @@ import type {
KnownDataPart,
SearchResultItem,
} from "metabase/api/ai-streaming/schemas";
+import type { FinishReason } from "metabase/api/ai-streaming/sse-types";
import type { MetabotProfileId } from "metabase/metabot/constants";
import type {
MetabotCodeEdit,
@@ -79,6 +80,14 @@ export type MetabotAgentTurnAbortedMessage = {
externalId?: string;
};
+export type MetabotAgentTurnIncompleteMessage = {
+ id: string;
+ role: "agent";
+ type: "turn_incomplete";
+ finishReason: Exclude<FinishReason, "stop" | "error">;
+ externalId?: string;
+};
+
export type MetabotAgentTurnDisplayError = {
type: "alert" | "locked" | "message";
message: string;
@@ -115,6 +124,7 @@ export type MetabotAgentChatMessage =
| MetabotDebugToolCallMessage
| MetabotAgentChainOfThoughtMessage
| MetabotAgentTurnAbortedMessage
+ | MetabotAgentTurnIncompleteMessage
| MetabotAgentTurnErroredMessage
| MetabotAgentTurnInProgressMessage;
diff --git a/frontend/src/metabase/metabot/tests/chat-message-rendering.unit.spec.tsx b/frontend/src/metabase/metabot/tests/chat-message-rendering.unit.spec.tsx
index d29c0e5c8f9..f88c6b3ebae 100644
--- a/frontend/src/metabase/metabot/tests/chat-message-rendering.unit.spec.tsx
+++ b/frontend/src/metabase/metabot/tests/chat-message-rendering.unit.spec.tsx
@@ -1,6 +1,7 @@
import userEvent from "@testing-library/user-event";
import fetchMock from "fetch-mock";
import { assocIn } from "icepick";
+import type { ComponentProps } from "react";
import {
createMockMetabotConversationDetail,
@@ -13,13 +14,16 @@ import { renderWithProviders, screen, within } from "__support__/ui";
import { METABOT_ERR_MSG } from "metabase/metabot/constants";
import type {
MetabotAgentChatMessage,
+ MetabotAgentTurnIncompleteMessage,
MetabotChatMessage,
} from "metabase/metabot/state";
import { getMetabotInitialState } from "metabase/metabot/state/reducer-utils";
import {
assertConversation,
+ continueResponseButton,
enterChatMessage,
input,
+ queryContinueResponseButton,
setup as renderMetabotChat,
thumbsDown,
thumbsUp,
@@ -47,7 +51,10 @@ const textMessage = (
message: string,
): FetchedChatMessage => ({ id: message, role, type: "text", message });
-const setup = (message: MetabotAgentChatMessage) =>
+const setup = (
+ message: MetabotAgentChatMessage,
+ props?: Partial<ComponentProps<typeof AgentMessage>>,
+) =>
renderWithProviders(
<AgentMessage
debug={false}
@@ -58,6 +65,7 @@ const setup = (message: MetabotAgentChatMessage) =>
submittedFeedback={undefined}
getCopyText={() => ""}
message={message}
+ {...props}
/>,
{
storeInitialState: {
@@ -325,6 +333,45 @@ describe("AgentMessage", () => {
expect(debugCard).toHaveTextContent(/boom/);
});
});
+
+ describe("turn_incomplete", () => {
+ const incompleteMessage = (
+ finishReason: MetabotAgentTurnIncompleteMessage["finishReason"],
+ ): MetabotAgentChatMessage => ({
+ id: "msg",
+ role: "agent",
+ type: "turn_incomplete",
+ finishReason,
+ });
+
+ it("offers to continue a length-limited response", async () => {
+ const onContinue = jest.fn();
+ setup(incompleteMessage("length"), { onContinue });
+
+ expect(
+ screen.getByText(/was cut off because it hit the maximum length/),
+ ).toBeInTheDocument();
+ await userEvent.click(await continueResponseButton());
+ expect(onContinue).toHaveBeenCalled();
+ });
+
+ it("explains a content-filtered response without offering to continue", () => {
+ setup(incompleteMessage("content-filter"), { onContinue: jest.fn() });
+
+ expect(
+ screen.getByText(/was stopped by a content filter/),
+ ).toBeInTheDocument();
+ expect(queryContinueResponseButton()).not.toBeInTheDocument();
+ });
+
+ it("falls back to a generic notice for other reasons", () => {
+ setup(incompleteMessage("other"));
+
+ expect(
+ screen.getByText(/stopped before it finished/),
+ ).toBeInTheDocument();
+ });
+ });
});
describe("UserMessage chart mentions", () => {
diff --git a/frontend/src/metabase/metabot/tests/message.unit.spec.tsx b/frontend/src/metabase/metabot/tests/message.unit.spec.tsx
index 7f00e00f36d..7d42c1bb076 100644
--- a/frontend/src/metabase/metabot/tests/message.unit.spec.tsx
+++ b/frontend/src/metabase/metabot/tests/message.unit.spec.tsx
@@ -1,12 +1,19 @@
+import userEvent from "@testing-library/user-event";
+
import { screen } from "__support__/ui";
+import type { SSEEvent } from "metabase/api/ai-streaming/sse-types";
import {
assertConversation,
+ continueResponseButton,
createMockSSEStream,
createPauses,
enterChatMessage,
input,
+ lastReqBody,
mockAgentEndpoint,
+ queryContinueResponseButton,
+ queryTurnAlert,
sendMessageButton,
setup,
whoIsYourFavoriteResponse,
@@ -102,3 +109,83 @@ describe("metabot > message", () => {
]);
});
});
+
+const truncatedResponse: SSEEvent[] = [
+ { type: "start", messageId: "msg_truncated" },
+ { type: "text-start", id: "t1" },
+ { type: "text-delta", id: "t1", delta: "Here is the start of a long answer" },
+ { type: "text-end", id: "t1" },
+ { type: "finish", finishReason: "length" },
+];
+
+describe("metabot > truncated responses", () => {
+ it("should keep the partial text and offer to continue", async () => {
+ setup();
+ mockAgentEndpoint({ events: truncatedResponse });
+
+ await enterChatMessage("Tell me everything");
+
+ await assertConversation([
+ ["user", "Tell me everything"],
+ ["agent", "Here is the start of a long answer"],
+ ["agent", /was cut off/],
+ ]);
+ expect(await continueResponseButton()).toBeInTheDocument();
+ });
+
+ it("should send a continuation user turn when continue is clicked", async () => {
+ setup();
+ mockAgentEndpoint({ events: truncatedResponse });
+ await enterChatMessage("Tell me everything");
+
+ const continuationSpy = mockAgentEndpoint({
+ events: [
+ { type: "start", messageId: "msg_continued" },
+ { type: "text-start", id: "t2" },
+ { type: "text-delta", id: "t2", delta: "and here is the rest." },
+ { type: "text-end", id: "t2" },
+ { type: "finish", finishReason: "stop" },
+ ],
+ });
+
+ await userEvent.click(await continueResponseButton());
+
+ await assertConversation([
+ ["user", "Tell me everything"],
+ ["agent", "Here is the start of a long answer"],
+ ["agent", /was cut off/],
+ ["user", /Your last response was cut off/],
+ ["agent", "and here is the rest."],
+ ]);
+ expect((await lastReqBody(continuationSpy))?.message).toMatch(
+ /Pick up exactly where you left off/,
+ );
+ });
+
+ it("should not offer continue on earlier turns", async () => {
+ setup();
+ mockAgentEndpoint({ events: truncatedResponse });
+ await enterChatMessage("Tell me everything");
+ expect(await continueResponseButton()).toBeInTheDocument();
+
+ mockAgentEndpoint({ events: whoIsYourFavoriteResponse });
+ await enterChatMessage("Nevermind, who is your favorite?");
+ expect(
+ await screen.findByText("You, but don't tell anyone."),
+ ).toBeInTheDocument();
+
+ expect(queryContinueResponseButton()).not.toBeInTheDocument();
+ });
+
+ it("should not show a notice for a normal stop", async () => {
+ setup();
+ mockAgentEndpoint({ events: whoIsYourFavoriteResponse });
+
+ await enterChatMessage("Who is your favorite?");
+ await assertConversation([
+ ["user", "Who is your favorite?"],
+ ["agent", "You, but don't tell anyone."],
+ ]);
+ expect(queryTurnAlert()).not.toBeInTheDocument();
+ });
+});
diff --git a/frontend/src/metabase/metabot/tests/utils.tsx b/frontend/src/metabase/metabot/tests/utils.tsx
index d34dd62af22..44db5bb43d6 100644
--- a/frontend/src/metabase/metabot/tests/utils.tsx
+++ b/frontend/src/metabase/metabot/tests/utils.tsx
@@ -110,6 +110,12 @@ export const sendMessageButton = () =>
screen.findByTestId("metabot-send-message");
export const stopResponseButton = () =>
screen.findByTestId("metabot-stop-response");
+export const continueResponseButton = () =>
+ screen.findByTestId("metabot-chat-message-continue");
+export const queryContinueResponseButton = () =>
+ screen.queryByTestId("metabot-chat-message-continue");
+export const queryTurnAlert = () =>
+ screen.queryByTestId("metabot-chat-message-turn-alert");
export const closeChatButton = () => screen.findByTestId("metabot-close-chat");
export const responseLoader = () =>
screen.findByTestId("metabot-response-loader");
diff --git a/frontend/src/metabase/metabot/utils/slack-mrkdwn.ts b/frontend/src/metabase/metabot/utils/slack-mrkdwn.ts
index d9f13372143..59516235f5d 100644
--- a/frontend/src/metabase/metabot/utils/slack-mrkdwn.ts
+++ b/frontend/src/metabase/metabot/utils/slack-mrkdwn.ts
@@ -37,6 +37,7 @@ export function convertSlackChatMessage(
.with({ role: "agent", type: "tool_call" }, (m) => m)
.with({ role: "agent", type: "chain_of_thought" }, (m) => m)
.with({ role: "agent", type: "turn_aborted" }, (m) => m)
+ .with({ role: "agent", type: "turn_incomplete" }, (m) => m)
.with({ role: "agent", type: "turn_errored" }, (m) => m)
.with({ role: "agent", type: "turn_in_progress" }, (m) => m)
.exhaustive();
diff --git a/src/metabase/metabot/agent/core.clj b/src/metabase/metabot/agent/core.clj
index 7da9f695646..1b466eb3568 100644
--- a/src/metabase/metabot/agent/core.clj
+++ b/src/metabase/metabot/agent/core.clj
@@ -194,17 +194,27 @@
(contains? success-ids (:id p))))
parts)))))
+(defn- truncated?
+ "Whether the provider cut this iteration off at its output-token limit. A truncated
+ turn may contain a half-streamed tool call, so it must not continue the loop."
+ [parts]
+ (some #(and (= (:type %) :usage)
+ (= (:finish-reason %) "length"))
+ parts))
+
(defn- should-continue?
"Determine if agent should continue iterating."
[iteration max-iterations terminal-tools parts]
(and (< iteration max-iterations)
(has-tool-calls? parts)
- (not (terminal-tool-call? terminal-tools parts))))
+ (not (terminal-tool-call? terminal-tools parts))
+ (not (truncated? parts))))
(defn- finish-reason
"Determine why the agent loop stopped."
[iteration max-iterations terminal-tools parts]
(cond
+ (truncated? parts) :length
(>= iteration max-iterations) :max-iterations
(terminal-tool-call? terminal-tools parts) :terminal-tool
:else :stop))
@@ -565,7 +575,9 @@
:else
(let [reason (finish-reason iteration max-iter terminal-tools parts)]
- (log/info "Agent loop complete" {:iterations iteration :reason reason})
+ (if (= reason :length)
+ (log/warn "Agent loop complete" {:iterations iteration :reason reason})
+ (log/info "Agent loop complete" {:iterations iteration :reason reason}))
(assoc loop-state
:status :done
;; surfaced so run-agent-loop can record it on the turn span
diff --git a/src/metabase/metabot/self/claude.clj b/src/metabase/metabot/self/claude.clj
index ff9b18f9957..16497f2f216 100644
--- a/src/metabase/metabot/self/claude.clj
+++ b/src/metabase/metabot/self/claude.clj
@@ -55,6 +55,16 @@
"Claude content-block types we translate into AI SDK chunks."
#{:text :tool_use :thinking :redacted_thinking})
+(def ^:private stop-reasons
+ "Anthropic `stop_reason` → AI SDK v5 `FinishReason`."
+ {"end_turn" "stop"
+ "stop_sequence" "stop"
+ "max_tokens" "length"
+ "model_context_window_exceeded" "length"
+ "tool_use" "tool-calls"
+ "refusal" "content-filter"
+ "pause_turn" "stop"})
+
(defn claude->aisdk-chunks-xf
"Translates Claude /v1/messages streaming events into AI SDK v5 protocol chunks.
@@ -85,6 +95,7 @@
;; normally, but if the stream is interrupted we flush the last known
;; usage in the completion arity so we don't lose data entirely.
last-usage (volatile! nil)
+ stop-reason (volatile! nil)
close! (fn [result]
(u/prog1 (if-let [end-type (case @current-type
:text :text-end
@@ -103,10 +114,12 @@
;; close up latest type if incomplete
@current-type (close!)
;; flush last-known usage if stream ended before message_delta.
- @last-usage (rf {:type :usage
- :usage (claude-usage->aisdk-usage @last-usage)
- :id @message-id
- :model @model-name})
+ @last-usage (rf (cond-> {:type :usage
+ :usage (claude-usage->aisdk-usage @last-usage)
+ :id @message-id
+ :model @model-name}
+ @stop-reason (assoc :finish-reason (core/stop-reason->finish-reason stop-reasons @stop-reason)
+ :raw-finish-reason @stop-reason)))
true (rf)))
([result {t :type :keys [message content_block delta error index] :as chunk}]
(let [block-type (when content_block
@@ -169,7 +182,8 @@
;; https://platform.claude.com/docs/en/build-with-claude/streaming#event-types
;; https://platform.claude.com/docs/en/api/cli/messages#message_delta_usage
(= t "message_delta") (u/prog1
- (vreset! last-usage (:usage chunk)))
+ (vreset! last-usage (:usage chunk))
+ (vreset! stop-reason (:stop_reason delta)))
;; end of message
(= t "message_stop") identity
;; catch errors if any
diff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.clj
index bb6763c36b4..fd1a034a2d1 100644
--- a/src/metabase/metabot/self/core.clj
+++ b/src/metabase/metabot/self/core.clj
@@ -126,6 +126,17 @@
;;; AISDK5
+(def finish-reasons
+ "The AI SDK v5 `FinishReason` values a provider stop reason may be translated to."
+ #{"stop" "length" "content-filter" "tool-calls" "error" "other"})
+
+(defn stop-reason->finish-reason
+ "Translate a raw provider stop reason to an AI SDK v5 `FinishReason` through that provider's `stop-reasons` table.
+ Unmapped reasons → \"other\"; nil → nil."
+ [stop-reasons raw]
+ (when raw
+ (get stop-reasons raw "other")))
+
(defn- parse-tool-arguments
"Parse concatenated tool input deltas as JSON.
Falls back to returning the raw string wrapped in a map when parsing fails,
@@ -368,6 +379,7 @@
(fn [rf]
(let [error? (volatile! false)
finish-error-code (volatile! nil)
+ finish-reason (volatile! nil)
started? (volatile! false)
usage-by-model (volatile! {})
;; non-nil while a text block is open; holds the block id so we can
@@ -411,7 +423,10 @@
(cond-> @started? (rf (format-sse-event {:type "finish-step"})))
(rf (format-sse-event
(cond-> {:type "finish"
- :finishReason (if @error? "error" "stop")}
+ :finishReason (cond
+ (= @finish-reason "length") "length"
+ @error? "error"
+ :else "stop")}
(seq metadata) (assoc :messageMetadata metadata))))
(rf done-sse-line)
(rf))))
@@ -501,6 +516,8 @@
;; cumulative per-model snapshot; last-wins, emitted on finish
(do
(vswap! usage-by-model assoc (or (:model part) "unknown") (:usage part))
+ (when-let [fr (:finish-reason part)]
+ (vreset! finish-reason fr))
result)
;; Unknown types: emit as data parts
diff --git a/src/metabase/metabot/self/mistral.clj b/src/metabase/metabot/self/mistral.clj
index 9023bb42da4..ece24e25d40 100644
--- a/src/metabase/metabot/self/mistral.clj
+++ b/src/metabase/metabot/self/mistral.clj
@@ -133,10 +133,17 @@
(catch Exception e
(core/rethrow-api-error! "mistral" mistral-error-msg e))))))
+(def ^:private stop-reasons
+ "Mistral adds `model_length` — the model's own context limit, a truncation just like `length` — and reports a
+ mid-generation failure as a finish reason instead of an error event."
+ (assoc chat-completions/stop-reasons
+ "model_length" "length"
+ "error" "error"))
+
(defn mistral->aisdk-chunks-xf
"Translates Mistral Chat Completions streaming chunks into AI SDK v5 protocol chunks."
[]
- (chat-completions/chat-completions->aisdk-chunks-xf))
+ (chat-completions/chat-completions->aisdk-chunks-xf stop-reasons))
(defn mistral
"Call the Mistral Chat Completions API, return AISDK stream."
diff --git a/src/metabase/metabot/self/openai.clj b/src/metabase/metabot/self/openai.clj
index 11f1c5f529d..af947fa73f5 100644
--- a/src/metabase/metabot/self/openai.clj
+++ b/src/metabase/metabot/self/openai.clj
@@ -17,6 +17,12 @@
"Output item types we translate into AI SDK chunks."
#{:text :function_call :reasoning})
+(def ^:private stop-reasons
+ "Responses API `incomplete_details.reason` → AI SDK v5 `FinishReason`. Only an incomplete response carries a reason,
+ so there is nothing here for a normal or tool-call finish."
+ {"max_output_tokens" "length"
+ "content_filter" "content-filter"})
+
(defn- openai-usage->aisdk-usage
"Convert an OpenAI Responses API `usage` block into the AISDK `:usage` shape.
@@ -164,11 +170,14 @@
;; An incomplete response (e.g. truncated at max_output_tokens or stopped by a content filter)
;; still has valid partial output, so we record its usage rather than treating it as an error.
(contains? #{"response.completed" "response.incomplete"} t)
- (rf {:type :usage
- :usage (openai-usage->aisdk-usage (:usage response))
- ;; non-standard extension, not in AISDK5
- :id (:id response)
- :model @model-name})
+ (rf (let [raw (get-in response [:incomplete_details :reason])]
+ (cond-> {:type :usage
+ :usage (openai-usage->aisdk-usage (:usage response))
+ ;; non-standard extension, not in AISDK5
+ :id (:id response)
+ :model @model-name}
+ raw (assoc :finish-reason (core/stop-reason->finish-reason stop-reasons raw)
+ :raw-finish-reason raw))))
;; `response.failed` is the Responses API's terminal failure event. Its error lives nested under
;; `response.error`, not in a top-level `error` event, so surface it explicitly.
(= t "response.failed") (rf {:type :error
diff --git a/src/metabase/metabot/self/openai/chat_completions.clj b/src/metabase/metabot/self/openai/chat_completions.clj
index 617a25a2547..fc934ec9057 100644
--- a/src/metabase/metabot/self/openai/chat_completions.clj
+++ b/src/metabase/metabot/self/openai/chat_completions.clj
@@ -110,6 +110,15 @@
;;; Streaming response → AISDK v5 chunks
+(def stop-reasons
+ "Chat Completions `finish_reason` → AI SDK v5 `FinishReason`. Adapters whose dialect adds reasons beyond OpenAI's
+ extend this and pass the result to [[chat-completions->aisdk-chunks-xf]]."
+ {"stop" "stop"
+ "length" "length"
+ "tool_calls" "tool-calls"
+ "function_call" "tool-calls"
+ "content_filter" "content-filter"})
+
(defn chat-completions->aisdk-chunks-xf
"Translates Chat Completions streaming chunks into AI SDK v5 protocol chunks.
@@ -135,97 +144,107 @@
previous block and opens a new one. That relies on providers sending `id` only
on a tool call's opening chunk — one that repeated it on continuation chunks
would lose their arguments, since neither the start branch (needs `:name`) nor
- the argument-delta branch (needs no `:id`) would fire."
- []
- (fn [rf]
- (let [current-type (volatile! nil) ;; :text | :function_call | nil
- current-id (volatile! nil) ;; active chunk id (text-id or tool call_id)
- message-id (volatile! nil)
- model-name (volatile! nil)
- payload (volatile! {}) ;; carried across start/delta/end, same as openai.clj
- close! (fn [result]
- (u/prog1 (rf result (merge {:type (case @current-type
- :text :text-end
- :function_call :tool-input-available)}
- @payload))
- (vreset! current-type nil)
- (vreset! current-id nil)
- (vreset! payload {})))]
- (fn
- ([result]
- (cond-> result
- @current-type (close!)
- true (rf)))
+ the argument-delta branch (needs no `:id`) would fire.
+
+ Takes the dialect's `finish_reason` table, defaulting to OpenAI's [[stop-reasons]]."
+ ([]
+ (chat-completions->aisdk-chunks-xf stop-reasons))
+ ([stop-reasons]
+ (fn [rf]
+ (let [current-type (volatile! nil) ;; :text | :function_call | nil
+ current-id (volatile! nil) ;; active chunk id (text-id or tool call_id)
+ message-id (volatile! nil)
+ model-name (volatile! nil)
+ payload (volatile! {}) ;; carried across start/delta/end, same as openai.clj
+ stop-reason (volatile! nil)
+ close! (fn [result]
+ (u/prog1 (rf result (merge {:type (case @current-type
+ :text :text-end
+ :function_call :tool-input-available)}
+ @payload))
+ (vreset! current-type nil)
+ (vreset! current-id nil)
+ (vreset! payload {})))]
+ (fn
+ ([result]
+ (cond-> result
+ @current-type (close!)
+ true (rf)))
- ([result {:keys [id model choices usage] :as _chunk}]
- (let [choice (first choices)
- delta (:delta choice)
- finish-reason (:finish_reason choice)
- tool-call (first (:tool_calls delta))
- ;; Determine what kind of content this chunk carries.
- ;; Empty-string content (common between tool calls) is ignored
- ;; to avoid spurious text blocks that would close open tools.
- chunk-type (cond
- (not-empty (:content delta)) :text
- (some? tool-call) :function_call
- :else nil)
- ;; For new tool calls, the id comes from the chunk; for deltas
- ;; on the same tool, we keep current-id.
- chunk-id (or (:id tool-call) @current-id (core/mkid))]
- (cond-> result
- ;; Emit :start on first chunk
- (and id (not @message-id)) (-> (rf {:type :start :messageId id})
- (u/prog1
- (vreset! message-id id)
- (vreset! model-name model)))
- ;; Close previous block when type changes, or when a new tool
- ;; call arrives (different id = different tool in parallel)
- (and @current-type
- (or (and chunk-type
- (not= chunk-type @current-type))
- (and (= chunk-type :function_call)
- (not= chunk-id @current-id)))) (close!)
- ;; Start a new text block
- (and (= chunk-type :text)
- (not= @current-type :text)) (-> (u/prog1
- (let [tid (core/mkid)]
- (vreset! current-type :text)
- (vreset! current-id tid)
- (vreset! payload {:id tid})))
- (rf (merge {:type :text-start} @payload)))
- ;; Text delta
- (and (= chunk-type :text)
- (some? (:content delta))) (rf {:type :text-delta
- :id @current-id
- :delta (:content delta)})
- ;; Start a new tool call block
- (and (= chunk-type :function_call)
- (:id tool-call)
- (:name (:function tool-call))) (-> (u/prog1
- (vreset! current-type :function_call)
- (vreset! current-id (:id tool-call))
- (vreset! payload {:toolCallId (:id tool-call)
- :toolName (:name (:function tool-call))}))
- (rf (merge {:type :tool-input-start} @payload))
- ;; Emit initial arguments if present
- (cond-> (not (str/blank? (:arguments (:function tool-call))))
- (rf {:type :tool-input-delta
- :toolCallId (:id tool-call)
- :inputTextDelta (:arguments (:function tool-call))})))
- ;; Tool argument delta (continuation of existing tool call)
- (and (= chunk-type :function_call)
- (not (:id tool-call))
- (some? (:arguments (:function tool-call)))) (rf {:type :tool-input-delta
- :toolCallId (:toolCallId @payload)
- :inputTextDelta (:arguments (:function tool-call))})
- ;; Finish reason — close whatever is open
- (some? finish-reason) (cond->
- @current-type (close!))
- ;; Usage (often on a separate final chunk with empty choices)
- (some? usage) (rf {:type :usage
- :usage (usage->aisdk-usage usage)
- :id @message-id
- :model @model-name}))))))))
+ ([result {:keys [id model choices usage] :as _chunk}]
+ (let [choice (first choices)
+ delta (:delta choice)
+ finish-reason (:finish_reason choice)
+ tool-call (first (:tool_calls delta))
+ ;; Determine what kind of content this chunk carries.
+ ;; Empty-string content (common between tool calls) is ignored
+ ;; to avoid spurious text blocks that would close open tools.
+ chunk-type (cond
+ (not-empty (:content delta)) :text
+ (some? tool-call) :function_call
+ :else nil)
+ ;; For new tool calls, the id comes from the chunk; for deltas
+ ;; on the same tool, we keep current-id.
+ chunk-id (or (:id tool-call) @current-id (core/mkid))]
+ (cond-> result
+ ;; Emit :start on first chunk
+ (and id (not @message-id)) (-> (rf {:type :start :messageId id})
+ (u/prog1
+ (vreset! message-id id)
+ (vreset! model-name model)))
+ ;; Close previous block when type changes, or when a new tool
+ ;; call arrives (different id = different tool in parallel)
+ (and @current-type
+ (or (and chunk-type
+ (not= chunk-type @current-type))
+ (and (= chunk-type :function_call)
+ (not= chunk-id @current-id)))) (close!)
+ ;; Start a new text block
+ (and (= chunk-type :text)
+ (not= @current-type :text)) (-> (u/prog1
+ (let [tid (core/mkid)]
+ (vreset! current-type :text)
+ (vreset! current-id tid)
+ (vreset! payload {:id tid})))
+ (rf (merge {:type :text-start} @payload)))
+ ;; Text delta
+ (and (= chunk-type :text)
+ (some? (:content delta))) (rf {:type :text-delta
+ :id @current-id
+ :delta (:content delta)})
+ ;; Start a new tool call block
+ (and (= chunk-type :function_call)
+ (:id tool-call)
+ (:name (:function tool-call))) (-> (u/prog1
+ (vreset! current-type :function_call)
+ (vreset! current-id (:id tool-call))
+ (vreset! payload {:toolCallId (:id tool-call)
+ :toolName (:name (:function tool-call))}))
+ (rf (merge {:type :tool-input-start} @payload))
+ ;; Emit initial arguments if present
+ (cond-> (not (str/blank? (:arguments (:function tool-call))))
+ (rf {:type :tool-input-delta
+ :toolCallId (:id tool-call)
+ :inputTextDelta (:arguments (:function tool-call))})))
+ ;; Tool argument delta (continuation of existing tool call)
+ (and (= chunk-type :function_call)
+ (not (:id tool-call))
+ (some? (:arguments (:function tool-call)))) (rf {:type :tool-input-delta
+ :toolCallId (:toolCallId @payload)
+ :inputTextDelta (:arguments (:function tool-call))})
+ ;; Finish reason — close whatever is open
+ (some? finish-reason) (-> (u/prog1
+ (vreset! stop-reason finish-reason))
+ (cond->
+ @current-type (close!)))
+ ;; Usage (often on a separate final chunk with empty choices)
+ (some? usage) (rf (cond-> {:type :usage
+ :usage (usage->aisdk-usage usage)
+ :id @message-id
+ :model @model-name}
+ @stop-reason
+ (assoc :finish-reason (core/stop-reason->finish-reason stop-reasons @stop-reason)
+ :raw-finish-reason @stop-reason)))))))))))
;;; Request body
diff --git a/src/metabase/metabot/self/openrouter.clj b/src/metabase/metabot/self/openrouter.clj
index dd21750c6a5..8276ef27e6b 100644
--- a/src/metabase/metabot/self/openrouter.clj
+++ b/src/metabase/metabot/self/openrouter.clj
@@ -115,12 +115,17 @@
;;; Streaming response → AISDK v5 chunks
+(def ^:private stop-reasons
+ "OpenRouter normalizes each upstream model's reason into the Chat Completions set (the raw value stays in
+ `native_finish_reason`), and adds `error` for a mid-generation upstream failure."
+ (assoc chat-completions/stop-reasons "error" "error"))
+
(defn openrouter->aisdk-chunks-xf
"Translates Chat Completions streaming chunks into AI SDK v5 protocol chunks.
OpenRouter streams the generic Chat Completions dialect; see
[[chat-completions/chat-completions->aisdk-chunks-xf]]."
[]
- (chat-completions/chat-completions->aisdk-chunks-xf))
+ (chat-completions/chat-completions->aisdk-chunks-xf stop-reasons))
;;; HTTP request
diff --git a/src/metabase/metabot/self/zai.clj b/src/metabase/metabot/self/zai.clj
index 6d41c2ca780..e2c19d171db 100644
--- a/src/metabase/metabot/self/zai.clj
+++ b/src/metabase/metabot/self/zai.clj
@@ -127,10 +127,17 @@
(catch Exception e
(core/rethrow-api-error! "zai" zai-error-msg e))))))
+(def ^:private stop-reasons
+ "Z.AI signals a filtered response with `sensitive` rather than OpenAI's `content_filter`, and reports an upstream
+ failure as a finish reason instead of an error event."
+ (assoc chat-completions/stop-reasons
+ "sensitive" "content-filter"
+ "network_error" "error"))
+
(defn zai->aisdk-chunks-xf
"Translates Z.AI Chat Completions streaming chunks into AI SDK v5 protocol chunks."
[]
- (chat-completions/chat-completions->aisdk-chunks-xf))
+ (chat-completions/chat-completions->aisdk-chunks-xf stop-reasons))
(defn zai
"Call the Z.AI Chat Completions API, return AISDK stream."
diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.clj
index 76ca94db7d2..21695b69e1b 100644
--- a/test/metabase/metabot/agent/core_test.clj
+++ b/test/metabase/metabot/agent/core_test.clj
@@ -64,6 +64,21 @@
(is (not (#'agent/should-continue? 0 max-iter no-term [{:type :usage}])))
(is (not (#'agent/should-continue? 0 max-iter no-term []))))))
+(deftest truncated-iteration-test
+ (let [truncated [{:type :tool-input :id "t1"}
+ {:type :usage :finish-reason "length" :raw-finish-reason "max_tokens"}]
+ complete [{:type :tool-input :id "t1"}
+ {:type :usage :finish-reason "tool-calls" :raw-finish-reason "tool_use"}]]
+ (testing "a provider-truncated iteration does not continue, even with a tool call present"
+ (is (not (#'agent/should-continue? 0 20 #{} truncated)))
+ (is (#'agent/should-continue? 0 20 #{} complete)))
+ (testing "finish-reason reports :length ahead of everything else"
+ (is (= :length (#'agent/finish-reason 0 20 #{} truncated)))
+ (is (= :length (#'agent/finish-reason 20 20 #{} truncated))
+ ":length wins over :max-iterations — it is the more specific cause"))
+ (testing "a usage part without a finish reason (older adapters) is not truncation"
+ (is (= :stop (#'agent/finish-reason 0 20 #{} [{:type :text} {:type :usage}]))))))
+
(deftest terminal-tool-call-test
(let [terminal #{"edit_sql_query" "create_sql_query" "replace_sql_query"}
success [{:type :tool-input :id "a" :function "edit_sql_query"}
@@ -142,6 +157,31 @@
(is (some #(= :data (:type %)) result))
;; Should have tool-related parts
(is (some #(= :tool-input (:type %)) result))))))
+ (testing "a provider-truncated turn stops the loop WITHOUT an error part (truncation is not an error;
+ the partial turn must stay replayable so the user can continue from it)"
+ (let [call-count (atom 0)]
+ (mt/with-dynamic-fn-redefs [openrouter/openrouter (fn [_]
+ (swap! call-count inc)
+ (mut/mock-llm-response
+ [{:type :tool-input
+ :id "t1"
+ :function "search"
+ :arguments {:query "test"}}
+ {:type :usage :model "m"
+ :usage {:promptTokens 1 :completionTokens 64 :totalTokens 65}
+ :finish-reason "length" :raw-finish-reason "max_tokens"}]))]
+ (let [result (into [] (agent/run-agent-loop
+ {:messages [{:role :user :content "Hi"}]
+ :state {}
+ :profile-id :embedding_next
+ :context {}}))]
+ (is (= 1 @call-count)
+ "does not iterate on a truncated turn despite the tool call")
+ (is (not-any? #(= :error (:type %)) result)
+ "no error part — the client learns of the truncation via finishReason \"length\"")
+ (is (some #(and (= :usage (:type %)) (= "length" (:finish-reason %))) result))
+ (is (some #(= :data (:type %)) result)
+ "state data part still closes the turn")))))
(testing "handles errors gracefully"
(mt/with-dynamic-fn-redefs [openrouter/openrouter (fn [_]
(throw (ex-info "Mock error" {})))]
diff --git a/test/metabase/metabot/self/claude_test.clj b/test/metabase/metabot/self/claude_test.clj
index 32b69c5385f..6304f69d1db 100644
--- a/test/metabase/metabot/self/claude_test.clj
+++ b/test/metabase/metabot/self/claude_test.clj
@@ -186,6 +186,27 @@
(is (= {:cacheCreationTokens 0 :cacheReadTokens 0}
(select-keys (:usage usage) [:cacheCreationTokens :cacheReadTokens]))))))
+(deftest ^:parallel claude-stop-reason-on-usage-chunk-test
+ (letfn [(usage-chunk [stop-reason]
+ (let [events [{:type "message_start"
+ :message {:id "msg-1" :model "claude-haiku-4-5" :usage {:input_tokens 10}}}
+ {:type "content_block_start" :index 0 :content_block {:type "text"}}
+ {:type "content_block_delta" :index 0 :delta {:type "text_delta" :text "hi"}}
+ {:type "message_delta"
+ :delta {:stop_reason stop-reason}
+ :usage {:input_tokens 10 :output_tokens 64}}
+ {:type "message_stop"}]]
+ (->> (into [] (claude/claude->aisdk-chunks-xf) events)
+ (filter #(= :usage (:type %)))
+ first)))]
+ (testing "the AI SDK finish reason rides the usage chunk alongside the raw provider value"
+ (are [raw finish-reason] (=? {:finish-reason finish-reason :raw-finish-reason raw}
+ (usage-chunk raw))
+ "max_tokens" "length"
+ "end_turn" "stop"
+ "pause_turn" "stop"
+ "compaction" "other"))))
+
(deftest ^:parallel claude-thinking-blocks-translated-test
(testing "thinking content blocks become reasoning chunks; signature rides the end"
(let [events [{:type "message_start"
diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.clj
index be87b11be1b..6dfd32c12dd 100644
--- a/test/metabase/metabot/self_test.clj
+++ b/test/metabase/metabot/self_test.clj
@@ -11,7 +11,11 @@
[metabase.metabot.self :as self]
[metabase.metabot.self.claude :as self.claude]
[metabase.metabot.self.core :as self.core]
+ [metabase.metabot.self.mistral :as self.mistral]
+ [metabase.metabot.self.openai :as self.openai]
+ [metabase.metabot.self.openai.chat-completions :as self.chat-completions]
[metabase.metabot.self.openrouter :as openrouter]
+ [metabase.metabot.self.zai :as self.zai]
[metabase.metabot.settings :as metabot.settings]
[metabase.metabot.test-util :as test-util]
[metabase.metabot.usage :as usage]
@@ -556,6 +560,61 @@
(str "event does not match the wire schema: " (pr-str event))))
events))))
+(def ^:private provider-stop-reasons
+ {:claude @#'self.claude/stop-reasons
+ :openai @#'self.openai/stop-reasons
+ :chat-completions self.chat-completions/stop-reasons
+ :mistral @#'self.mistral/stop-reasons
+ :openrouter @#'openrouter/stop-reasons
+ :zai @#'self.zai/stop-reasons})
+
+(deftest ^:parallel stop-reason->finish-reason-test
+ (testing "every provider maps only to legal AI SDK FinishReasons"
+ (doseq [[provider stop-reasons] provider-stop-reasons]
+ (testing provider
+ (is (every? self.core/finish-reasons (vals stop-reasons))))))
+ (testing "truncation — the value the agent loop keys off"
+ (are [provider raw] (= "length" (self.core/stop-reason->finish-reason (provider-stop-reasons provider) raw))
+ :claude "max_tokens"
+ :openai "max_output_tokens"
+ :chat-completions "length"
+ ;; Mistral's own context limit, which has no OpenAI equivalent
+ :mistral "model_length"))
+ (testing "content filtering, which each dialect names differently"
+ (are [provider raw] (= "content-filter" (self.core/stop-reason->finish-reason (provider-stop-reasons provider) raw))
+ :claude "refusal"
+ :openai "content_filter"
+ :chat-completions "content_filter"
+ :zai "sensitive"))
+ (testing "upstream failure, which some dialects report as a finish reason rather than an error event"
+ (are [provider raw] (= "error" (self.core/stop-reason->finish-reason (provider-stop-reasons provider) raw))
+ :mistral "error"
+ :openrouter "error"
+ :zai "network_error"))
+ (testing "unmapped → \"other\"; nil → nil"
+ (is (= "other" (self.core/stop-reason->finish-reason @#'self.claude/stop-reasons "something_new")))
+ ;; a stop reason belonging to another provider is not silently translated
+ (is (= "other" (self.core/stop-reason->finish-reason @#'self.claude/stop-reasons "max_output_tokens")))
+ (is (= "other" (self.core/stop-reason->finish-reason self.chat-completions/stop-reasons "sensitive")))
+ (is (nil? (self.core/stop-reason->finish-reason @#'self.claude/stop-reasons nil)))))
+
+(deftest parts->aisdk-sse-xf-finish-reason-test
+ (testing "a truncated turn emits finishReason \"length\" and without an error"
+ (is (= ["text-start" "text-delta" "text-end" "finish"]
+ (mapv :type
+ (sse-events [{:type :text :id "t1" :text "partial"}
+ {:type :usage :model "m" :usage {:promptTokens 1 :completionTokens 2 :totalTokens 3}
+ :finish-reason "length" :raw-finish-reason "max_tokens"}]))))
+ (is (=? {:type "finish" :finishReason "length"}
+ (last (sse-events [{:type :text :id "t1" :text "partial"}
+ {:type :usage :model "m" :usage {:promptTokens 1 :completionTokens 2 :totalTokens 3}
+ :finish-reason "length" :raw-finish-reason "max_tokens"}])))))
+ (testing "an in-turn error part does not override a length finish"
+ (is (=? {:type "finish" :finishReason "length"}
+ (last (sse-events [{:type :error :error {:message "tool blew up mid-turn"}}
+ {:type :usage :model "m" :usage {:promptTokens 1 :completionTokens 2 :totalTokens 3}
+ :finish-reason "length" :raw-finish-reason "max_tokens"}]))))))
+
(deftest parts->aisdk-sse-xf-lifecycle-test
(testing "first :start opens the message and a step; later :start is a step boundary; completion closes"
(is (= [["start" "msg-1"] ["start-step" nil]