Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(in-app-agent): surface output-token truncation instead of a blank turn (#16803)

David Traina8cad3e2f2265
diff --git a/packages/shared/src/features/inAppAgent/types.ts b/packages/shared/src/features/inAppAgent/types.ts
index 4275beddcd..d7e6fa2ab3 100644
--- a/packages/shared/src/features/inAppAgent/types.ts
+++ b/packages/shared/src/features/inAppAgent/types.ts
@@ -66,6 +66,11 @@ export enum InAppAgentRunErrorCode {
* configured wall but did not produce a final answer.
*/
STEP_LIMIT = "step_limit",
+ /**
+ * The last model step ended with a `length` finish: the response hit the
+ * provider's output-token limit before a final answer.
+ */
+ OUTPUT_LIMIT = "output_limit",
}
/**
diff --git a/web/src/features/in-app-agent/lib/backgroundExecutionSession.clienttest.ts b/web/src/features/in-app-agent/lib/backgroundExecutionSession.clienttest.ts
index b7b59d0495..b4c5f2061a 100644
--- a/web/src/features/in-app-agent/lib/backgroundExecutionSession.clienttest.ts
+++ b/web/src/features/in-app-agent/lib/backgroundExecutionSession.clienttest.ts
@@ -18,6 +18,7 @@ import { BackgroundExecutionConnectionError } from "./backgroundExecutionErrors"
import {
BackgroundExecutionSessionController,
getBackgroundRunNotice,
+ getSettledActivityOutcome,
type BackgroundExecutionView,
} from "./backgroundExecutionSession";
@@ -1405,6 +1406,41 @@ describe("getBackgroundRunNotice", () => {
tone: "info",
});
});
+
+ const truncatedRun = (errorCode: InAppAgentRunErrorCode) => ({
+ id: "run-1",
+ status: InAppAgentRunStatus.SUCCEEDED,
+ errorCode,
+ cancelRequested: false,
+ });
+
+ it.each([
+ [
+ InAppAgentRunErrorCode.STEP_LIMIT,
+ "The assistant had to stop before finishing this answer. Too many steps in one turn. Send another message to continue.",
+ ],
+ [
+ InAppAgentRunErrorCode.OUTPUT_LIMIT,
+ "The assistant had to stop before finishing this answer. The response hit the model's output limit. Send another message to continue.",
+ ],
+ ] as const)("warns and marks SUCCEEDED %s as stopped", (errorCode, text) => {
+ const run = truncatedRun(errorCode);
+
+ expect(getBackgroundRunNotice(run)).toEqual({ text, tone: "warning" });
+ expect(getSettledActivityOutcome(run)).toBe("stopped");
+ });
+
+ it("shows no notice and marks a plain SUCCEEDED run as worked", () => {
+ const run = {
+ id: "run-1",
+ status: InAppAgentRunStatus.SUCCEEDED,
+ errorCode: null,
+ cancelRequested: false,
+ };
+
+ expect(getBackgroundRunNotice(run)).toBeNull();
+ expect(getSettledActivityOutcome(run)).toBe("worked");
+ });
});
function sseFrame(frame: Record<string, unknown>): string {
diff --git a/web/src/features/in-app-agent/lib/backgroundExecutionSession.ts b/web/src/features/in-app-agent/lib/backgroundExecutionSession.ts
index d4ee98db7c..249dadcbb3 100644
--- a/web/src/features/in-app-agent/lib/backgroundExecutionSession.ts
+++ b/web/src/features/in-app-agent/lib/backgroundExecutionSession.ts
@@ -605,6 +605,23 @@ export type BackgroundRunNotice = {
const STEP_LIMIT_NOTICE =
"The assistant had to stop before finishing this answer. Too many steps in one turn. Send another message to continue.";
+const OUTPUT_LIMIT_NOTICE =
+ "The assistant had to stop before finishing this answer. The response hit the model's output limit. Send another message to continue.";
+
+/** Notices for SUCCEEDED runs cut short before a final answer, by error code. */
+const TRUNCATION_NOTICES: Readonly<Record<string, string>> = {
+ [InAppAgentRunErrorCode.STEP_LIMIT]: STEP_LIMIT_NOTICE,
+ [InAppAgentRunErrorCode.OUTPUT_LIMIT]: OUTPUT_LIMIT_NOTICE,
+};
+
+/** A SUCCEEDED run that was cut short before producing a final answer. */
+function isTruncatedRun(run: BackgroundExecutionRunView): boolean {
+ return (
+ run.status === InAppAgentRunStatus.SUCCEEDED &&
+ !!run.errorCode &&
+ run.errorCode in TRUNCATION_NOTICES
+ );
+}
export function getBackgroundRunNotice(
run: BackgroundExecutionRunView | null,
@@ -624,11 +641,11 @@ export function getBackgroundRunNotice(
};
}
- if (
- run.status === InAppAgentRunStatus.SUCCEEDED &&
- run.errorCode === InAppAgentRunErrorCode.STEP_LIMIT
- ) {
- return { text: STEP_LIMIT_NOTICE, tone: "warning" };
+ const truncationNotice = isTruncatedRun(run)
+ ? TRUNCATION_NOTICES[run.errorCode ?? ""]
+ : undefined;
+ if (truncationNotice !== undefined) {
+ return { text: truncationNotice, tone: "warning" };
}
return null;
@@ -643,11 +660,7 @@ export function getSettledActivityOutcome(
return "worked";
}
- if (
- (run.status === InAppAgentRunStatus.SUCCEEDED &&
- run.errorCode === InAppAgentRunErrorCode.STEP_LIMIT) ||
- run.status === InAppAgentRunStatus.CANCELLED
- ) {
+ if (isTruncatedRun(run) || run.status === InAppAgentRunStatus.CANCELLED) {
return "stopped";
}
diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts
index ac44defe69..38bbba58a6 100644
--- a/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts
+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts
@@ -55,7 +55,11 @@ type AgentScenario = (ctx: {
};
onEvent: (event: unknown) => Promise<void> | void;
onApprovedToolCallExecuted?: () => Promise<void> | void;
- onComplete: (outcome?: { truncatedByStepLimit?: boolean }) => Promise<void>;
+ onComplete: (outcome?: {
+ reachedStepLimit?: boolean;
+ truncatedByStepLimit?: boolean;
+ truncatedByOutputLimit?: boolean;
+ }) => Promise<void>;
onAbort: () => Promise<void>;
onError: (error: unknown) => Promise<void>;
onFinish: () => Promise<void>;
@@ -565,6 +569,27 @@ describe("executeInAppAgentRun", () => {
expect(finished.errorMessage).toMatch(/step limit/i);
});
+ it("records output_limit on SUCCEEDED when the last step ends with a length finish", async () => {
+ const { projectId, run } = await seedBackgroundRun();
+
+ scenarioRef.current = async ({ options }) => {
+ await options.onEvent(textChunk("Here is the beginning of a long"));
+ await options.onComplete({
+ reachedStepLimit: false,
+ truncatedByStepLimit: false,
+ truncatedByOutputLimit: true,
+ });
+ await options.onFinish();
+ };
+
+ await executeInAppAgentRun({ projectId, runId: run.id });
+
+ const finished = await getRun(projectId, run.id);
+ expect(finished.status).toBe("SUCCEEDED");
+ expect(finished.errorCode).toBe("output_limit");
+ expect(finished.errorMessage).toMatch(/output-token limit/i);
+ });
+
it("acknowledges duplicate delivery without executing (claim CAS returns no row)", async () => {
const { projectId, run } = await seedBackgroundRun({ status: "RUNNING" });
diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.ts
index a04a4a05a4..e867232af8 100644
--- a/worker/src/features/in-app-agent/executeInAppAgentRun.ts
+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.ts
@@ -524,18 +524,14 @@ export async function executeInAppAgentRun(params: {
if (outcome?.reachedStepLimit) {
recordIncrement("langfuse.in_app_agent.step_limit_reached", 1);
}
+ if (outcome?.truncatedByOutputLimit) {
+ recordIncrement("langfuse.in_app_agent.output_limit_reached", 1);
+ }
await flushPersistedRunEvents(
interruptRequest
? { status: InAppAgentRunStatus.AWAITING_APPROVAL }
- : outcome?.truncatedByStepLimit
- ? {
- status: InAppAgentRunStatus.SUCCEEDED,
- errorCode: InAppAgentRunErrorCode.STEP_LIMIT,
- errorMessage:
- "The run reached the step limit before a final answer",
- }
- : { status: InAppAgentRunStatus.SUCCEEDED },
+ : resolveCompletedRunFinish(outcome),
);
},
onAbort: async () => {
@@ -716,6 +712,28 @@ async function resolveUserProjectAccess(params: {
};
}
+function resolveCompletedRunFinish(outcome?: {
+ truncatedByStepLimit: boolean;
+ truncatedByOutputLimit: boolean;
+}): NonNullable<Parameters<typeof flushPendingRunEvents>[0]["finish"]> {
+ if (outcome?.truncatedByOutputLimit) {
+ return {
+ status: InAppAgentRunStatus.SUCCEEDED,
+ errorCode: InAppAgentRunErrorCode.OUTPUT_LIMIT,
+ errorMessage:
+ "The response hit the model's output-token limit before a final answer",
+ };
+ }
+ if (outcome?.truncatedByStepLimit) {
+ return {
+ status: InAppAgentRunStatus.SUCCEEDED,
+ errorCode: InAppAgentRunErrorCode.STEP_LIMIT,
+ errorMessage: "The run reached the step limit before a final answer",
+ };
+ }
+ return { status: InAppAgentRunStatus.SUCCEEDED };
+}
+
function findPersistedApprovalRequest(
events: readonly PersistedConversationEvent[],
request: Extract<InAppAgentRunRequest, { kind: "approvalDecision" }>,
diff --git a/worker/src/features/in-app-agent/runtime/agent.test.ts b/worker/src/features/in-app-agent/runtime/agent.test.ts
index 73f786cf7f..740361ac13 100644
--- a/worker/src/features/in-app-agent/runtime/agent.test.ts
+++ b/worker/src/features/in-app-agent/runtime/agent.test.ts
@@ -103,6 +103,8 @@ const readAgentInstructions = (agentConfig: MockedAgentConfig | undefined) => {
const adapterEvents = vi.hoisted(() => ({
items: [] as AgUiEvent[],
+ /** Runs after the mocked adapter emits `items`, before it completes. */
+ beforeComplete: undefined as (() => void) | undefined,
cleanup: vi.fn().mockResolvedValue(undefined),
inputs: [] as unknown[],
createScoreConfigExecute: vi.fn().mockResolvedValue({
@@ -263,6 +265,7 @@ vi.mock("@ag-ui/mastra", () => ({
for (const event of adapterEvents.items) {
subscriber.next(event);
}
+ adapterEvents.beforeComplete?.();
subscriber.complete();
return { unsubscribe: vi.fn() };
},
@@ -557,6 +560,7 @@ describe("createAgUiStream", () => {
beforeEach(() => {
vi.clearAllMocks();
bedrockMocks.streamParts = [];
+ adapterEvents.beforeComplete = undefined;
promptMocks.getPrompt.mockResolvedValue({
name: "in-app-agent-system-prompt",
version: 2,
@@ -567,6 +571,7 @@ describe("createAgUiStream", () => {
const initializeBasicTracedAgent = async (
runId: string,
model = testBedrockModel("test-model"),
+ onComplete?: (outcome?: unknown) => void,
) => {
const { createAgUiStream } = await import("./agent");
const input = {
@@ -602,6 +607,7 @@ describe("createAgUiStream", () => {
signal: new AbortController().signal,
options: {
model,
+ onComplete,
langfuseMcp: {
url: "https://example.com/api/public/mcp",
publicKey: "pk",
@@ -619,6 +625,17 @@ describe("createAgUiStream", () => {
await readStream(stream);
};
+ const completeIterationWith = (finishReason: string, iteration = 1) => {
+ adapterEvents.beforeComplete = () => {
+ getLastAgentConfig()?.defaultOptions?.onIterationComplete?.({
+ iteration,
+ maxIterations: IN_APP_AGENT_MAX_STEPS,
+ isFinal: true,
+ finishReason,
+ });
+ };
+ };
+
it("uses the shared Bedrock default-credential auth", async () => {
const { createAmazonBedrock } = await import("ai-sdk-amazon-bedrock-v4");
@@ -927,6 +944,62 @@ describe("createAgUiStream", () => {
);
});
+ it.each([
+ {
+ iteration: 1,
+ finishReason: "stop",
+ byStepLimit: false,
+ byOutputLimit: false,
+ },
+ {
+ iteration: 1,
+ finishReason: "length",
+ byStepLimit: false,
+ byOutputLimit: true,
+ },
+ {
+ iteration: IN_APP_AGENT_MAX_STEPS,
+ finishReason: "tool-calls",
+ byStepLimit: true,
+ byOutputLimit: false,
+ },
+ {
+ iteration: IN_APP_AGENT_MAX_STEPS,
+ finishReason: "length",
+ byStepLimit: false,
+ byOutputLimit: true,
+ },
+ ])(
+ "reports truncation for a $finishReason finish on step $iteration",
+ async ({ iteration, finishReason, byStepLimit, byOutputLimit }) => {
+ const onComplete = vi.fn();
+ completeIterationWith(finishReason, iteration);
+
+ await initializeBasicTracedAgent(
+ `run-finish-${finishReason}-${iteration}`,
+ undefined,
+ onComplete,
+ );
+
+ expect(onComplete).toHaveBeenCalledWith({
+ reachedStepLimit: false,
+ truncatedByStepLimit: byStepLimit,
+ truncatedByOutputLimit: byOutputLimit,
+ });
+ expect(instrumentationMocks.instrumentation.end).toHaveBeenCalledWith(
+ byStepLimit || byOutputLimit
+ ? {
+ result: {
+ truncatedByStepLimit: byStepLimit,
+ truncatedByOutputLimit: byOutputLimit,
+ finishReason,
+ },
+ }
+ : {},
+ );
+ },
+ );
+
it("serializes valid events including adapter snapshots and reasoning messages", async () => {
const { createAgUiStream } = await import("./agent");
const input = {
diff --git a/worker/src/features/in-app-agent/runtime/agent.ts b/worker/src/features/in-app-agent/runtime/agent.ts
index a947241414..d095f08dd6 100644
--- a/worker/src/features/in-app-agent/runtime/agent.ts
+++ b/worker/src/features/in-app-agent/runtime/agent.ts
@@ -265,6 +265,8 @@ type InAppAgentCompleteOutcome = {
/** The turn reached the step cap, whether or not wrap-up rescued it. */
reachedStepLimit: boolean;
truncatedByStepLimit: boolean;
+ /** The last model step ended with a `length` finish. */
+ truncatedByOutputLimit: boolean;
};
type StepLimitState = {
@@ -276,10 +278,15 @@ type StepLimitState = {
function isTruncatedByStepLimit(state: StepLimitState): boolean {
return (
state.iteration >= IN_APP_AGENT_MAX_STEPS &&
- state.lastFinishReason !== "stop"
+ state.lastFinishReason !== "stop" &&
+ state.lastFinishReason !== "length"
);
}
+function isTruncatedByOutputLimit(state: StepLimitState): boolean {
+ return state.lastFinishReason === "length";
+}
+
type CreateAgUiStreamOptions = {
onEvent?: (event: AgUiEvent) => void | Promise<void>;
onMcpToolCallCompleted?: (toolCall: CompletedInAppAgentMcpToolCall) => void;
@@ -897,12 +904,15 @@ export async function createAgUiStream(params: {
? async () => {
const truncatedByStepLimit =
isTruncatedByStepLimit(stepLimitState);
+ const truncatedByOutputLimit =
+ isTruncatedByOutputLimit(stepLimitState);
recordInstrumentation("end", (instrumentation) =>
instrumentation.end(
- truncatedByStepLimit
+ truncatedByStepLimit || truncatedByOutputLimit
? {
result: {
- truncatedByStepLimit: true,
+ truncatedByStepLimit,
+ truncatedByOutputLimit,
finishReason: stepLimitState.lastFinishReason,
},
}
@@ -915,6 +925,7 @@ export async function createAgUiStream(params: {
return params.options.onComplete?.({
reachedStepLimit: stepLimitState.wrapUp,
truncatedByStepLimit,
+ truncatedByOutputLimit,
});
}
: async () => {