🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(in-app-agent): surface output-token truncation instead of a blank turn (#16803)
David Traina8cad3e2f2265
diff --git a/packages/shared/src/features/inAppAgent/types.ts b/packages/shared/src/features/inAppAgent/types.tsindex 4275beddcd..d7e6fa2ab3 100644--- a/packages/shared/src/features/inAppAgent/types.ts+++ b/packages/shared/src/features/inAppAgent/types.ts@@ -66,6 +66,11 @@ export enum InAppAgentRunErrorCode {* configured wall but did not produce a final answer.*/STEP_LIMIT = "step_limit",+ /**+ * The last model step ended with a `length` finish: the response hit the+ * provider's output-token limit before a final answer.+ */+ OUTPUT_LIMIT = "output_limit",}/**diff --git a/web/src/features/in-app-agent/lib/backgroundExecutionSession.clienttest.ts b/web/src/features/in-app-agent/lib/backgroundExecutionSession.clienttest.tsindex b7b59d0495..b4c5f2061a 100644--- a/web/src/features/in-app-agent/lib/backgroundExecutionSession.clienttest.ts+++ b/web/src/features/in-app-agent/lib/backgroundExecutionSession.clienttest.ts@@ -18,6 +18,7 @@ import { BackgroundExecutionConnectionError } from "./backgroundExecutionErrors"import {BackgroundExecutionSessionController,getBackgroundRunNotice,+ getSettledActivityOutcome,type BackgroundExecutionView,} from "./backgroundExecutionSession";@@ -1405,6 +1406,41 @@ describe("getBackgroundRunNotice", () => {tone: "info",});});++ const truncatedRun = (errorCode: InAppAgentRunErrorCode) => ({+ id: "run-1",+ status: InAppAgentRunStatus.SUCCEEDED,+ errorCode,+ cancelRequested: false,+ });++ it.each([+ [+ InAppAgentRunErrorCode.STEP_LIMIT,+ "The assistant had to stop before finishing this answer. Too many steps in one turn. Send another message to continue.",+ ],+ [+ InAppAgentRunErrorCode.OUTPUT_LIMIT,+ "The assistant had to stop before finishing this answer. The response hit the model's output limit. Send another message to continue.",+ ],+ ] as const)("warns and marks SUCCEEDED %s as stopped", (errorCode, text) => {+ const run = truncatedRun(errorCode);++ expect(getBackgroundRunNotice(run)).toEqual({ text, tone: "warning" });+ expect(getSettledActivityOutcome(run)).toBe("stopped");+ });++ it("shows no notice and marks a plain SUCCEEDED run as worked", () => {+ const run = {+ id: "run-1",+ status: InAppAgentRunStatus.SUCCEEDED,+ errorCode: null,+ cancelRequested: false,+ };++ expect(getBackgroundRunNotice(run)).toBeNull();+ expect(getSettledActivityOutcome(run)).toBe("worked");+ });});function sseFrame(frame: Record<string, unknown>): string {diff --git a/web/src/features/in-app-agent/lib/backgroundExecutionSession.ts b/web/src/features/in-app-agent/lib/backgroundExecutionSession.tsindex d4ee98db7c..249dadcbb3 100644--- a/web/src/features/in-app-agent/lib/backgroundExecutionSession.ts+++ b/web/src/features/in-app-agent/lib/backgroundExecutionSession.ts@@ -605,6 +605,23 @@ export type BackgroundRunNotice = {const STEP_LIMIT_NOTICE ="The assistant had to stop before finishing this answer. Too many steps in one turn. Send another message to continue.";+const OUTPUT_LIMIT_NOTICE =+ "The assistant had to stop before finishing this answer. The response hit the model's output limit. Send another message to continue.";++/** Notices for SUCCEEDED runs cut short before a final answer, by error code. */+const TRUNCATION_NOTICES: Readonly<Record<string, string>> = {+ [InAppAgentRunErrorCode.STEP_LIMIT]: STEP_LIMIT_NOTICE,+ [InAppAgentRunErrorCode.OUTPUT_LIMIT]: OUTPUT_LIMIT_NOTICE,+};++/** A SUCCEEDED run that was cut short before producing a final answer. */+function isTruncatedRun(run: BackgroundExecutionRunView): boolean {+ return (+ run.status === InAppAgentRunStatus.SUCCEEDED &&+ !!run.errorCode &&+ run.errorCode in TRUNCATION_NOTICES+ );+}export function getBackgroundRunNotice(run: BackgroundExecutionRunView | null,@@ -624,11 +641,11 @@ export function getBackgroundRunNotice(};}- if (- run.status === InAppAgentRunStatus.SUCCEEDED &&- run.errorCode === InAppAgentRunErrorCode.STEP_LIMIT- ) {- return { text: STEP_LIMIT_NOTICE, tone: "warning" };+ const truncationNotice = isTruncatedRun(run)+ ? TRUNCATION_NOTICES[run.errorCode ?? ""]+ : undefined;+ if (truncationNotice !== undefined) {+ return { text: truncationNotice, tone: "warning" };}return null;@@ -643,11 +660,7 @@ export function getSettledActivityOutcome(return "worked";}- if (- (run.status === InAppAgentRunStatus.SUCCEEDED &&- run.errorCode === InAppAgentRunErrorCode.STEP_LIMIT) ||- run.status === InAppAgentRunStatus.CANCELLED- ) {+ if (isTruncatedRun(run) || run.status === InAppAgentRunStatus.CANCELLED) {return "stopped";}diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.test.tsindex ac44defe69..38bbba58a6 100644--- a/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts@@ -55,7 +55,11 @@ type AgentScenario = (ctx: {};onEvent: (event: unknown) => Promise<void> | void;onApprovedToolCallExecuted?: () => Promise<void> | void;- onComplete: (outcome?: { truncatedByStepLimit?: boolean }) => Promise<void>;+ onComplete: (outcome?: {+ reachedStepLimit?: boolean;+ truncatedByStepLimit?: boolean;+ truncatedByOutputLimit?: boolean;+ }) => Promise<void>;onAbort: () => Promise<void>;onError: (error: unknown) => Promise<void>;onFinish: () => Promise<void>;@@ -565,6 +569,27 @@ describe("executeInAppAgentRun", () => {expect(finished.errorMessage).toMatch(/step limit/i);});+ it("records output_limit on SUCCEEDED when the last step ends with a length finish", async () => {+ const { projectId, run } = await seedBackgroundRun();++ scenarioRef.current = async ({ options }) => {+ await options.onEvent(textChunk("Here is the beginning of a long"));+ await options.onComplete({+ reachedStepLimit: false,+ truncatedByStepLimit: false,+ truncatedByOutputLimit: true,+ });+ await options.onFinish();+ };++ await executeInAppAgentRun({ projectId, runId: run.id });++ const finished = await getRun(projectId, run.id);+ expect(finished.status).toBe("SUCCEEDED");+ expect(finished.errorCode).toBe("output_limit");+ expect(finished.errorMessage).toMatch(/output-token limit/i);+ });+it("acknowledges duplicate delivery without executing (claim CAS returns no row)", async () => {const { projectId, run } = await seedBackgroundRun({ status: "RUNNING" });diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.tsindex a04a4a05a4..e867232af8 100644--- a/worker/src/features/in-app-agent/executeInAppAgentRun.ts+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.ts@@ -524,18 +524,14 @@ export async function executeInAppAgentRun(params: {if (outcome?.reachedStepLimit) {recordIncrement("langfuse.in_app_agent.step_limit_reached", 1);}+ if (outcome?.truncatedByOutputLimit) {+ recordIncrement("langfuse.in_app_agent.output_limit_reached", 1);+ }await flushPersistedRunEvents(interruptRequest? { status: InAppAgentRunStatus.AWAITING_APPROVAL }- : outcome?.truncatedByStepLimit- ? {- status: InAppAgentRunStatus.SUCCEEDED,- errorCode: InAppAgentRunErrorCode.STEP_LIMIT,- errorMessage:- "The run reached the step limit before a final answer",- }- : { status: InAppAgentRunStatus.SUCCEEDED },+ : resolveCompletedRunFinish(outcome),);},onAbort: async () => {@@ -716,6 +712,28 @@ async function resolveUserProjectAccess(params: {};}+function resolveCompletedRunFinish(outcome?: {+ truncatedByStepLimit: boolean;+ truncatedByOutputLimit: boolean;+}): NonNullable<Parameters<typeof flushPendingRunEvents>[0]["finish"]> {+ if (outcome?.truncatedByOutputLimit) {+ return {+ status: InAppAgentRunStatus.SUCCEEDED,+ errorCode: InAppAgentRunErrorCode.OUTPUT_LIMIT,+ errorMessage:+ "The response hit the model's output-token limit before a final answer",+ };+ }+ if (outcome?.truncatedByStepLimit) {+ return {+ status: InAppAgentRunStatus.SUCCEEDED,+ errorCode: InAppAgentRunErrorCode.STEP_LIMIT,+ errorMessage: "The run reached the step limit before a final answer",+ };+ }+ return { status: InAppAgentRunStatus.SUCCEEDED };+}+function findPersistedApprovalRequest(events: readonly PersistedConversationEvent[],request: Extract<InAppAgentRunRequest, { kind: "approvalDecision" }>,diff --git a/worker/src/features/in-app-agent/runtime/agent.test.ts b/worker/src/features/in-app-agent/runtime/agent.test.tsindex 73f786cf7f..740361ac13 100644--- a/worker/src/features/in-app-agent/runtime/agent.test.ts+++ b/worker/src/features/in-app-agent/runtime/agent.test.ts@@ -103,6 +103,8 @@ const readAgentInstructions = (agentConfig: MockedAgentConfig | undefined) => {const adapterEvents = vi.hoisted(() => ({items: [] as AgUiEvent[],+ /** Runs after the mocked adapter emits `items`, before it completes. */+ beforeComplete: undefined as (() => void) | undefined,cleanup: vi.fn().mockResolvedValue(undefined),inputs: [] as unknown[],createScoreConfigExecute: vi.fn().mockResolvedValue({@@ -263,6 +265,7 @@ vi.mock("@ag-ui/mastra", () => ({for (const event of adapterEvents.items) {subscriber.next(event);}+ adapterEvents.beforeComplete?.();subscriber.complete();return { unsubscribe: vi.fn() };},@@ -557,6 +560,7 @@ describe("createAgUiStream", () => {beforeEach(() => {vi.clearAllMocks();bedrockMocks.streamParts = [];+ adapterEvents.beforeComplete = undefined;promptMocks.getPrompt.mockResolvedValue({name: "in-app-agent-system-prompt",version: 2,@@ -567,6 +571,7 @@ describe("createAgUiStream", () => {const initializeBasicTracedAgent = async (runId: string,model = testBedrockModel("test-model"),+ onComplete?: (outcome?: unknown) => void,) => {const { createAgUiStream } = await import("./agent");const input = {@@ -602,6 +607,7 @@ describe("createAgUiStream", () => {signal: new AbortController().signal,options: {model,+ onComplete,langfuseMcp: {url: "https://example.com/api/public/mcp",publicKey: "pk",@@ -619,6 +625,17 @@ describe("createAgUiStream", () => {await readStream(stream);};+ const completeIterationWith = (finishReason: string, iteration = 1) => {+ adapterEvents.beforeComplete = () => {+ getLastAgentConfig()?.defaultOptions?.onIterationComplete?.({+ iteration,+ maxIterations: IN_APP_AGENT_MAX_STEPS,+ isFinal: true,+ finishReason,+ });+ };+ };+it("uses the shared Bedrock default-credential auth", async () => {const { createAmazonBedrock } = await import("ai-sdk-amazon-bedrock-v4");@@ -927,6 +944,62 @@ describe("createAgUiStream", () => {);});+ it.each([+ {+ iteration: 1,+ finishReason: "stop",+ byStepLimit: false,+ byOutputLimit: false,+ },+ {+ iteration: 1,+ finishReason: "length",+ byStepLimit: false,+ byOutputLimit: true,+ },+ {+ iteration: IN_APP_AGENT_MAX_STEPS,+ finishReason: "tool-calls",+ byStepLimit: true,+ byOutputLimit: false,+ },+ {+ iteration: IN_APP_AGENT_MAX_STEPS,+ finishReason: "length",+ byStepLimit: false,+ byOutputLimit: true,+ },+ ])(+ "reports truncation for a $finishReason finish on step $iteration",+ async ({ iteration, finishReason, byStepLimit, byOutputLimit }) => {+ const onComplete = vi.fn();+ completeIterationWith(finishReason, iteration);++ await initializeBasicTracedAgent(+ `run-finish-${finishReason}-${iteration}`,+ undefined,+ onComplete,+ );++ expect(onComplete).toHaveBeenCalledWith({+ reachedStepLimit: false,+ truncatedByStepLimit: byStepLimit,+ truncatedByOutputLimit: byOutputLimit,+ });+ expect(instrumentationMocks.instrumentation.end).toHaveBeenCalledWith(+ byStepLimit || byOutputLimit+ ? {+ result: {+ truncatedByStepLimit: byStepLimit,+ truncatedByOutputLimit: byOutputLimit,+ finishReason,+ },+ }+ : {},+ );+ },+ );+it("serializes valid events including adapter snapshots and reasoning messages", async () => {const { createAgUiStream } = await import("./agent");const input = {diff --git a/worker/src/features/in-app-agent/runtime/agent.ts b/worker/src/features/in-app-agent/runtime/agent.tsindex a947241414..d095f08dd6 100644--- a/worker/src/features/in-app-agent/runtime/agent.ts+++ b/worker/src/features/in-app-agent/runtime/agent.ts@@ -265,6 +265,8 @@ type InAppAgentCompleteOutcome = {/** The turn reached the step cap, whether or not wrap-up rescued it. */reachedStepLimit: boolean;truncatedByStepLimit: boolean;+ /** The last model step ended with a `length` finish. */+ truncatedByOutputLimit: boolean;};type StepLimitState = {@@ -276,10 +278,15 @@ type StepLimitState = {function isTruncatedByStepLimit(state: StepLimitState): boolean {return (state.iteration >= IN_APP_AGENT_MAX_STEPS &&- state.lastFinishReason !== "stop"+ state.lastFinishReason !== "stop" &&+ state.lastFinishReason !== "length");}+function isTruncatedByOutputLimit(state: StepLimitState): boolean {+ return state.lastFinishReason === "length";+}+type CreateAgUiStreamOptions = {onEvent?: (event: AgUiEvent) => void | Promise<void>;onMcpToolCallCompleted?: (toolCall: CompletedInAppAgentMcpToolCall) => void;@@ -897,12 +904,15 @@ export async function createAgUiStream(params: {? async () => {const truncatedByStepLimit =isTruncatedByStepLimit(stepLimitState);+ const truncatedByOutputLimit =+ isTruncatedByOutputLimit(stepLimitState);recordInstrumentation("end", (instrumentation) =>instrumentation.end(- truncatedByStepLimit+ truncatedByStepLimit || truncatedByOutputLimit? {result: {- truncatedByStepLimit: true,+ truncatedByStepLimit,+ truncatedByOutputLimit,finishReason: stepLimitState.lastFinishReason,},}@@ -915,6 +925,7 @@ export async function createAgUiStream(params: {return params.options.onComplete?.({reachedStepLimit: stepLimitState.wrapUp,truncatedByStepLimit,+ truncatedByOutputLimit,});}: async () => {