🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(in-app-agent): rewrite step-limit wrap-up so Bedrock never sees assistant prefill (#16424)
Lysander Kiesel416e3829d2a5
diff --git a/worker/src/features/in-app-agent/runtime/agent.test.ts b/worker/src/features/in-app-agent/runtime/agent.test.tsindex 442ebcdab3..3c6c135ca8 100644--- a/worker/src/features/in-app-agent/runtime/agent.test.ts+++ b/worker/src/features/in-app-agent/runtime/agent.test.ts@@ -59,6 +59,13 @@ type MockedAgentConfig = {stream: ReadableStream<unknown>;}>;};+ inputProcessors?: Array<{+ id: string;+ processInputStep?: (args: {+ stepNumber: number;+ sendSignal?: (signal: unknown) => Promise<unknown>;+ }) => Promise<unknown>;+ }>;defaultOptions?: {onIterationComplete?: (context: {iteration: number;@@ -632,28 +639,47 @@ describe("createAgUiStream", () => {).toHaveBeenNthCalledWith(2, finishPart);});- it("injects wrap-up instructions after the penultimate Mastra iteration", async () => {+ it("sends wrap-up as a last-step signal instead of assistant feedback", async () => {await initializeBasicTracedAgent("run-step-limit-wrap-up");const agentConfig = getLastAgentConfig();- const onIterationComplete =- agentConfig?.defaultOptions?.onIterationComplete;- expect(onIterationComplete).toEqual(expect.any(Function));-+ const processor = agentConfig?.inputProcessors?.find(+ (item) => item.id === "ensure-final-response",+ );+ expect(processor?.processInputStep).toEqual(expect.any(Function));expect(readAgentInstructions(agentConfig)).not.toContain("Do not call any more tools",);+ expect(readAgentInstructions(agentConfig)).not.toContain("system-reminder");++ const sendSignal = vi.fn();+ await processor?.processInputStep?.({+ stepNumber: IN_APP_AGENT_MAX_STEPS - 2,+ sendSignal,+ });+ expect(sendSignal).not.toHaveBeenCalled();- const wrapUp = await onIterationComplete?.({+ await processor?.processInputStep?.({+ stepNumber: IN_APP_AGENT_MAX_STEPS - 1,+ sendSignal,+ });+ expect(sendSignal).toHaveBeenCalledWith({+ type: "reactive",+ tagName: "step_limit_wrap_up",+ contents: expect.stringContaining("Do not call any more tools"),+ attributes: {+ reason: "max-steps-reached",+ step: IN_APP_AGENT_MAX_STEPS,+ },+ });++ const wrapUp = await agentConfig?.defaultOptions?.onIterationComplete?.({iteration: IN_APP_AGENT_MAX_STEPS - 1,maxIterations: IN_APP_AGENT_MAX_STEPS,isFinal: false,finishReason: "tool-calls",});-- expect(wrapUp).toEqual({- feedback: expect.stringContaining("<step_limit_wrap_up>"),- });- expect(readAgentInstructions(agentConfig)).toContain(+ expect(wrapUp).toBeUndefined();+ expect(readAgentInstructions(agentConfig)).not.toContain("Do not call any more tools",);});diff --git a/worker/src/features/in-app-agent/runtime/agent.ts b/worker/src/features/in-app-agent/runtime/agent.tsindex 9ffdea038b..2f77330389 100644--- a/worker/src/features/in-app-agent/runtime/agent.ts+++ b/worker/src/features/in-app-agent/runtime/agent.ts@@ -3,6 +3,7 @@ import { MastraAgent } from "@ag-ui/mastra";import { IN_APP_AGENT_SYSTEM_PROMPT_TEMPLATE } from "@langfuse/shared/in-app-agent/server/systemPrompt";import { createAmazonBedrock } from "ai-sdk-amazon-bedrock-v4";import { Agent } from "@mastra/core/agent";+import type { ProcessInputStepArgs, Processor } from "@mastra/core/processors";import { MCPClient } from "@mastra/mcp";import type { Langfuse } from "langfuse";@@ -154,9 +155,34 @@ The sandbox workspace from earlier turns in this conversation expired and has be- Do not assume a path exists because you created it earlier in this conversation. Read it first, and recreate what you still need.</sandbox_workspace_reset>`;-const STEP_LIMIT_WRAP_UP_INSTRUCTION = `<step_limit_wrap_up>-This is your final step. Do not call any more tools. Summarize what you have found and give the user a complete final answer now.-</step_limit_wrap_up>`;+const STEP_LIMIT_WRAP_UP_INSTRUCTION =+ "This is your final step. Do not call any more tools. Summarize what you have found and give the user a complete final answer now.";++class EnsureFinalResponseProcessor implements Processor {+ readonly id = "ensure-final-response";++ constructor(+ private readonly maxSteps: number,+ private readonly onLastStep?: () => void,+ ) {}++ async processInputStep({ stepNumber, sendSignal }: ProcessInputStepArgs) {+ if (stepNumber !== this.maxSteps - 1) {+ return;+ }++ this.onLastStep?.();+ await sendSignal?.({+ type: "reactive",+ tagName: "step_limit_wrap_up",+ contents: STEP_LIMIT_WRAP_UP_INSTRUCTION,+ attributes: {+ reason: "max-steps-reached",+ step: stepNumber + 1,+ },+ });+ }+}export type InAppAgentCompleteOutcome = {/** The turn reached the step cap, whether or not wrap-up rescued it. */@@ -1125,40 +1151,29 @@ async function createMastraAdapter(params: {id: "langfuse-in-app-assistant",name: ASSISTANT_TITLE,instructions: () =>- [- params.instructions,- developerGuidance,- params.stepLimitState.wrapUp- ? STEP_LIMIT_WRAP_UP_INSTRUCTION- : undefined,- ]- .filter(Boolean)- .join("\n\n"),+ [params.instructions, developerGuidance].filter(Boolean).join("\n\n"),model,skills: LANGFUSE_IN_APP_AGENT_SKILLS,tools,maxRetries: 2,+ inputProcessors: [+ new EnsureFinalResponseProcessor(IN_APP_AGENT_MAX_STEPS, () => {+ params.stepLimitState.wrapUp = true;+ logger.info("In-app agent step-limit wrap-up injected", {+ runId: params.input.runId,+ threadId: params.input.threadId,+ maxSteps: IN_APP_AGENT_MAX_STEPS,+ });+ }),+ ],defaultOptions: {abortSignal: params.signal,maxSteps: IN_APP_AGENT_MAX_STEPS,// Mastra's logical step counter — not provider doStream/finish parts,// which retry and inflate independently of maxSteps.- onIterationComplete: ({- iteration,- maxIterations,- isFinal,- finishReason,- }) => {+ onIterationComplete: ({ iteration, finishReason }) => {params.stepLimitState.iteration = iteration;params.stepLimitState.lastFinishReason = finishReason;- if (- iteration === (maxIterations ?? IN_APP_AGENT_MAX_STEPS) - 1 &&- !isFinal- ) {- params.stepLimitState.wrapUp = true;- // instructions() is snapshotted at stream start; feedback reaches the next call.- return { feedback: STEP_LIMIT_WRAP_UP_INSTRUCTION };- }},...(params.options.sandboxWorkspaceWasReset? { system: SANDBOX_WORKSPACE_RESET_INSTRUCTION }