🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(in-app-agent): instrumentation fixes - mark structured tool failures as ERROR (#16022)
Lysander Kiesel08d21a8bac82
diff --git a/packages/shared/src/in-app-agent/server/agent.ts b/packages/shared/src/in-app-agent/server/agent.tsindex 06d0b3f087..a7e9ed8e21 100644--- a/packages/shared/src/in-app-agent/server/agent.ts+++ b/packages/shared/src/in-app-agent/server/agent.ts@@ -22,6 +22,7 @@ import type {InAppAgentTracingConfig,} from "./instrumentation";import { createInAppAgentInstrumentation } from "./instrumentation";+import { getToolFailureMessage } from "./toolErrors";import {createSandboxTools,createRedirectActionTool,@@ -983,11 +984,10 @@ type MastraApprovalStreamChunk = {// those tools and the bridge has no case for it, so approvals would never// surface as on_interrupt events. Map approvals onto the suspend protocol.// Non-background tool-error chunks are likewise swallowed by the bridge, so-// convert them to tool results carrying the error message as content. Note:-// the bridge emits TOOL_CALL_RESULT without a top-level `error` field, so the-// failure renders with the error message in the result body but a "succeeded"-// status; the model is unaffected (Mastra's loop feeds it the real error).-// Status fidelity is tracked as a follow-up.+// convert them to tool results carrying a structured failure payload. The+// bridge still emits TOOL_CALL_RESULT without a top-level `error` field;+// normalizeAdapterEvent stamps that field from the structured payload so+// instrumentation and persistence see an explicit failure.export function patchMastraApprovalChunks(adapter: MastraAgent) {const patchableAdapter = adapter as unknown as PatchableMastraAgent;const createChunkProcessor = patchableAdapter.createChunkProcessor;@@ -1062,8 +1062,12 @@ export function patchMastraApprovalChunks(adapter: MastraAgent) {isError: true,// Raw object, not pre-stringified: the bridge JSON-stringifies// payload.result into the event content, so a string here would- // double-encode.- result: { error: getToolErrorMessage(mastraChunk) },+ // double-encode. Keep the same {error:true,message} shape used+ // by schema-validation failures so detection stays structured.+ result: {+ error: true,+ message: getToolErrorMessage(mastraChunk),+ },},});}@@ -1160,6 +1164,19 @@ function normalizeAdapterEvent(];}+ if (event.type === EventType.TOOL_CALL_RESULT) {+ // Never overwrite an existing top-level error (e.g. JSON-encoded approval+ // rejections). Only stamp when the bridge omitted the field entirely.+ if (typeof event.error === "string" && event.error.trim()) {+ return [event];+ }++ const failureMessage = getToolFailureMessage(undefined, event.content);+ if (failureMessage) {+ return [{ ...event, error: failureMessage }];+ }+ }+return [event];}diff --git a/packages/shared/src/in-app-agent/server/instrumentation.ts b/packages/shared/src/in-app-agent/server/instrumentation.tsindex f047bd6576..cec9a4d940 100644--- a/packages/shared/src/in-app-agent/server/instrumentation.ts+++ b/packages/shared/src/in-app-agent/server/instrumentation.ts@@ -7,6 +7,12 @@ import {} from "../constants";import type { AgUiEvent, AgUiMessage, AgUiRunAgentInput } from "../schema";import { compactTextMessageChunks } from "./eventCompaction";+import {+ getToolFailureMessage,+ isRecord,+ normalizeToolOutput,+ parseJsonOrString,+} from "./toolErrors";import type { InAppAgentUserAccess } from "./tools";import { assertUnreachable } from "../../utils/typeChecks";@@ -91,6 +97,15 @@ type ToolObservationBody = {metadata?: Record<string, unknown>;};type ToolCallApprovalStatus = "approved" | "rejected";+type AgentRunToolSpan = {+ name: string;+ startTime: Date;+ args: string;+ argsComplete: boolean;+ output?: unknown;+ failureMessage?: string;+ parentMessageId?: string;+};export function createInAppAgentInstrumentation({input,@@ -128,17 +143,7 @@ export class InAppAgentInstrumentation {private readonly agentRun: InAppAgentGeneration;private agentRunInput: unknown;private readonly prompt?: InAppAgentPromptMetadata;- private readonly toolSpans = new Map<- string,- {- name: string;- startTime: Date;- args: string;- argsComplete: boolean;- output?: unknown;- parentMessageId?: string;- }- >();+ private readonly toolSpans = new Map<string, AgentRunToolSpan>();private readonly toolCallApprovals = new Map<string,ToolCallApprovalStatus@@ -541,6 +546,7 @@ export class InAppAgentInstrumentation {const tool = this.toolSpans.get(event.toolCallId);if (tool) {tool.output = event.content;+ tool.failureMessage = getToolFailureMessage(event.error, event.content);this.endToolSpanIfComplete(event.toolCallId, tool);}}@@ -559,17 +565,7 @@ export class InAppAgentInstrumentation {this.endToolSpanIfComplete(event.toolCallId, tool);}- private endToolSpanIfComplete(- toolCallId: string,- tool: {- name: string;- startTime: Date;- args: string;- argsComplete: boolean;- output?: unknown;- parentMessageId?: string;- },- ) {+ private endToolSpanIfComplete(toolCallId: string, tool: AgentRunToolSpan) {if (!tool.argsComplete || tool.output === undefined) {return;}@@ -580,14 +576,7 @@ export class InAppAgentInstrumentation {private createToolObservation(toolCallId: string,- tool: {- name: string;- startTime: Date;- args: string;- argsComplete: boolean;- output?: unknown;- parentMessageId?: string;- },+ tool: AgentRunToolSpan,options?: {metadata?: Record<string, unknown>;statusMessage?: string;@@ -596,7 +585,10 @@ export class InAppAgentInstrumentation {const input = parseJsonOrUndefined(tool.args);const output =tool.output === undefined ? undefined : normalizeToolOutput(tool.output);- const isError = options?.statusMessage !== undefined || isToolError(output);+ const failureMessage =+ options?.statusMessage ??+ tool.failureMessage ??+ getToolFailureMessage(undefined, output);const toolCallApproval = this.toolCallApprovals.get(toolCallId);const body: ToolObservationBody = {id: toolCallId,@@ -608,9 +600,8 @@ export class InAppAgentInstrumentation {completionStartTime: tool.startTime,input,output,- ...(isError ? { level: "ERROR" } : {}),- ...(options?.statusMessage- ? { statusMessage: options.statusMessage }+ ...(failureMessage+ ? { level: "ERROR", statusMessage: failureMessage }: {}),metadata: {...(options?.metadata ?? {}),@@ -973,32 +964,6 @@ function parseContextValue(description: string, value: string): unknown {return parsed;}-function normalizeToolOutput(output: unknown): unknown {- const parsedOutput =- typeof output === "string" ? parseJsonOrString(output) : output;-- if (!isRecord(parsedOutput) || !Array.isArray(parsedOutput.content)) {- return parsedOutput;- }-- const firstContent: unknown = parsedOutput.content[0];-- if (- parsedOutput.content.length !== 1 ||- !isRecord(firstContent) ||- firstContent.type !== "text" ||- typeof firstContent.text !== "string"- ) {- return parsedOutput;- }-- return parseJsonOrString(firstContent.text);-}--function isToolError(output: unknown): boolean {- return isRecord(output) && output.error === true;-}-function getFiniteNumber(value: unknown): number | undefined {return typeof value === "number" && Number.isFinite(value)? value@@ -1081,22 +1046,6 @@ function parseJsonOrUndefined(value: string): unknown {return parseJsonOrString(value);}-function parseJsonOrString(value: string): unknown {- if (!value) {- return value;- }-- try {- return JSON.parse(value);- } catch {- return value;- }-}--function isRecord(value: unknown): value is Record<string, unknown> {- return typeof value === "object" && value !== null && !Array.isArray(value);-}-function getStringValue(value: unknown): string | undefined {return typeof value === "string" && value.trim() ? value : undefined;}diff --git a/packages/shared/src/in-app-agent/server/toolErrors.ts b/packages/shared/src/in-app-agent/server/toolErrors.tsnew file mode 100644index 0000000000..fd965aa616--- /dev/null+++ b/packages/shared/src/in-app-agent/server/toolErrors.ts@@ -0,0 +1,80 @@+/**+ * Structured tool-failure extraction for in-app-agent instrumentation.+ *+ * Only trusts explicit AG-UI `event.error` and boolean failure markers+ * (`error: true` / `isError: true`). Does not infer failure from string-valued+ * `error` fields or from message text such as "MCP error …".+ */+export function getToolFailureMessage(+ eventError: unknown,+ output: unknown,+): string | undefined {+ const explicitError = getStringValue(eventError);+ if (explicitError) {+ // Manual approval rejections encode `{code,message}` as JSON in event.error.+ // Prefer the human-readable message for statusMessage while leaving the+ // raw event.error untouched for callers that need the structured code.+ const parsed = parseJsonOrString(explicitError);+ if (isRecord(parsed)) {+ const message = getStringValue(parsed.message);+ if (message) {+ return message;+ }+ }++ return explicitError;+ }++ const normalizedOutput = normalizeToolOutput(output);+ if (!isRecord(normalizedOutput)) {+ return undefined;+ }++ if (normalizedOutput.error === true || normalizedOutput.isError === true) {+ return getStringValue(normalizedOutput.message) ?? "Tool returned an error";+ }++ return undefined;+}++export function normalizeToolOutput(output: unknown): unknown {+ const parsedOutput =+ typeof output === "string" ? parseJsonOrString(output) : output;++ if (!isRecord(parsedOutput) || !Array.isArray(parsedOutput.content)) {+ return parsedOutput;+ }++ const firstContent: unknown = parsedOutput.content[0];++ if (+ parsedOutput.content.length !== 1 ||+ !isRecord(firstContent) ||+ firstContent.type !== "text" ||+ typeof firstContent.text !== "string"+ ) {+ return parsedOutput;+ }++ return parseJsonOrString(firstContent.text);+}++export function parseJsonOrString(value: string): unknown {+ if (!value) {+ return value;+ }++ try {+ return JSON.parse(value);+ } catch {+ return value;+ }+}++export function isRecord(value: unknown): value is Record<string, unknown> {+ return typeof value === "object" && value !== null && !Array.isArray(value);+}++function getStringValue(value: unknown): string | undefined {+ return typeof value === "string" && value.trim() ? value.trim() : undefined;+}diff --git a/web/src/__tests__/server/in-app-agent-stream.servertest.ts b/web/src/__tests__/server/in-app-agent-stream.servertest.tsindex 4ac436d39f..185d6c16b1 100644--- a/web/src/__tests__/server/in-app-agent-stream.servertest.ts+++ b/web/src/__tests__/server/in-app-agent-stream.servertest.ts@@ -413,7 +413,10 @@ describe("patchMastraApprovalChunks", () => {toolName: "bash",args: { command: "date" },isError: true,- result: { error: "Error: Region is missing" },+ result: {+ error: true,+ message: "Error: Region is missing",+ },},},]);@@ -910,6 +913,104 @@ describe("createAgUiStream", () => {expect(agentConfig?.defaultOptions).not.toHaveProperty("providerOptions");});+ it("stamps top-level error on TOOL_CALL_RESULT from structured failure payloads", async () => {+ const { createAgUiStream } =+ await import("@langfuse/shared/in-app-agent/server/agent");+ const input = {+ threadId: "conversation-1",+ runId: "run-1",+ messages: [+ { id: "user-message-1", role: "user" as const, content: "hello" },+ ],+ tools: [],+ context: [],+ state: {+ type: "existingConversation" as const,+ projectId: "project-1",+ conversationId: "conversation-1",+ },+ forwardedProps: {},+ };+ const failureMessage = "MCP error -32602: invalid score config";+ const structuredFailure = {+ error: true,+ message: failureMessage,+ };+ const persistedEvents: AgUiEvent[] = [];+ const langfuseClient = {+ getPrompt: promptMocks.getPrompt,+ } as unknown as Langfuse;++ adapterEvents.items = [+ {+ type: EventType.RUN_STARTED,+ threadId: input.threadId,+ runId: input.runId,+ },+ {+ type: EventType.TOOL_CALL_START,+ toolCallId: "tool-call-1",+ toolCallName: "createScoreConfig",+ },+ {+ type: EventType.TOOL_CALL_ARGS,+ toolCallId: "tool-call-1",+ delta: "{}",+ },+ {+ type: EventType.TOOL_CALL_END,+ toolCallId: "tool-call-1",+ },+ {+ type: EventType.TOOL_CALL_RESULT,+ messageId: "tool-result-1",+ toolCallId: "tool-call-1",+ content: JSON.stringify(structuredFailure),+ role: "tool",+ },+ {+ type: EventType.RUN_FINISHED,+ threadId: input.threadId,+ runId: input.runId,+ },+ ];++ const stream = await createAgUiStream({+ input,+ signal: new AbortController().signal,+ options: {+ onEvent: (event) => {+ persistedEvents.push(event);+ },+ awsBedrock: { modelId: "test-model" },+ langfuseMcp: {+ url: "https://example.com/api/public/mcp",+ publicKey: "pk",+ secretKey: "sk",+ toolPolicy: defaultInAppAgentToolPolicy,+ runOverride: "run-override",+ },+ redirectAction: {+ projectId: "project-1",+ isV4Enabled: false,+ },+ langfuseClient,+ useLocalPrompt: false,+ },+ });++ await readStream(stream);++ expect(persistedEvents).toContainEqual({+ type: EventType.TOOL_CALL_RESULT,+ messageId: "tool-result-1",+ toolCallId: "tool-call-1",+ content: JSON.stringify(structuredFailure),+ role: "tool",+ error: failureMessage,+ });+ });+it("stops gating a tool the user approved for the conversation", async () => {const { createAgUiStream } =await import("@langfuse/shared/in-app-agent/server/agent");diff --git a/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts b/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.tsindex d226e11a53..4f67154543 100644--- a/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts+++ b/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts@@ -408,7 +408,7 @@ describe("InAppAgentInstrumentation", () => {);});- it("marks tool observations with error outputs as error level", () => {+ it("marks tool observations with structured error outputs as error level", () => {const instrumentation = createInstrumentation();instrumentation.recordEvents([@@ -453,11 +453,142 @@ describe("InAppAgentInstrumentation", () => {message: "Tool input validation failed",},level: "ERROR",+ statusMessage: "Tool input validation failed",}),);- expect(toolCreateBody).not.toHaveProperty("statusMessage");});+ it.each([+ {+ name: "top-level event errors",+ content: "Tool execution failed",+ error: "Tool execution failed",+ expectedOutput: "Tool execution failed",+ expectedMessage: "Tool execution failed",+ },+ {+ name: "boolean isError payloads",+ content: JSON.stringify({+ isError: true,+ message: "Tool returned an error",+ }),+ expectedOutput: {+ isError: true,+ message: "Tool returned an error",+ },+ expectedMessage: "Tool returned an error",+ },+ {+ name: "JSON-encoded approval rejection errors",+ content: "Tool call was not approved by the user.",+ error: JSON.stringify({+ code: "tool_call_rejected",+ message: "Tool call was not approved by the user.",+ }),+ expectedOutput: "Tool call was not approved by the user.",+ expectedMessage: "Tool call was not approved by the user.",+ },+ ])(+ "marks $name as failed with statusMessage",+ ({ content, error, expectedOutput, expectedMessage }) => {+ const instrumentation = createInstrumentation();++ instrumentation.recordEvents([+ {+ type: EventType.TOOL_CALL_START,+ toolCallId: "tool-1",+ toolCallName: "createScoreConfig",+ },+ {+ type: EventType.TOOL_CALL_ARGS,+ toolCallId: "tool-1",+ delta: "{}",+ },+ {+ type: EventType.TOOL_CALL_END,+ toolCallId: "tool-1",+ },+ {+ type: EventType.TOOL_CALL_RESULT,+ toolCallId: "tool-1",+ content,+ ...(error ? { error } : {}),+ },+ ]);++ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "tool-create",+ expect.objectContaining({+ output: expectedOutput,+ level: "ERROR",+ statusMessage: expectedMessage,+ }),+ );+ },+ );++ it.each([+ {+ name: "benign content mentioning error",+ content: JSON.stringify({+ message: "This record describes an error handling policy",+ }),+ expectedOutput: {+ message: "This record describes an error handling policy",+ },+ },+ {+ name: "string-valued error fields",+ content: JSON.stringify({+ error: "McpError: MCP error -32602: invalid score config",+ }),+ expectedOutput: {+ error: "McpError: MCP error -32602: invalid score config",+ },+ },+ {+ name: "bare MCP error strings",+ content: "MCP error -32600: Invalid Request",+ expectedOutput: "MCP error -32600: Invalid Request",+ },+ ])(+ "does not classify $name as a structured tool failure",+ ({ content, expectedOutput }) => {+ const instrumentation = createInstrumentation();++ instrumentation.recordEvents([+ {+ type: EventType.TOOL_CALL_START,+ toolCallId: "tool-1",+ toolCallName: "search",+ },+ {+ type: EventType.TOOL_CALL_ARGS,+ toolCallId: "tool-1",+ delta: "{}",+ },+ {+ type: EventType.TOOL_CALL_END,+ toolCallId: "tool-1",+ },+ {+ type: EventType.TOOL_CALL_RESULT,+ toolCallId: "tool-1",+ content,+ },+ ]);++ const toolCreateBody = mocks.handler.langfuse.enqueue.mock.calls[0]?.[1];+ expect(toolCreateBody).toEqual(+ expect.objectContaining({+ output: expectedOutput,+ }),+ );+ expect(toolCreateBody).not.toHaveProperty("level");+ expect(toolCreateBody).not.toHaveProperty("statusMessage");+ },+ );+it("sets static prompt metadata on the trace and agent generation", () => {const instrumentation = createInstrumentation(undefined, {name: "in-app-agent-system-prompt",