Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(in-app-agent): instrumentation fixes - mark structured tool failures as ERROR (#16022)

Lysander Kiesel08d21a8bac82
diff --git a/packages/shared/src/in-app-agent/server/agent.ts b/packages/shared/src/in-app-agent/server/agent.ts
index 06d0b3f087..a7e9ed8e21 100644
--- a/packages/shared/src/in-app-agent/server/agent.ts
+++ b/packages/shared/src/in-app-agent/server/agent.ts
@@ -22,6 +22,7 @@ import type {
InAppAgentTracingConfig,
} from "./instrumentation";
import { createInAppAgentInstrumentation } from "./instrumentation";
+import { getToolFailureMessage } from "./toolErrors";
import {
createSandboxTools,
createRedirectActionTool,
@@ -983,11 +984,10 @@ type MastraApprovalStreamChunk = {
// those tools and the bridge has no case for it, so approvals would never
// surface as on_interrupt events. Map approvals onto the suspend protocol.
// Non-background tool-error chunks are likewise swallowed by the bridge, so
-// convert them to tool results carrying the error message as content. Note:
-// the bridge emits TOOL_CALL_RESULT without a top-level `error` field, so the
-// failure renders with the error message in the result body but a "succeeded"
-// status; the model is unaffected (Mastra's loop feeds it the real error).
-// Status fidelity is tracked as a follow-up.
+// convert them to tool results carrying a structured failure payload. The
+// bridge still emits TOOL_CALL_RESULT without a top-level `error` field;
+// normalizeAdapterEvent stamps that field from the structured payload so
+// instrumentation and persistence see an explicit failure.
export function patchMastraApprovalChunks(adapter: MastraAgent) {
const patchableAdapter = adapter as unknown as PatchableMastraAgent;
const createChunkProcessor = patchableAdapter.createChunkProcessor;
@@ -1062,8 +1062,12 @@ export function patchMastraApprovalChunks(adapter: MastraAgent) {
isError: true,
// Raw object, not pre-stringified: the bridge JSON-stringifies
// payload.result into the event content, so a string here would
- // double-encode.
- result: { error: getToolErrorMessage(mastraChunk) },
+ // double-encode. Keep the same {error:true,message} shape used
+ // by schema-validation failures so detection stays structured.
+ result: {
+ error: true,
+ message: getToolErrorMessage(mastraChunk),
+ },
},
});
}
@@ -1160,6 +1164,19 @@ function normalizeAdapterEvent(
];
}
+ if (event.type === EventType.TOOL_CALL_RESULT) {
+ // Never overwrite an existing top-level error (e.g. JSON-encoded approval
+ // rejections). Only stamp when the bridge omitted the field entirely.
+ if (typeof event.error === "string" && event.error.trim()) {
+ return [event];
+ }
+
+ const failureMessage = getToolFailureMessage(undefined, event.content);
+ if (failureMessage) {
+ return [{ ...event, error: failureMessage }];
+ }
+ }
+
return [event];
}
diff --git a/packages/shared/src/in-app-agent/server/instrumentation.ts b/packages/shared/src/in-app-agent/server/instrumentation.ts
index f047bd6576..cec9a4d940 100644
--- a/packages/shared/src/in-app-agent/server/instrumentation.ts
+++ b/packages/shared/src/in-app-agent/server/instrumentation.ts
@@ -7,6 +7,12 @@ import {
} from "../constants";
import type { AgUiEvent, AgUiMessage, AgUiRunAgentInput } from "../schema";
import { compactTextMessageChunks } from "./eventCompaction";
+import {
+ getToolFailureMessage,
+ isRecord,
+ normalizeToolOutput,
+ parseJsonOrString,
+} from "./toolErrors";
import type { InAppAgentUserAccess } from "./tools";
import { assertUnreachable } from "../../utils/typeChecks";
@@ -91,6 +97,15 @@ type ToolObservationBody = {
metadata?: Record<string, unknown>;
};
type ToolCallApprovalStatus = "approved" | "rejected";
+type AgentRunToolSpan = {
+ name: string;
+ startTime: Date;
+ args: string;
+ argsComplete: boolean;
+ output?: unknown;
+ failureMessage?: string;
+ parentMessageId?: string;
+};
export function createInAppAgentInstrumentation({
input,
@@ -128,17 +143,7 @@ export class InAppAgentInstrumentation {
private readonly agentRun: InAppAgentGeneration;
private agentRunInput: unknown;
private readonly prompt?: InAppAgentPromptMetadata;
- private readonly toolSpans = new Map<
- string,
- {
- name: string;
- startTime: Date;
- args: string;
- argsComplete: boolean;
- output?: unknown;
- parentMessageId?: string;
- }
- >();
+ private readonly toolSpans = new Map<string, AgentRunToolSpan>();
private readonly toolCallApprovals = new Map<
string,
ToolCallApprovalStatus
@@ -541,6 +546,7 @@ export class InAppAgentInstrumentation {
const tool = this.toolSpans.get(event.toolCallId);
if (tool) {
tool.output = event.content;
+ tool.failureMessage = getToolFailureMessage(event.error, event.content);
this.endToolSpanIfComplete(event.toolCallId, tool);
}
}
@@ -559,17 +565,7 @@ export class InAppAgentInstrumentation {
this.endToolSpanIfComplete(event.toolCallId, tool);
}
- private endToolSpanIfComplete(
- toolCallId: string,
- tool: {
- name: string;
- startTime: Date;
- args: string;
- argsComplete: boolean;
- output?: unknown;
- parentMessageId?: string;
- },
- ) {
+ private endToolSpanIfComplete(toolCallId: string, tool: AgentRunToolSpan) {
if (!tool.argsComplete || tool.output === undefined) {
return;
}
@@ -580,14 +576,7 @@ export class InAppAgentInstrumentation {
private createToolObservation(
toolCallId: string,
- tool: {
- name: string;
- startTime: Date;
- args: string;
- argsComplete: boolean;
- output?: unknown;
- parentMessageId?: string;
- },
+ tool: AgentRunToolSpan,
options?: {
metadata?: Record<string, unknown>;
statusMessage?: string;
@@ -596,7 +585,10 @@ export class InAppAgentInstrumentation {
const input = parseJsonOrUndefined(tool.args);
const output =
tool.output === undefined ? undefined : normalizeToolOutput(tool.output);
- const isError = options?.statusMessage !== undefined || isToolError(output);
+ const failureMessage =
+ options?.statusMessage ??
+ tool.failureMessage ??
+ getToolFailureMessage(undefined, output);
const toolCallApproval = this.toolCallApprovals.get(toolCallId);
const body: ToolObservationBody = {
id: toolCallId,
@@ -608,9 +600,8 @@ export class InAppAgentInstrumentation {
completionStartTime: tool.startTime,
input,
output,
- ...(isError ? { level: "ERROR" } : {}),
- ...(options?.statusMessage
- ? { statusMessage: options.statusMessage }
+ ...(failureMessage
+ ? { level: "ERROR", statusMessage: failureMessage }
: {}),
metadata: {
...(options?.metadata ?? {}),
@@ -973,32 +964,6 @@ function parseContextValue(description: string, value: string): unknown {
return parsed;
}
-function normalizeToolOutput(output: unknown): unknown {
- const parsedOutput =
- typeof output === "string" ? parseJsonOrString(output) : output;
-
- if (!isRecord(parsedOutput) || !Array.isArray(parsedOutput.content)) {
- return parsedOutput;
- }
-
- const firstContent: unknown = parsedOutput.content[0];
-
- if (
- parsedOutput.content.length !== 1 ||
- !isRecord(firstContent) ||
- firstContent.type !== "text" ||
- typeof firstContent.text !== "string"
- ) {
- return parsedOutput;
- }
-
- return parseJsonOrString(firstContent.text);
-}
-
-function isToolError(output: unknown): boolean {
- return isRecord(output) && output.error === true;
-}
-
function getFiniteNumber(value: unknown): number | undefined {
return typeof value === "number" && Number.isFinite(value)
? value
@@ -1081,22 +1046,6 @@ function parseJsonOrUndefined(value: string): unknown {
return parseJsonOrString(value);
}
-function parseJsonOrString(value: string): unknown {
- if (!value) {
- return value;
- }
-
- try {
- return JSON.parse(value);
- } catch {
- return value;
- }
-}
-
-function isRecord(value: unknown): value is Record<string, unknown> {
- return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-
function getStringValue(value: unknown): string | undefined {
return typeof value === "string" && value.trim() ? value : undefined;
}
diff --git a/packages/shared/src/in-app-agent/server/toolErrors.ts b/packages/shared/src/in-app-agent/server/toolErrors.ts
new file mode 100644
index 0000000000..fd965aa616
--- /dev/null
+++ b/packages/shared/src/in-app-agent/server/toolErrors.ts
@@ -0,0 +1,80 @@
+/**
+ * Structured tool-failure extraction for in-app-agent instrumentation.
+ *
+ * Only trusts explicit AG-UI `event.error` and boolean failure markers
+ * (`error: true` / `isError: true`). Does not infer failure from string-valued
+ * `error` fields or from message text such as "MCP error …".
+ */
+export function getToolFailureMessage(
+ eventError: unknown,
+ output: unknown,
+): string | undefined {
+ const explicitError = getStringValue(eventError);
+ if (explicitError) {
+ // Manual approval rejections encode `{code,message}` as JSON in event.error.
+ // Prefer the human-readable message for statusMessage while leaving the
+ // raw event.error untouched for callers that need the structured code.
+ const parsed = parseJsonOrString(explicitError);
+ if (isRecord(parsed)) {
+ const message = getStringValue(parsed.message);
+ if (message) {
+ return message;
+ }
+ }
+
+ return explicitError;
+ }
+
+ const normalizedOutput = normalizeToolOutput(output);
+ if (!isRecord(normalizedOutput)) {
+ return undefined;
+ }
+
+ if (normalizedOutput.error === true || normalizedOutput.isError === true) {
+ return getStringValue(normalizedOutput.message) ?? "Tool returned an error";
+ }
+
+ return undefined;
+}
+
+export function normalizeToolOutput(output: unknown): unknown {
+ const parsedOutput =
+ typeof output === "string" ? parseJsonOrString(output) : output;
+
+ if (!isRecord(parsedOutput) || !Array.isArray(parsedOutput.content)) {
+ return parsedOutput;
+ }
+
+ const firstContent: unknown = parsedOutput.content[0];
+
+ if (
+ parsedOutput.content.length !== 1 ||
+ !isRecord(firstContent) ||
+ firstContent.type !== "text" ||
+ typeof firstContent.text !== "string"
+ ) {
+ return parsedOutput;
+ }
+
+ return parseJsonOrString(firstContent.text);
+}
+
+export function parseJsonOrString(value: string): unknown {
+ if (!value) {
+ return value;
+ }
+
+ try {
+ return JSON.parse(value);
+ } catch {
+ return value;
+ }
+}
+
+export function isRecord(value: unknown): value is Record<string, unknown> {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function getStringValue(value: unknown): string | undefined {
+ return typeof value === "string" && value.trim() ? value.trim() : undefined;
+}
diff --git a/web/src/__tests__/server/in-app-agent-stream.servertest.ts b/web/src/__tests__/server/in-app-agent-stream.servertest.ts
index 4ac436d39f..185d6c16b1 100644
--- a/web/src/__tests__/server/in-app-agent-stream.servertest.ts
+++ b/web/src/__tests__/server/in-app-agent-stream.servertest.ts
@@ -413,7 +413,10 @@ describe("patchMastraApprovalChunks", () => {
toolName: "bash",
args: { command: "date" },
isError: true,
- result: { error: "Error: Region is missing" },
+ result: {
+ error: true,
+ message: "Error: Region is missing",
+ },
},
},
]);
@@ -910,6 +913,104 @@ describe("createAgUiStream", () => {
expect(agentConfig?.defaultOptions).not.toHaveProperty("providerOptions");
});
+ it("stamps top-level error on TOOL_CALL_RESULT from structured failure payloads", async () => {
+ const { createAgUiStream } =
+ await import("@langfuse/shared/in-app-agent/server/agent");
+ const input = {
+ threadId: "conversation-1",
+ runId: "run-1",
+ messages: [
+ { id: "user-message-1", role: "user" as const, content: "hello" },
+ ],
+ tools: [],
+ context: [],
+ state: {
+ type: "existingConversation" as const,
+ projectId: "project-1",
+ conversationId: "conversation-1",
+ },
+ forwardedProps: {},
+ };
+ const failureMessage = "MCP error -32602: invalid score config";
+ const structuredFailure = {
+ error: true,
+ message: failureMessage,
+ };
+ const persistedEvents: AgUiEvent[] = [];
+ const langfuseClient = {
+ getPrompt: promptMocks.getPrompt,
+ } as unknown as Langfuse;
+
+ adapterEvents.items = [
+ {
+ type: EventType.RUN_STARTED,
+ threadId: input.threadId,
+ runId: input.runId,
+ },
+ {
+ type: EventType.TOOL_CALL_START,
+ toolCallId: "tool-call-1",
+ toolCallName: "createScoreConfig",
+ },
+ {
+ type: EventType.TOOL_CALL_ARGS,
+ toolCallId: "tool-call-1",
+ delta: "{}",
+ },
+ {
+ type: EventType.TOOL_CALL_END,
+ toolCallId: "tool-call-1",
+ },
+ {
+ type: EventType.TOOL_CALL_RESULT,
+ messageId: "tool-result-1",
+ toolCallId: "tool-call-1",
+ content: JSON.stringify(structuredFailure),
+ role: "tool",
+ },
+ {
+ type: EventType.RUN_FINISHED,
+ threadId: input.threadId,
+ runId: input.runId,
+ },
+ ];
+
+ const stream = await createAgUiStream({
+ input,
+ signal: new AbortController().signal,
+ options: {
+ onEvent: (event) => {
+ persistedEvents.push(event);
+ },
+ awsBedrock: { modelId: "test-model" },
+ langfuseMcp: {
+ url: "https://example.com/api/public/mcp",
+ publicKey: "pk",
+ secretKey: "sk",
+ toolPolicy: defaultInAppAgentToolPolicy,
+ runOverride: "run-override",
+ },
+ redirectAction: {
+ projectId: "project-1",
+ isV4Enabled: false,
+ },
+ langfuseClient,
+ useLocalPrompt: false,
+ },
+ });
+
+ await readStream(stream);
+
+ expect(persistedEvents).toContainEqual({
+ type: EventType.TOOL_CALL_RESULT,
+ messageId: "tool-result-1",
+ toolCallId: "tool-call-1",
+ content: JSON.stringify(structuredFailure),
+ role: "tool",
+ error: failureMessage,
+ });
+ });
+
it("stops gating a tool the user approved for the conversation", async () => {
const { createAgUiStream } =
await import("@langfuse/shared/in-app-agent/server/agent");
diff --git a/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts b/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts
index d226e11a53..4f67154543 100644
--- a/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts
+++ b/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts
@@ -408,7 +408,7 @@ describe("InAppAgentInstrumentation", () => {
);
});
- it("marks tool observations with error outputs as error level", () => {
+ it("marks tool observations with structured error outputs as error level", () => {
const instrumentation = createInstrumentation();
instrumentation.recordEvents([
@@ -453,11 +453,142 @@ describe("InAppAgentInstrumentation", () => {
message: "Tool input validation failed",
},
level: "ERROR",
+ statusMessage: "Tool input validation failed",
}),
);
- expect(toolCreateBody).not.toHaveProperty("statusMessage");
});
+ it.each([
+ {
+ name: "top-level event errors",
+ content: "Tool execution failed",
+ error: "Tool execution failed",
+ expectedOutput: "Tool execution failed",
+ expectedMessage: "Tool execution failed",
+ },
+ {
+ name: "boolean isError payloads",
+ content: JSON.stringify({
+ isError: true,
+ message: "Tool returned an error",
+ }),
+ expectedOutput: {
+ isError: true,
+ message: "Tool returned an error",
+ },
+ expectedMessage: "Tool returned an error",
+ },
+ {
+ name: "JSON-encoded approval rejection errors",
+ content: "Tool call was not approved by the user.",
+ error: JSON.stringify({
+ code: "tool_call_rejected",
+ message: "Tool call was not approved by the user.",
+ }),
+ expectedOutput: "Tool call was not approved by the user.",
+ expectedMessage: "Tool call was not approved by the user.",
+ },
+ ])(
+ "marks $name as failed with statusMessage",
+ ({ content, error, expectedOutput, expectedMessage }) => {
+ const instrumentation = createInstrumentation();
+
+ instrumentation.recordEvents([
+ {
+ type: EventType.TOOL_CALL_START,
+ toolCallId: "tool-1",
+ toolCallName: "createScoreConfig",
+ },
+ {
+ type: EventType.TOOL_CALL_ARGS,
+ toolCallId: "tool-1",
+ delta: "{}",
+ },
+ {
+ type: EventType.TOOL_CALL_END,
+ toolCallId: "tool-1",
+ },
+ {
+ type: EventType.TOOL_CALL_RESULT,
+ toolCallId: "tool-1",
+ content,
+ ...(error ? { error } : {}),
+ },
+ ]);
+
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "tool-create",
+ expect.objectContaining({
+ output: expectedOutput,
+ level: "ERROR",
+ statusMessage: expectedMessage,
+ }),
+ );
+ },
+ );
+
+ it.each([
+ {
+ name: "benign content mentioning error",
+ content: JSON.stringify({
+ message: "This record describes an error handling policy",
+ }),
+ expectedOutput: {
+ message: "This record describes an error handling policy",
+ },
+ },
+ {
+ name: "string-valued error fields",
+ content: JSON.stringify({
+ error: "McpError: MCP error -32602: invalid score config",
+ }),
+ expectedOutput: {
+ error: "McpError: MCP error -32602: invalid score config",
+ },
+ },
+ {
+ name: "bare MCP error strings",
+ content: "MCP error -32600: Invalid Request",
+ expectedOutput: "MCP error -32600: Invalid Request",
+ },
+ ])(
+ "does not classify $name as a structured tool failure",
+ ({ content, expectedOutput }) => {
+ const instrumentation = createInstrumentation();
+
+ instrumentation.recordEvents([
+ {
+ type: EventType.TOOL_CALL_START,
+ toolCallId: "tool-1",
+ toolCallName: "search",
+ },
+ {
+ type: EventType.TOOL_CALL_ARGS,
+ toolCallId: "tool-1",
+ delta: "{}",
+ },
+ {
+ type: EventType.TOOL_CALL_END,
+ toolCallId: "tool-1",
+ },
+ {
+ type: EventType.TOOL_CALL_RESULT,
+ toolCallId: "tool-1",
+ content,
+ },
+ ]);
+
+ const toolCreateBody = mocks.handler.langfuse.enqueue.mock.calls[0]?.[1];
+ expect(toolCreateBody).toEqual(
+ expect.objectContaining({
+ output: expectedOutput,
+ }),
+ );
+ expect(toolCreateBody).not.toHaveProperty("level");
+ expect(toolCreateBody).not.toHaveProperty("statusMessage");
+ },
+ );
+
it("sets static prompt metadata on the trace and agent generation", () => {
const instrumentation = createInstrumentation(undefined, {
name: "in-app-agent-system-prompt",