Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(in-app-agent): record tool approval source in the event stream (#16372)

Lysander Kieselc34022568c59
diff --git a/packages/shared/src/in-app-agent/approvalEvents.ts b/packages/shared/src/in-app-agent/approvalEvents.ts
index e2a1736dc7..3b7635c9dc 100644
--- a/packages/shared/src/in-app-agent/approvalEvents.ts
+++ b/packages/shared/src/in-app-agent/approvalEvents.ts
@@ -7,16 +7,40 @@ import type { AgUiCustomEvent, AgUiEvent } from "./schema";
export const IN_APP_AGENT_APPROVAL_DECISION_EVENT_NAME =
"langfuse_approval_decision";
+export const IN_APP_AGENT_TOOL_APPROVAL_EVENT_NAME = "langfuse_tool_approval";
+
export const InAppAgentApprovalDecisionSchema = z.object({
toolCallId: z.string().min(1),
approved: z.boolean(),
decidedByUserId: z.string().min(1),
+ alwaysAllow: z.literal(true).optional(),
+ toolName: z.string().min(1).optional(),
});
export type InAppAgentApprovalDecision = z.infer<
typeof InAppAgentApprovalDecisionSchema
>;
+export const InAppAgentToolApprovalSourceSchema = z.enum([
+ "auto",
+ "human",
+ "conversation_grant",
+]);
+
+export type InAppAgentToolApprovalSource = z.infer<
+ typeof InAppAgentToolApprovalSourceSchema
+>;
+
+export const InAppAgentToolApprovalSchema = z.object({
+ toolCallId: z.string().min(1),
+ toolName: z.string().min(1),
+ source: InAppAgentToolApprovalSourceSchema,
+});
+
+export type InAppAgentToolApproval = z.infer<
+ typeof InAppAgentToolApprovalSchema
+>;
+
export function buildInAppAgentApprovalDecisionEvent(
decision: InAppAgentApprovalDecision,
): AgUiCustomEvent {
@@ -40,3 +64,28 @@ export function parseInAppAgentApprovalDecisionEvent(
const parsed = InAppAgentApprovalDecisionSchema.safeParse(event.value);
return parsed.success ? parsed.data : undefined;
}
+
+export function buildInAppAgentToolApprovalEvent(
+ approval: InAppAgentToolApproval,
+): AgUiCustomEvent {
+ return {
+ type: EventType.CUSTOM,
+ name: IN_APP_AGENT_TOOL_APPROVAL_EVENT_NAME,
+ toolCallId: approval.toolCallId,
+ value: approval,
+ };
+}
+
+export function parseInAppAgentToolApprovalEvent(
+ event: AgUiEvent,
+): InAppAgentToolApproval | undefined {
+ if (
+ event.type !== EventType.CUSTOM ||
+ event.name !== IN_APP_AGENT_TOOL_APPROVAL_EVENT_NAME
+ ) {
+ return undefined;
+ }
+
+ const parsed = InAppAgentToolApprovalSchema.safeParse(event.value);
+ return parsed.success ? parsed.data : undefined;
+}
diff --git a/packages/shared/src/in-app-agent/server/mcpPolicy.ts b/packages/shared/src/in-app-agent/server/mcpPolicy.ts
index 48d86488d6..f22d385162 100644
--- a/packages/shared/src/in-app-agent/server/mcpPolicy.ts
+++ b/packages/shared/src/in-app-agent/server/mcpPolicy.ts
@@ -1,8 +1,15 @@
+import { EventType } from "@ag-ui/core";
+import { z } from "zod";
+
import type { ProjectScope } from "../../features/rbac/projectAccessRights";
import { hasProjectAccessByRole } from "../../features/rbac/projectAccessRights";
import { Role } from "../../db";
import { IN_APP_AGENT_REDIRECT_TOOL_NAME } from "../constants";
-import { z } from "zod";
+import {
+ buildInAppAgentToolApprovalEvent,
+ type InAppAgentToolApprovalSource,
+} from "../approvalEvents";
+import type { AgUiCustomEvent, AgUiEvent } from "../schema";
type InAppAgentMcpToolApproval = "auto" | "approval";
@@ -566,16 +573,95 @@ function isInAppAgentAutoApprovedToolName(
toolName: string,
policy: InAppAgentToolPolicy,
): boolean {
+ const source = getInAppAgentToolApprovalSource({
+ toolName,
+ policy,
+ toolCallId: "",
+ });
+
+ return source !== undefined && source !== "human";
+}
+
+export function getInAppAgentToolApprovalSource(params: {
+ toolName: string;
+ policy: InAppAgentToolPolicy;
+ toolCallId: string;
+ humanApprovedToolCallId?: string;
+}): InAppAgentToolApprovalSource | undefined {
if (
- toolName.startsWith("langfuseDocs_") ||
- IN_APP_AGENT_LOCAL_AUTO_APPROVED_TOOL_NAMES.has(toolName)
+ params.humanApprovedToolCallId &&
+ params.toolCallId === params.humanApprovedToolCallId
) {
- return true;
+ return "human";
}
- const registryToolName = getInAppAgentRegistryToolName(toolName);
+ if (
+ params.toolName.startsWith("langfuseDocs_") ||
+ IN_APP_AGENT_LOCAL_AUTO_APPROVED_TOOL_NAMES.has(params.toolName)
+ ) {
+ return "auto";
+ }
- return (
- registryToolName !== undefined && policy.autoApproved.has(registryToolName)
- );
+ const registryToolName = getInAppAgentRegistryToolName(params.toolName);
+
+ if (
+ registryToolName === undefined ||
+ !params.policy.autoApproved.has(registryToolName)
+ ) {
+ return undefined;
+ }
+
+ return IN_APP_AGENT_LANGFUSE_MCP_TOOL_POLICIES[registryToolName].approval ===
+ "auto"
+ ? "auto"
+ : "conversation_grant";
+}
+
+export function buildInAppAgentToolApprovalSidecar(params: {
+ toolCallId: string;
+ toolName: string;
+ policy: InAppAgentToolPolicy;
+ humanApprovedToolCallId?: string;
+}): AgUiCustomEvent | undefined {
+ const source = getInAppAgentToolApprovalSource(params);
+
+ if (!source) {
+ return undefined;
+ }
+
+ return buildInAppAgentToolApprovalEvent({
+ toolCallId: params.toolCallId,
+ toolName: params.toolName,
+ source,
+ });
+}
+
+export function withInAppAgentToolApprovalSidecars(params: {
+ events: readonly AgUiEvent[];
+ policy: InAppAgentToolPolicy;
+ humanApprovedToolCallId?: string;
+}): AgUiEvent[] {
+ return params.events.flatMap((event) => {
+ if (event.type !== EventType.TOOL_CALL_START) {
+ return [event];
+ }
+
+ const toolCallId =
+ typeof event.toolCallId === "string" ? event.toolCallId : undefined;
+ const toolName =
+ typeof event.toolCallName === "string" ? event.toolCallName : undefined;
+
+ if (!toolCallId || !toolName) {
+ return [event];
+ }
+
+ const sidecar = buildInAppAgentToolApprovalSidecar({
+ toolCallId,
+ toolName,
+ policy: params.policy,
+ humanApprovedToolCallId: params.humanApprovedToolCallId,
+ });
+
+ return sidecar ? [event, sidecar] : [event];
+ });
}
diff --git a/packages/shared/src/in-app-agent/server/persistence.test.ts b/packages/shared/src/in-app-agent/server/persistence.test.ts
index 8b4a699c22..30c5f8f09e 100644
--- a/packages/shared/src/in-app-agent/server/persistence.test.ts
+++ b/packages/shared/src/in-app-agent/server/persistence.test.ts
@@ -2,9 +2,12 @@ import { EventType } from "@ag-ui/core";
import { describe, expect, it } from "vitest";
import type { PrismaClient } from "../../db";
+import { IN_APP_AGENT_REDIRECT_TOOL_NAME } from "../constants";
+import { buildInAppAgentToolApprovalEvent } from "../approvalEvents";
import {
createSandboxToolCallFileAccumulator,
getConversationMessages,
+ partitionPendingRunEvents,
} from "./persistence";
describe("getConversationMessages", () => {
@@ -109,3 +112,32 @@ describe("createSandboxToolCallFileAccumulator", () => {
]);
});
});
+
+describe("partitionPendingRunEvents", () => {
+ it("retains a redirect approval sidecar until the redirect result arrives", () => {
+ const sidecar = buildInAppAgentToolApprovalEvent({
+ toolCallId: "redirect-1",
+ toolName: IN_APP_AGENT_REDIRECT_TOOL_NAME,
+ source: "auto",
+ });
+ const start = {
+ type: EventType.TOOL_CALL_START,
+ toolCallId: "redirect-1",
+ toolCallName: IN_APP_AGENT_REDIRECT_TOOL_NAME,
+ };
+ const args = {
+ type: EventType.TOOL_CALL_ARGS,
+ toolCallId: "redirect-1",
+ delta: "{}",
+ };
+ const end = {
+ type: EventType.TOOL_CALL_END,
+ toolCallId: "redirect-1",
+ };
+
+ expect(partitionPendingRunEvents([start, sidecar, args, end])).toEqual({
+ eventsToAppend: [],
+ retainedEvents: [start, sidecar, args, end],
+ });
+ });
+});
diff --git a/packages/shared/src/in-app-agent/server/persistence.ts b/packages/shared/src/in-app-agent/server/persistence.ts
index 9d81750099..6155931e03 100644
--- a/packages/shared/src/in-app-agent/server/persistence.ts
+++ b/packages/shared/src/in-app-agent/server/persistence.ts
@@ -653,7 +653,7 @@ export function shouldFlushPersistedEvent(event: AgUiEvent) {
);
}
-function partitionPendingRunEvents(events: readonly AgUiEvent[]): {
+export function partitionPendingRunEvents(events: readonly AgUiEvent[]): {
eventsToAppend: AgUiEvent[];
retainedEvents: AgUiEvent[];
} {
diff --git a/packages/shared/src/in-app-agent/server/runLifecycle.ts b/packages/shared/src/in-app-agent/server/runLifecycle.ts
index 65986dcbd8..28b3272e24 100644
--- a/packages/shared/src/in-app-agent/server/runLifecycle.ts
+++ b/packages/shared/src/in-app-agent/server/runLifecycle.ts
@@ -351,6 +351,12 @@ export async function decideToolApproval(params: {
toolCallId: params.toolCallId,
approved: params.approved,
decidedByUserId: params.decidedByUserId,
+ ...(params.alwaysAllowToolName
+ ? {
+ alwaysAllow: true as const,
+ toolName: params.alwaysAllowToolName,
+ }
+ : {}),
}),
});
diff --git a/packages/shared/src/in-app-agent/server/tools.test.ts b/packages/shared/src/in-app-agent/server/tools.test.ts
index a2cad946d4..460b727c6a 100644
--- a/packages/shared/src/in-app-agent/server/tools.test.ts
+++ b/packages/shared/src/in-app-agent/server/tools.test.ts
@@ -1,6 +1,9 @@
import { describe, expect, it } from "vitest";
-import { createInAppAgentToolPolicy } from "./mcpPolicy";
+import {
+ createInAppAgentToolPolicy,
+ getInAppAgentToolApprovalSource,
+} from "./mcpPolicy";
describe("createInAppAgentToolPolicy", () => {
it("drops a stored grant the user's role no longer covers", () => {
@@ -23,3 +26,50 @@ describe("createInAppAgentToolPolicy", () => {
expect(asMember.autoApproved.has("createModel")).toBe(false);
});
});
+
+describe("getInAppAgentToolApprovalSource", () => {
+ const ownerPolicy = createInAppAgentToolPolicy({
+ userAccess: { projectRole: "OWNER", isAdmin: false },
+ alwaysAllowedTools: ["langfuse_createTextPrompt"],
+ });
+
+ it.each([
+ {
+ name: "human",
+ toolName: "langfuse_createTextPrompt",
+ toolCallId: "call-1",
+ humanApprovedToolCallId: "call-1",
+ source: "human",
+ },
+ {
+ name: "conversation_grant",
+ toolName: "langfuse_createTextPrompt",
+ toolCallId: "call-2",
+ source: "conversation_grant",
+ },
+ {
+ name: "auto for MCP policy",
+ toolName: "langfuse_listAnnotationQueues",
+ toolCallId: "call-3",
+ source: "auto",
+ },
+ {
+ name: "auto for sandbox tools",
+ toolName: "read",
+ toolCallId: "call-4",
+ source: "auto",
+ },
+ ] as const)(
+ "classifies $name",
+ ({ toolName, toolCallId, humanApprovedToolCallId, source }) => {
+ expect(
+ getInAppAgentToolApprovalSource({
+ toolName,
+ policy: ownerPolicy,
+ toolCallId,
+ humanApprovedToolCallId,
+ }),
+ ).toBe(source);
+ },
+ );
+});
diff --git a/web/src/__tests__/server/in-app-agent-background-run.servertest.ts b/web/src/__tests__/server/in-app-agent-background-run.servertest.ts
index abcb495e22..cb137b9cf6 100644
--- a/web/src/__tests__/server/in-app-agent-background-run.servertest.ts
+++ b/web/src/__tests__/server/in-app-agent-background-run.servertest.ts
@@ -732,7 +732,31 @@ describe("in-app agent background runs", () => {
});
expect(decisionEvent.event).toMatchObject({
name: IN_APP_AGENT_APPROVAL_DECISION_EVENT_NAME,
- value: { toolCallId: "tool-call-grant" },
+ value: {
+ toolCallId: "tool-call-grant",
+ approved: true,
+ alwaysAllow: true,
+ toolName: "langfuse_createTextPrompt",
+ },
+ });
+
+ const onceDecisionEvent = await prisma.inAppAgentEvent.findFirstOrThrow({
+ where: {
+ projectId,
+ conversationId: onceConversation.id,
+ runId: onceRunId,
+ },
+ orderBy: { sequenceNumber: "desc" },
+ });
+ expect(onceDecisionEvent.event).toMatchObject({
+ name: IN_APP_AGENT_APPROVAL_DECISION_EVENT_NAME,
+ value: {
+ toolCallId: "tool-call-once",
+ approved: true,
+ },
+ });
+ expect(onceDecisionEvent.event).not.toMatchObject({
+ value: { alwaysAllow: true },
});
});
diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts
index 09b768ac0c..c4c19599a1 100644
--- a/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts
+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts
@@ -4,6 +4,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import { createOrgProjectAndApiKey, logger } from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
import { env as sharedEnv } from "@langfuse/shared/src/env";
+import { IN_APP_AGENT_TOOL_APPROVAL_EVENT_NAME } from "@langfuse/shared/in-app-agent";
import { createAndAddApiKeysToDb } from "@langfuse/shared/src/server/auth/apiKeys";
import { ResumeForwardedPropsSchema } from "./runtime/types";
import { env } from "../../env";
@@ -503,6 +504,45 @@ describe("executeInAppAgentRun", () => {
expect((merged!.event as { delta?: string }).delta).toBe("Hello world");
});
+ it("persists a tool-approval source next to TOOL_CALL_START", async () => {
+ const { projectId, conversation, run } = await seedBackgroundRun();
+
+ scenarioRef.current = async ({ options }) => {
+ await options.onEvent({
+ type: "TOOL_CALL_START",
+ toolCallId: "tool-call-read",
+ toolCallName: "read",
+ });
+ await options.onEvent({
+ type: "TOOL_CALL_END",
+ toolCallId: "tool-call-read",
+ });
+ await options.onComplete();
+ await options.onFinish();
+ };
+
+ await executeInAppAgentRun({ projectId, runId: run.id });
+
+ const events = await prisma.inAppAgentEvent.findMany({
+ where: { projectId, conversationId: conversation.id },
+ orderBy: { sequenceNumber: "asc" },
+ });
+ expect(events.map((event) => event.type)).toEqual(
+ expect.arrayContaining(["TOOL_CALL_START", "CUSTOM", "TOOL_CALL_END"]),
+ );
+ expect(events).toHaveLength(3);
+ expect(
+ events.find((event) => event.type === "CUSTOM")?.event,
+ ).toMatchObject({
+ name: IN_APP_AGENT_TOOL_APPROVAL_EVENT_NAME,
+ value: {
+ toolCallId: "tool-call-read",
+ toolName: "read",
+ source: "auto",
+ },
+ });
+ });
+
it("records step_limit on SUCCEEDED when the loop hits the cap without a stop finish", async () => {
const { projectId, run } = await seedBackgroundRun();
diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.ts
index 6e522df55a..e9d2b11eb9 100644
--- a/worker/src/features/in-app-agent/executeInAppAgentRun.ts
+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.ts
@@ -45,6 +45,7 @@ import {
reconcileConversationRuns,
} from "@langfuse/shared/in-app-agent/server/runLifecycle";
import {
+ buildInAppAgentToolApprovalSidecar,
createInAppAgentMcpRunOverride,
createInAppAgentToolPolicy,
getInAppAgentMcpAllowedToolNames,
@@ -474,6 +475,33 @@ export async function executeInAppAgentRun(params: {
}
pendingPersistedEvents.push(persistedEvent);
+
+ if (persistedEvent.type === "TOOL_CALL_START") {
+ const toolCallId =
+ typeof persistedEvent.toolCallId === "string"
+ ? persistedEvent.toolCallId
+ : undefined;
+ const toolName =
+ typeof persistedEvent.toolCallName === "string"
+ ? persistedEvent.toolCallName
+ : undefined;
+ const sidecar =
+ toolCallId && toolName
+ ? buildInAppAgentToolApprovalSidecar({
+ toolCallId,
+ toolName,
+ policy: toolPolicy,
+ humanApprovedToolCallId: isApprovedContinuation
+ ? approvalRequest?.toolCallId
+ : undefined,
+ })
+ : undefined;
+
+ if (sidecar) {
+ pendingPersistedEvents.push(sidecar);
+ }
+ }
+
sandboxToolCallFiles.processEvent({
event: persistedEvent,
runId,
diff --git a/worker/src/features/in-app-agent/runtime/agent.test.ts b/worker/src/features/in-app-agent/runtime/agent.test.ts
index a8439aed71..b3e0bb7503 100644
--- a/worker/src/features/in-app-agent/runtime/agent.test.ts
+++ b/worker/src/features/in-app-agent/runtime/agent.test.ts
@@ -7,6 +7,7 @@ import type { AgUiEvent } from "@langfuse/shared/in-app-agent";
import {
IN_APP_AGENT_MCP_TOOL_OVERRIDE_HEADER,
IN_APP_AGENT_REDIRECT_TOOL_NAME,
+ IN_APP_AGENT_TOOL_APPROVAL_EVENT_NAME,
IN_APP_AGENT_TOOL_REJECTION_ERROR_CODE,
} from "@langfuse/shared/in-app-agent";
import { createInAppAgentToolPolicy } from "@langfuse/shared/in-app-agent/server/mcpPolicy";
@@ -2390,6 +2391,22 @@ describe("createAgUiStream", () => {
toolCallId: "tool-call-1",
status: "rejected",
});
+ expect(
+ instrumentationMocks.instrumentation.recordEvents.mock.calls.flatMap(
+ ([events]) => events,
+ ),
+ ).not.toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ type: EventType.CUSTOM,
+ name: IN_APP_AGENT_TOOL_APPROVAL_EVENT_NAME,
+ value: expect.objectContaining({
+ toolCallId: "tool-call-1",
+ source: "human",
+ }),
+ }),
+ ]),
+ );
expect(onComplete).toHaveBeenCalledOnce();
});
diff --git a/worker/src/features/in-app-agent/runtime/agent.ts b/worker/src/features/in-app-agent/runtime/agent.ts
index 1d56316745..03656a0d4f 100644
--- a/worker/src/features/in-app-agent/runtime/agent.ts
+++ b/worker/src/features/in-app-agent/runtime/agent.ts
@@ -45,6 +45,7 @@ import {
getInAppAgentRegistryToolName,
type InAppAgentToolPolicy,
withInAppAgentToolApproval,
+ withInAppAgentToolApprovalSidecars,
} from "@langfuse/shared/in-app-agent/server/mcpPolicy";
import { LANGFUSE_IN_APP_AGENT_SKILLS } from "./skills";
import type { InAppAgentSandbox } from "./sandbox";
@@ -673,6 +674,10 @@ export async function createAgUiStream(params: {
onApprovedToolCallExecuted:
params.options.onApprovedToolCallExecuted,
});
+ const humanApprovedToolCallId =
+ runInput.toolCallApproval?.status === "approved"
+ ? runInput.toolCallApproval.toolCallId
+ : undefined;
const pendingSyntheticEvents = [...runInput.syntheticEvents];
currentAdapter.setDeveloperGuidance(runInput.developerGuidance);
@@ -757,7 +762,13 @@ export async function createAgUiStream(params: {
);
recordInstrumentation("recordEvents", (instrumentation) =>
- instrumentation.recordEvents(agUiEvents),
+ instrumentation.recordEvents(
+ withInAppAgentToolApprovalSidecars({
+ events: agUiEvents,
+ policy: params.options.langfuseMcp.toolPolicy,
+ humanApprovedToolCallId,
+ }),
+ ),
);
for (const agUiEvent of agUiEvents) {
@@ -787,7 +798,13 @@ export async function createAgUiStream(params: {
),
);
recordInstrumentation("recordEvents", (instrumentation) =>
- instrumentation.recordEvents(pendingSyntheticEvents),
+ instrumentation.recordEvents(
+ withInAppAgentToolApprovalSidecars({
+ events: pendingSyntheticEvents,
+ policy: params.options.langfuseMcp.toolPolicy,
+ humanApprovedToolCallId,
+ }),
+ ),
);
for (const syntheticEvent of pendingSyntheticEvents) {
enqueueEvent(syntheticEvent);
diff --git a/worker/src/features/in-app-agent/runtime/instrumentation.test.ts b/worker/src/features/in-app-agent/runtime/instrumentation.test.ts
index 97fd6c687e..b2731e6756 100644
--- a/worker/src/features/in-app-agent/runtime/instrumentation.test.ts
+++ b/worker/src/features/in-app-agent/runtime/instrumentation.test.ts
@@ -5,6 +5,7 @@ import {
getInAppAgentInstrumentationObservationId,
getInAppAgentInstrumentationTraceId,
getInAppAgentLlmCallObservationId,
+ IN_APP_AGENT_TOOL_APPROVAL_EVENT_NAME,
} from "@langfuse/shared/in-app-agent";
import type { AgUiRunAgentInput } from "./types";
import { InAppAgentInstrumentation } from "./instrumentation";
@@ -273,6 +274,52 @@ describe("InAppAgentInstrumentation", () => {
);
});
+ it("copies langfuse_tool_approval source onto the TOOL observation", () => {
+ const instrumentation = createInstrumentation();
+
+ instrumentation.recordEvents([
+ {
+ type: EventType.TOOL_CALL_START,
+ toolCallId: "tool-1",
+ toolCallName: "read",
+ parentMessageId: "assistant-message-1",
+ },
+ {
+ type: EventType.CUSTOM,
+ name: IN_APP_AGENT_TOOL_APPROVAL_EVENT_NAME,
+ value: {
+ toolCallId: "tool-1",
+ toolName: "read",
+ source: "auto",
+ },
+ },
+ {
+ type: EventType.TOOL_CALL_ARGS,
+ toolCallId: "tool-1",
+ delta: '{"path":"README.md"}',
+ },
+ {
+ type: EventType.TOOL_CALL_END,
+ toolCallId: "tool-1",
+ },
+ {
+ type: EventType.TOOL_CALL_RESULT,
+ toolCallId: "tool-1",
+ content: "ok",
+ },
+ ]);
+
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "tool-create",
+ expect.objectContaining({
+ metadata: expect.objectContaining({
+ toolCallId: "tool-1",
+ toolCallApprovalSource: "auto",
+ }),
+ }),
+ );
+ });
+
it("records rejected approvals without fabricating tool execution duration", () => {
vi.useFakeTimers();
try {
diff --git a/worker/src/features/in-app-agent/runtime/instrumentation.ts b/worker/src/features/in-app-agent/runtime/instrumentation.ts
index 6ed9eb8ec5..4f708e93e6 100644
--- a/worker/src/features/in-app-agent/runtime/instrumentation.ts
+++ b/worker/src/features/in-app-agent/runtime/instrumentation.ts
@@ -6,6 +6,8 @@ import {
getInAppAgentInstrumentationObservationId,
getInAppAgentInstrumentationTraceId,
getInAppAgentLlmCallObservationId,
+ parseInAppAgentToolApprovalEvent,
+ type InAppAgentToolApprovalSource,
} from "@langfuse/shared/in-app-agent";
import type { AgUiEvent, AgUiMessage } from "@langfuse/shared/in-app-agent";
import {
@@ -190,6 +192,10 @@ export class InAppAgentInstrumentation {
string,
ToolCallApprovalStatus
>();
+ private readonly toolCallApprovalSources = new Map<
+ string,
+ InAppAgentToolApprovalSource
+ >();
private readonly toolExecutionTimes = new Map<string, ToolExecutionTimes>();
private readonly metadata: Record<string, unknown>;
private readonly agentRunOutputMessages: AgentRunChatMessage[] = [];
@@ -517,6 +523,15 @@ export class InAppAgentInstrumentation {
return;
}
+ const toolApproval = parseInAppAgentToolApprovalEvent(event);
+ if (toolApproval) {
+ this.toolCallApprovalSources.set(
+ toolApproval.toolCallId,
+ toolApproval.source,
+ );
+ return;
+ }
+
if (event.type === EventType.TOOL_CALL_ARGS) {
this.appendToolArgs(event);
return;
@@ -666,6 +681,7 @@ export class InAppAgentInstrumentation {
tool.failureMessage ??
getToolFailureMessage(undefined, output);
const toolCallApproval = this.toolCallApprovals.get(toolCallId);
+ const toolCallApprovalSource = this.toolCallApprovalSources.get(toolCallId);
const executionTimes = this.toolExecutionTimes.get(toolCallId);
const rawStartTime = executionTimes?.startTime ?? tool.startTime;
const endTime =
@@ -691,6 +707,7 @@ export class InAppAgentInstrumentation {
...(options?.metadata ?? {}),
toolCallId,
...(toolCallApproval ? { toolCallApproval } : {}),
+ ...(toolCallApprovalSource ? { toolCallApprovalSource } : {}),
...(tool.argsComplete ? {} : { argsComplete: false }),
...(tool.parentMessageId
? { parentMessageId: tool.parentMessageId }
@@ -700,6 +717,7 @@ export class InAppAgentInstrumentation {
this.recordToolCall(toolCallId, tool, output);
this.toolCallApprovals.delete(toolCallId);
+ this.toolCallApprovalSources.delete(toolCallId);
this.toolExecutionTimes.delete(toolCallId);
this.enqueueObservation("tool-create", body);