Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(in-app-agent): trace complete agent turns across approvals (#16037)

Lysander Kiesel2ef787e7e2fc
diff --git a/packages/shared/src/features/inAppAgent/types.ts b/packages/shared/src/features/inAppAgent/types.ts
index fd5724e6e7..dbff305eab 100644
--- a/packages/shared/src/features/inAppAgent/types.ts
+++ b/packages/shared/src/features/inAppAgent/types.ts
@@ -79,8 +79,16 @@ export const InAppAgentRunRequestSchema = z.discriminatedUnion("kind", [
}),
z.object({
kind: z.literal("approvalDecision"),
- /** Provenance for debugging only; no code path queries lineage. */
+ /** Immediate parent run used to derive durable approval-chain lineage. */
parentRunId: z.string(),
+ /** Original user-message run whose trace owns the complete approval chain. */
+ rootRunId: z.string().optional(),
+ /** Stable root trace timestamp propagated across durable continuations. */
+ traceStartedAt: z.iso.datetime({ offset: true }).optional(),
+ /** Time at which the parent run parked and began waiting for approval. */
+ approvalRequestedAt: z.iso.datetime({ offset: true }).optional(),
+ /** One-based position in the current user turn's approval continuation chain. */
+ continuationNumber: z.number().int().positive().optional(),
toolCallId: z.string(),
approved: z.boolean(),
/** Inherited sanitized context; defaults for legacy continuation rows. */
diff --git a/packages/shared/src/in-app-agent/constants.ts b/packages/shared/src/in-app-agent/constants.ts
index 2d9cf98d69..3f738b0262 100644
--- a/packages/shared/src/in-app-agent/constants.ts
+++ b/packages/shared/src/in-app-agent/constants.ts
@@ -29,3 +29,10 @@ export const getInAppAgentInstrumentationObservationId = (runId: string) =>
// level telemetry and feedback aggregate on the same per-turn trace.
export const getInAppAgentInstrumentationTraceId = (runId: string) =>
`${runId}-trace`;
+
+// Per-LLM-call generation ids stay deterministic from the run so retries and
+// late flushes can address the same observation.
+export const getInAppAgentLlmCallObservationId = (
+ runId: string,
+ stepNumber: number,
+) => `${runId}-llm-${stepNumber}`;
diff --git a/packages/shared/src/in-app-agent/schema.ts b/packages/shared/src/in-app-agent/schema.ts
index c07d96ade0..b277458789 100644
--- a/packages/shared/src/in-app-agent/schema.ts
+++ b/packages/shared/src/in-app-agent/schema.ts
@@ -239,6 +239,11 @@ export const ResumeForwardedPropsSchema = z.object({
command: z.object({
resume: z.object({
approved: z.boolean(),
+ continuationNumber: z.number().int().positive().optional(),
+ rootRunId: z.string().min(1).optional(),
+ traceStartedAt: z.iso.datetime({ offset: true }).optional(),
+ approvalRequestedAt: z.iso.datetime({ offset: true }).optional(),
+ approvalDecidedAt: z.iso.datetime({ offset: true }).optional(),
approvalRequest: InAppAgentToolApprovalRequestSchema,
}),
}),
diff --git a/packages/shared/src/in-app-agent/server/agent.ts b/packages/shared/src/in-app-agent/server/agent.ts
index 3403c300fa..e5485b6dca 100644
--- a/packages/shared/src/in-app-agent/server/agent.ts
+++ b/packages/shared/src/in-app-agent/server/agent.ts
@@ -33,6 +33,7 @@ import {
filterInAppAgentAvailableLangfuseMcpTools,
getInAppAgentMcpAllowedToolNames,
getInAppAgentRegistryToolName,
+ getToolCallId,
hasCallableExecute,
type CompletedInAppAgentMcpToolCall,
type InAppAgentToolPolicy,
@@ -237,10 +238,63 @@ export async function createAgUiStream(params: {
: undefined,
model: params.options.awsBedrock.modelId,
});
- instrumentation?.recordAvailableSkills?.(LANGFUSE_IN_APP_AGENT_SKILLS);
+ const recordInstrumentation = (
+ operation: string,
+ callback: (
+ activeInstrumentation: NonNullable<typeof instrumentation>,
+ ) => void,
+ ) => {
+ if (!instrumentation) {
+ return;
+ }
+
+ try {
+ callback(instrumentation);
+ } catch (error) {
+ logger.warn("Failed to record in-app agent Langfuse tracing", {
+ error,
+ operation,
+ runId: params.input.runId,
+ threadId: params.input.threadId,
+ });
+ }
+ };
+ recordInstrumentation("recordAvailableSkills", (instrumentation) =>
+ instrumentation.recordAvailableSkills?.(LANGFUSE_IN_APP_AGENT_SKILLS),
+ );
const onStepFinish = instrumentation
? (event: unknown) => {
- instrumentation.recordStepFinish?.(event);
+ recordInstrumentation("recordStepFinish", (instrumentation) =>
+ instrumentation.recordStepFinish?.(event),
+ );
+ }
+ : undefined;
+ const onChunk = instrumentation
+ ? (chunk: unknown) => {
+ recordInstrumentation("recordStreamChunk", (instrumentation) =>
+ instrumentation.recordStreamChunk?.(chunk),
+ );
+ }
+ : undefined;
+ const onModelCallFinish = instrumentation
+ ? (event: unknown) => {
+ recordInstrumentation("recordModelCallFinish", (instrumentation) =>
+ instrumentation.recordModelCallFinish?.(event),
+ );
+ }
+ : undefined;
+ const onToolExecutionStart = instrumentation
+ ? (toolCallId: string) => {
+ recordInstrumentation("recordToolExecutionStart", (instrumentation) =>
+ instrumentation.recordToolExecutionStart?.(toolCallId),
+ );
+ }
+ : undefined;
+ const onToolExecutionEnd = instrumentation
+ ? (toolCallId: string) => {
+ recordInstrumentation("recordToolExecutionEnd", (instrumentation) =>
+ instrumentation.recordToolExecutionEnd?.(toolCallId),
+ );
}
: undefined;
@@ -324,8 +378,12 @@ export async function createAgUiStream(params: {
return;
}
- instrumentation?.endWithError(error);
- instrumentation?.flush();
+ recordInstrumentation("endWithError", (instrumentation) =>
+ instrumentation.endWithError(error),
+ );
+ recordInstrumentation("flush", (instrumentation) =>
+ instrumentation.flush(),
+ );
ending = true;
closed = true;
shouldEnqueue = false;
@@ -429,8 +487,12 @@ export async function createAgUiStream(params: {
return;
}
- instrumentation?.end({ aborted: true });
- instrumentation?.flush();
+ recordInstrumentation("end", (instrumentation) =>
+ instrumentation.end({ aborted: true }),
+ );
+ recordInstrumentation("flush", (instrumentation) =>
+ instrumentation.flush(),
+ );
ending = true;
shouldEnqueue = false;
removeAbortHandler();
@@ -489,8 +551,14 @@ export async function createAgUiStream(params: {
awsProfile,
instructions,
onToolsAvailable: (tools) =>
- instrumentation?.recordAvailableTools?.(tools),
+ recordInstrumentation("recordAvailableTools", (instrumentation) =>
+ instrumentation.recordAvailableTools?.(tools),
+ ),
onStepFinish,
+ onChunk,
+ onModelCallFinish,
+ onToolExecutionStart,
+ onToolExecutionEnd,
})
.then(async (initialAdapter) => {
if (ending || closed || params.signal.aborted) {
@@ -559,6 +627,10 @@ export async function createAgUiStream(params: {
awsProfile,
instructions,
onStepFinish,
+ onChunk,
+ onModelCallFinish,
+ onToolExecutionStart,
+ onToolExecutionEnd,
});
if (ending || closed || params.signal.aborted) {
@@ -595,7 +667,9 @@ export async function createAgUiStream(params: {
params.input,
);
- instrumentation?.recordEvents(agUiEvents);
+ recordInstrumentation("recordEvents", (instrumentation) =>
+ instrumentation.recordEvents(agUiEvents),
+ );
for (const agUiEvent of agUiEvents) {
if (
@@ -616,10 +690,16 @@ export async function createAgUiStream(params: {
agUiEvent.type === EventType.RUN_STARTED &&
pendingSyntheticEvents.length > 0
) {
- instrumentation?.recordToolCallApproval(
- runInput.toolCallApproval,
+ recordInstrumentation(
+ "recordToolCallApproval",
+ (instrumentation) =>
+ instrumentation.recordToolCallApproval(
+ runInput.toolCallApproval,
+ ),
+ );
+ recordInstrumentation("recordEvents", (instrumentation) =>
+ instrumentation.recordEvents(pendingSyntheticEvents),
);
- instrumentation?.recordEvents(pendingSyntheticEvents);
for (const syntheticEvent of pendingSyntheticEvents) {
enqueueEvent(syntheticEvent);
}
@@ -639,7 +719,9 @@ export async function createAgUiStream(params: {
if (streamedRunError !== null) {
closeController(() => {
- instrumentation?.flush();
+ recordInstrumentation("flush", (instrumentation) =>
+ instrumentation.flush(),
+ );
return handleStreamedRunError();
});
return;
@@ -652,11 +734,17 @@ export async function createAgUiStream(params: {
});
const runErrorEvent = createRunErrorEvent(params.input, error);
- instrumentation?.recordEvents([runErrorEvent]);
+ recordInstrumentation("recordEvents", (instrumentation) =>
+ instrumentation.recordEvents([runErrorEvent]),
+ );
enqueueEvent(runErrorEvent, () =>
params.options.onError?.(error),
);
- closeController(() => instrumentation?.flush());
+ closeController(() =>
+ recordInstrumentation("flush", (instrumentation) =>
+ instrumentation.flush(),
+ ),
+ );
},
complete() {
if (ending || closed) {
@@ -671,12 +759,18 @@ export async function createAgUiStream(params: {
closeController(
streamedRunError === null
? () => {
- instrumentation?.end({});
- instrumentation?.flush();
+ recordInstrumentation("end", (instrumentation) =>
+ instrumentation.end({}),
+ );
+ recordInstrumentation("flush", (instrumentation) =>
+ instrumentation.flush(),
+ );
return params.options.onComplete?.();
}
: () => {
- instrumentation?.flush();
+ recordInstrumentation("flush", (instrumentation) =>
+ instrumentation.flush(),
+ );
return handleStreamedRunError();
},
);
@@ -700,9 +794,15 @@ export async function createAgUiStream(params: {
});
const runErrorEvent = createRunErrorEvent(params.input, error);
- instrumentation?.recordEvents([runErrorEvent]);
+ recordInstrumentation("recordEvents", (instrumentation) =>
+ instrumentation.recordEvents([runErrorEvent]),
+ );
enqueueEvent(runErrorEvent, () => params.options.onError?.(error));
- closeController(() => instrumentation?.flush());
+ closeController(() =>
+ recordInstrumentation("flush", (instrumentation) =>
+ instrumentation.flush(),
+ ),
+ );
});
},
cancel() {
@@ -711,8 +811,12 @@ export async function createAgUiStream(params: {
}
ending = true;
- instrumentation?.end({ aborted: true });
- instrumentation?.flush();
+ recordInstrumentation("end", (instrumentation) =>
+ instrumentation.end({ aborted: true }),
+ );
+ recordInstrumentation("flush", (instrumentation) =>
+ instrumentation.flush(),
+ );
shouldEnqueue = false;
removeAbortHandler();
interruptAdapter?.();
@@ -751,6 +855,85 @@ type ExecutableInAppAgentTool = {
toModelOutput?: (output: unknown) => unknown | PromiseLike<unknown>;
};
+type BedrockLanguageModel = ReturnType<ReturnType<typeof createAmazonBedrock>>;
+
+function withModelCallFinish(
+ model: BedrockLanguageModel,
+ onFinish?: (event: unknown) => void,
+): BedrockLanguageModel {
+ if (!onFinish) {
+ return model;
+ }
+
+ return {
+ specificationVersion: model.specificationVersion,
+ provider: model.provider,
+ modelId: model.modelId,
+ supportedUrls: model.supportedUrls,
+ doGenerate: (options) => model.doGenerate(options),
+ doStream: async (options) => {
+ const result = await model.doStream(options);
+
+ return {
+ ...result,
+ stream: result.stream.pipeThrough(
+ new TransformStream({
+ transform(part, controller) {
+ if (part.type === "finish") {
+ onFinish(part);
+ }
+ controller.enqueue(part);
+ },
+ }),
+ ),
+ };
+ },
+ };
+}
+
+function withToolExecutionTiming<TTool>(params: {
+ tools: Record<string, TTool>;
+ onStart?: (toolCallId: string) => void;
+ onEnd?: (toolCallId: string) => void;
+}): Record<string, TTool> {
+ if (!params.onStart && !params.onEnd) {
+ return params.tools;
+ }
+
+ return Object.fromEntries(
+ Object.entries(params.tools).map(([toolName, tool]) => {
+ if (!hasCallableExecute(tool)) {
+ return [toolName, tool];
+ }
+
+ const execute = tool.execute.bind(tool) as (
+ inputData: unknown,
+ context: unknown,
+ ) => Promise<unknown>;
+
+ return [
+ toolName,
+ {
+ ...tool,
+ execute: async (inputData: unknown, context: unknown) => {
+ const toolCallId = getToolCallId(context);
+ if (!toolCallId) {
+ return execute(inputData, context);
+ }
+
+ params.onStart?.(toolCallId);
+ try {
+ return await execute(inputData, context);
+ } finally {
+ params.onEnd?.(toolCallId);
+ }
+ },
+ } as TTool,
+ ];
+ }),
+ );
+}
+
async function createMastraAdapter(params: {
input: AgUiRunAgentInput;
signal: AbortSignal;
@@ -760,6 +943,10 @@ async function createMastraAdapter(params: {
instructions: string;
onToolsAvailable?: (tools: Record<string, unknown>) => void;
onStepFinish?: (event: unknown) => void;
+ onChunk?: (chunk: unknown) => void;
+ onModelCallFinish?: (event: unknown) => void;
+ onToolExecutionStart?: (toolCallId: string) => void;
+ onToolExecutionEnd?: (toolCallId: string) => void;
}) {
const bedrock = createAmazonBedrock({
...(params.options.awsBedrock.region
@@ -842,39 +1029,43 @@ async function createMastraAdapter(params: {
// agent.stream(..., { toolsets }) call. Keep Mastra's per-request MCP
// discovery, then prefix tool names for constructor-based tools so the
// model sees the same names that later appear in AG-UI tool-call events.
- const tools = withInAppAgentToolApproval(
- {
- ...withOptionalSilentMcpOutput({
- tools: withLangfuseMcpRateLimitWait({
- tools: prefixToolsetTools(
- "langfuse",
- filterInAppAgentAvailableLangfuseMcpTools({
- tools: toolsets.langfuse,
- policy: params.options.langfuseMcp.toolPolicy,
- }),
- ),
- signal: params.signal,
- runId: params.input.runId,
- threadId: params.input.threadId,
+ const tools = withToolExecutionTiming({
+ tools: withInAppAgentToolApproval(
+ {
+ ...withOptionalSilentMcpOutput({
+ tools: withLangfuseMcpRateLimitWait({
+ tools: prefixToolsetTools(
+ "langfuse",
+ filterInAppAgentAvailableLangfuseMcpTools({
+ tools: toolsets.langfuse,
+ policy: params.options.langfuseMcp.toolPolicy,
+ }),
+ ),
+ signal: params.signal,
+ runId: params.input.runId,
+ threadId: params.input.threadId,
+ }),
+ sandbox: params.options.sandbox,
+ onToolCallCompleted: params.options.onMcpToolCallCompleted,
}),
- sandbox: params.options.sandbox,
- onToolCallCompleted: params.options.onMcpToolCallCompleted,
- }),
- ...withOptionalSilentMcpOutput({
- tools: prefixToolsetTools("langfuseDocs", toolsets.langfuseDocs),
- sandbox: params.options.sandbox,
- onToolCallCompleted: params.options.onMcpToolCallCompleted,
- }),
- [IN_APP_AGENT_REDIRECT_TOOL_NAME]: createRedirectActionTool({
- projectId: params.options.redirectAction.projectId,
- isV4Enabled: params.options.redirectAction.isV4Enabled,
- }),
- ...(params.options.sandbox
- ? createSandboxTools(params.options.sandbox)
- : {}),
- },
- params.options.langfuseMcp.toolPolicy,
- );
+ ...withOptionalSilentMcpOutput({
+ tools: prefixToolsetTools("langfuseDocs", toolsets.langfuseDocs),
+ sandbox: params.options.sandbox,
+ onToolCallCompleted: params.options.onMcpToolCallCompleted,
+ }),
+ [IN_APP_AGENT_REDIRECT_TOOL_NAME]: createRedirectActionTool({
+ projectId: params.options.redirectAction.projectId,
+ isV4Enabled: params.options.redirectAction.isV4Enabled,
+ }),
+ ...(params.options.sandbox
+ ? createSandboxTools(params.options.sandbox)
+ : {}),
+ },
+ params.options.langfuseMcp.toolPolicy,
+ ),
+ onStart: params.onToolExecutionStart,
+ onEnd: params.onToolExecutionEnd,
+ });
params.onToolsAvailable?.(tools);
const reasoningProviderOptions = getBedrockReasoningProviderOptions(
@@ -887,14 +1078,18 @@ async function createMastraAdapter(params: {
// instruction channel so the model receives the same higher-priority
// guidance on resumed runs.
let developerGuidance: string | undefined;
+ const model = withModelCallFinish(
+ bedrock(
+ params.options.awsBedrock.modelId as Parameters<typeof bedrock>[0],
+ ),
+ params.onModelCallFinish,
+ );
const agent = new Agent({
id: "langfuse-in-app-assistant",
name: ASSISTANT_TITLE,
instructions: () =>
[params.instructions, developerGuidance].filter(Boolean).join("\n\n"),
- model: bedrock(
- params.options.awsBedrock.modelId as Parameters<typeof bedrock>[0],
- ),
+ model,
skills: LANGFUSE_IN_APP_AGENT_SKILLS,
tools,
maxRetries: 2,
@@ -907,6 +1102,8 @@ async function createMastraAdapter(params: {
// Fires once per LLM call with that call's token usage; the AG-UI
// event stream itself never carries usage.
...(params.onStepFinish ? { onStepFinish: params.onStepFinish } : {}),
+ // Tool execution windows (and optional step-start timing bookmarks).
+ ...(params.onChunk ? { onChunk: params.onChunk } : {}),
...(reasoningProviderOptions
? { providerOptions: reasoningProviderOptions }
: {}),
diff --git a/packages/shared/src/in-app-agent/server/instrumentation.ts b/packages/shared/src/in-app-agent/server/instrumentation.ts
index cec9a4d940..1b163f206a 100644
--- a/packages/shared/src/in-app-agent/server/instrumentation.ts
+++ b/packages/shared/src/in-app-agent/server/instrumentation.ts
@@ -4,8 +4,14 @@ import { getInternalTracingHandler, logger } from "../../server";
import {
getInAppAgentInstrumentationObservationId,
getInAppAgentInstrumentationTraceId,
+ getInAppAgentLlmCallObservationId,
} from "../constants";
-import type { AgUiEvent, AgUiMessage, AgUiRunAgentInput } from "../schema";
+import {
+ ResumeForwardedPropsSchema,
+ type AgUiEvent,
+ type AgUiMessage,
+ type AgUiRunAgentInput,
+} from "../schema";
import { compactTextMessageChunks } from "./eventCompaction";
import {
getToolFailureMessage,
@@ -43,11 +49,11 @@ export type InAppAgentPromptMetadata = {
};
const IN_APP_AGENT_TURN_NAME = "agent-turn";
+const IN_APP_AGENT_LLM_CALL_NAME = "invoke-model";
type InternalTracingHandler = ReturnType<typeof getInternalTracingHandler>;
type InAppAgentTrace = ReturnType<
InternalTracingHandler["handler"]["langfuse"]["trace"]
>;
-type InAppAgentGeneration = ReturnType<InAppAgentTrace["generation"]>;
type InAppAgentLangfuse = InternalTracingHandler["handler"]["langfuse"];
type AgentRunToolCall = {
id: string;
@@ -82,19 +88,23 @@ type AgentRunChatMessage = {
tool_calls?: AgentRunToolCall[];
tool_call_id?: string;
};
-type ToolObservationBody = {
+type ObservationBody = {
id: string;
traceId: string;
- parentObservationId: string | null;
+ parentObservationId?: string | null;
name: string;
startTime: Date;
endTime: Date;
- completionStartTime: Date;
+ completionStartTime?: Date;
input?: unknown;
output?: unknown;
- level?: "ERROR";
- statusMessage?: string;
+ model?: string;
+ usageDetails?: Record<string, number>;
+ level?: "DEFAULT" | "ERROR";
+ statusMessage?: string | null;
metadata?: Record<string, unknown>;
+ promptName?: string;
+ promptVersion?: number;
};
type ToolCallApprovalStatus = "approved" | "rejected";
type AgentRunToolSpan = {
@@ -106,6 +116,32 @@ type AgentRunToolSpan = {
failureMessage?: string;
parentMessageId?: string;
};
+// Lightweight bookmark for the in-flight Mastra step. Generations are emitted
+// from onStepFinish (which carries usage); this only supplies start timing and
+// the input-message boundary for context-growth visibility.
+type OpenLlmStep = {
+ stepNumber: number;
+ startTime: Date;
+ inputMessageCount: number;
+ completionStartTime?: Date;
+ providerFinish?: Record<string, unknown>;
+ providerFinishTime?: Date;
+};
+type ToolExecutionTimes = {
+ startTime?: Date;
+ endTime?: Date;
+};
+type ApprovalContinuation = {
+ status: ToolCallApprovalStatus;
+ continuationNumber: number;
+ rootRunId: string;
+ toolCallId: string;
+ toolName: string;
+ traceStartedAt?: Date;
+ approvalRequestedAt?: Date;
+ approvalDecidedAt?: Date;
+ metadata: Record<string, unknown>;
+};
export function createInAppAgentInstrumentation({
input,
@@ -140,24 +176,39 @@ export class InAppAgentInstrumentation {
private readonly processTracedEvents: () => Promise<void>;
private readonly langfuse: InAppAgentLangfuse;
private readonly trace: InAppAgentTrace;
- private readonly agentRun: InAppAgentGeneration;
+ private readonly runId: string;
+ private readonly traceId: string;
+ private readonly rootObservationId: string;
+ private readonly traceStartTime: Date;
+ private readonly agentRunStartTime: Date;
private agentRunInput: unknown;
+ private readonly approvalContinuation?: ApprovalContinuation;
private readonly prompt?: InAppAgentPromptMetadata;
private readonly toolSpans = new Map<string, AgentRunToolSpan>();
private readonly toolCallApprovals = new Map<
string,
ToolCallApprovalStatus
>();
+ private readonly toolExecutionTimes = new Map<string, ToolExecutionTimes>();
private readonly metadata: Record<string, unknown>;
private readonly agentRunOutputMessages: AgentRunChatMessage[] = [];
private readonly agentRunToolCalls: AgentRunToolCall[] = [];
private output = "";
private currentReasoning = "";
private pendingThinking: AgentRunThinkingPart[] = [];
- private completionStartTime?: Date;
private ended = false;
private readonly model?: string;
- private usageDetails?: Record<string, number>;
+ private nextStepNumber = 0;
+ private openLlmStep?: OpenLlmStep;
+ // Tools that finished while an LLM step was still open. Emit after the
+ // generation so the tree sorts model → tools (tool-call chunks arrive during
+ // the LLM stream, before onStepFinish).
+ private readonly deferredToolCompletions = new Map<
+ string,
+ AgentRunToolSpan
+ >();
+ // Clamp tool execution windows to start at/after the last generation end.
+ private lastLlmGenerationEndTime?: Date;
constructor(params: {
input: AgUiRunAgentInput;
@@ -172,6 +223,7 @@ export class InAppAgentInstrumentation {
prompt?: InAppAgentPromptMetadata;
model?: string;
}) {
+ this.approvalContinuation = getApprovalContinuation(params.input);
this.metadata = {
...params.metadata,
...(params.userEmail ? { langfuse_user_email: params.userEmail } : {}),
@@ -185,14 +237,28 @@ export class InAppAgentInstrumentation {
prompt_version: params.prompt.version,
}
: {}),
+ ...(this.approvalContinuation
+ ? {
+ approval_continuation_count:
+ this.approvalContinuation.continuationNumber,
+ }
+ : {}),
};
this.agentRunInput = getAgentRunInput(params.input);
this.prompt = params.prompt;
this.model = params.model;
+ this.runId = params.runId;
+ const rootRunId = this.approvalContinuation?.rootRunId ?? params.runId;
+ this.traceId = getInAppAgentInstrumentationTraceId(rootRunId);
+ this.rootObservationId =
+ getInAppAgentInstrumentationObservationId(rootRunId);
+ this.agentRunStartTime = new Date();
+ this.traceStartTime =
+ this.approvalContinuation?.traceStartedAt ?? this.agentRunStartTime;
const traceSinkParams = {
targetProjectId: params.targetProjectId,
- traceId: getInAppAgentInstrumentationTraceId(params.runId),
+ traceId: this.traceId,
traceName: IN_APP_AGENT_TURN_NAME,
environment: params.environment,
userId: params.userId,
@@ -205,26 +271,21 @@ export class InAppAgentInstrumentation {
this.langfuse = handler.langfuse;
this.trace = this.langfuse.trace({
- id: getInAppAgentInstrumentationTraceId(params.runId),
+ id: this.traceId,
name: IN_APP_AGENT_TURN_NAME,
userId: params.userId,
sessionId: params.input.threadId,
- input: this.agentRunInput,
+ timestamp: this.traceStartTime,
+ input: this.getTraceInput(),
metadata: this.metadata,
tags: ["in-app-agent"],
});
- this.agentRun = this.trace.generation({
- id: getInAppAgentInstrumentationObservationId(params.input.runId),
- name: IN_APP_AGENT_TURN_NAME,
- input: this.agentRunInput,
- metadata: this.metadata,
- ...(params.prompt
- ? {
- promptName: params.prompt.name,
- promptVersion: params.prompt.version,
- }
- : {}),
- });
+
+ // Create the root observation immediately so tool/generation children
+ // emitted during the run have a parent that already exists (avoids
+ // orphans that attach to the trace as siblings of the late agent).
+ this.emitRootAgentObservation({ metadata: this.metadata });
+ this.emitApprovalWaitObservation();
}
recordEvents(events: AgUiEvent[]) {
@@ -267,48 +328,140 @@ export class InAppAgentInstrumentation {
this.toolCallApprovals.set(approval.toolCallId, approval.status);
}
- // Receives Mastra's per-LLM-call onStepFinish event and accumulates its
- // token usage onto the agent-turn generation, since the AG-UI event stream
- // itself never carries usage.
- recordStepFinish(event: unknown) {
+ recordToolExecutionStart(toolCallId: string) {
if (this.ended) {
return;
}
- const usage = isRecord(event) ? event.usage : undefined;
- if (!isRecord(usage)) {
+ const times = this.toolExecutionTimes.get(toolCallId) ?? {};
+ times.startTime ??= new Date();
+ this.toolExecutionTimes.set(toolCallId, times);
+ }
+
+ recordToolExecutionEnd(toolCallId: string) {
+ if (this.ended) {
return;
}
- const inputTokens = getFiniteNumber(usage.inputTokens);
- const outputTokens = getFiniteNumber(usage.outputTokens);
- if (inputTokens === undefined && outputTokens === undefined) {
+ const times = this.toolExecutionTimes.get(toolCallId) ?? {};
+ times.endTime ??= new Date();
+ this.toolExecutionTimes.set(toolCallId, times);
+ }
+
+ // Bedrock's V3 stream emits exact usage before Mastra executes tools. Keep
+ // it on the open step as a fallback because approval suspension can end the
+ // run without Mastra ever invoking onStepFinish.
+ recordModelCallFinish(event: unknown) {
+ if (!isRecord(event)) {
return;
}
- const cacheReadTokens = getFiniteNumber(usage.cachedInputTokens) ?? 0;
- const cacheWriteTokens =
- getFiniteNumber(usage.cacheCreationInputTokens) ?? 0;
- const totals = (this.usageDetails ??= {
- input: 0,
- output: 0,
- cache_read_input_tokens: 0,
- cache_creation_input_tokens: 0,
- total: 0,
- });
+ if (this.ended) {
+ return;
+ }
- // Mastra's inputTokens includes cache reads/writes, while Langfuse model
- // prices bill `input` as non-cached input tokens with separate cache keys.
- totals.input += Math.max(
- 0,
- (inputTokens ?? 0) - cacheReadTokens - cacheWriteTokens,
- );
- totals.output += outputTokens ?? 0;
- totals.cache_read_input_tokens += cacheReadTokens;
- totals.cache_creation_input_tokens += cacheWriteTokens;
- totals.total +=
- getFiniteNumber(usage.totalTokens) ??
- (inputTokens ?? 0) + (outputTokens ?? 0);
+ const now = new Date();
+ this.ensureOpenLlmStep(now);
+ if (this.openLlmStep) {
+ this.openLlmStep.providerFinish = event;
+ this.openLlmStep.providerFinishTime = now;
+ }
+ }
+
+ // Mastra stream chunks: step-start (or first model delta) bookmarks the LLM
+ // window; tool-result ends tool execution. Do not treat tool-call chunks as
+ // execution start — they fire while the model is still streaming.
+ // Usage and output still come from recordStepFinish.
+ recordStreamChunk(chunk: unknown) {
+ if (this.ended || !isRecord(chunk)) {
+ return;
+ }
+
+ const type = chunk.type;
+ if (typeof type !== "string") {
+ return;
+ }
+
+ const payload = isRecord(chunk.payload) ? chunk.payload : {};
+ const now = new Date();
+
+ if (type === "step-start") {
+ // A prior step should have been closed by onStepFinish. If not, emit it
+ // as an error so we never silently drop a generation.
+ if (this.openLlmStep) {
+ const providerFinish = this.openLlmStep.providerFinish;
+ this.emitLlmGeneration(
+ this.openLlmStep,
+ undefined,
+ providerFinish
+ ? undefined
+ : {
+ level: "ERROR",
+ statusMessage: "LLM step closed by a subsequent step-start",
+ },
+ );
+ this.flushDeferredToolCompletions();
+ }
+
+ this.openLlmStep = {
+ stepNumber: ++this.nextStepNumber,
+ startTime: now,
+ inputMessageCount: this.agentRunOutputMessages.length,
+ };
+ return;
+ }
+
+ // tool-call means "model requested this tool", not "tool started".
+ // Ensure an LLM step is open so missing step-start still gets a real window.
+ if (type === "tool-call") {
+ this.ensureOpenLlmStep(now);
+ return;
+ }
+
+ if (type === "tool-result" || type === "tool-error") {
+ const toolCallId = getStringValue(payload.toolCallId);
+ if (!toolCallId) {
+ return;
+ }
+
+ const times = this.toolExecutionTimes.get(toolCallId) ?? {};
+ times.endTime ??= now;
+ this.toolExecutionTimes.set(toolCallId, times);
+ return;
+ }
+
+ if (
+ type === "text-delta" ||
+ type === "reasoning-delta" ||
+ type === "reasoning"
+ ) {
+ this.ensureOpenLlmStep(now);
+ if (this.openLlmStep) {
+ this.openLlmStep.completionStartTime ??= now;
+ }
+ }
+ }
+
+ // Receives Mastra's per-LLM-call onStepFinish event and emits one generation
+ // with that call's usage/model. The AG-UI event stream itself never carries usage.
+ recordStepFinish(event: unknown) {
+ if (this.ended) {
+ return;
+ }
+
+ const step =
+ this.openLlmStep ??
+ ({
+ stepNumber: ++this.nextStepNumber,
+ // Prefer a non-zero window when step-start never arrived: backdate to
+ // the last generation end (or run start) so tools do not sort above us.
+ startTime: this.lastLlmGenerationEndTime ?? this.agentRunStartTime,
+ inputMessageCount: this.agentRunOutputMessages.length,
+ } satisfies OpenLlmStep);
+
+ this.emitLlmGeneration(step, event);
+ this.openLlmStep = undefined;
+ this.flushDeferredToolCompletions();
}
recordAvailableSkills(skills: unknown[]) {
@@ -334,35 +487,22 @@ export class InAppAgentInstrumentation {
}
const message = error instanceof Error ? error.message : String(error);
+ this.endOpenLlmStep({ error: message }, message);
this.endOpenToolSpans({ error: message }, message);
this.flushTrailingThinking();
- this.agentRun.update({
- name: IN_APP_AGENT_TURN_NAME,
- input: this.agentRunInput,
- output: this.getAgentRunOutput(),
- ...(this.completionStartTime
- ? { completionStartTime: this.completionStartTime }
- : {}),
- ...this.getAgentRunUsage(),
+ this.emitRootAgentObservation({
level: "ERROR",
statusMessage: message,
metadata: {
...this.metadata,
error: message,
},
- ...(this.prompt
- ? {
- promptName: this.prompt.name,
- promptVersion: this.prompt.version,
- }
- : {}),
});
this.trace.update({
- input: this.agentRunInput,
+ input: this.getTraceInput(),
output: this.getAgentRunOutput(),
metadata: { ...this.metadata, error: message },
});
- this.agentRun.end();
this.ended = true;
}
@@ -371,6 +511,11 @@ export class InAppAgentInstrumentation {
return;
}
+ if (params?.aborted) {
+ this.endOpenLlmStep({ aborted: true }, "aborted");
+ } else {
+ this.endOpenLlmStep();
+ }
this.endOpenToolSpans(params?.aborted ? { aborted: true } : undefined);
this.flushTrailingThinking();
const metadata = {
@@ -378,28 +523,12 @@ export class InAppAgentInstrumentation {
...(params?.aborted ? { aborted: true } : {}),
...(params?.result ? { result: params.result } : {}),
};
- this.agentRun.update({
- name: IN_APP_AGENT_TURN_NAME,
- input: this.agentRunInput,
- output: this.getAgentRunOutput(),
- ...(this.completionStartTime
- ? { completionStartTime: this.completionStartTime }
- : {}),
- ...this.getAgentRunUsage(),
- metadata,
- ...(this.prompt
- ? {
- promptName: this.prompt.name,
- promptVersion: this.prompt.version,
- }
- : {}),
- });
+ this.emitRootAgentObservation({ metadata });
this.trace.update({
- input: this.agentRunInput,
+ input: this.getTraceInput(),
output: this.getAgentRunOutput(),
metadata,
});
- this.agentRun.end();
this.ended = true;
}
@@ -513,6 +642,8 @@ export class InAppAgentInstrumentation {
const name =
typeof event.toolCallName === "string" ? event.toolCallName : "tool-call";
+ // Execution start = AG-UI tool-call start (or later clamp to generation end).
+ // Mastra tool-call chunks are request-time and must not set this.
this.toolSpans.set(event.toolCallId, {
name,
startTime: new Date(),
@@ -570,6 +701,14 @@ export class InAppAgentInstrumentation {
return;
}
+ // Tool-call args/results often arrive while the LLM step is still open.
+ // Hold the observation until step-finish so the tree sorts generation → tools.
+ if (this.openLlmStep) {
+ this.deferredToolCompletions.set(toolCallId, tool);
+ this.toolSpans.delete(toolCallId);
+ return;
+ }
+
this.createToolObservation(toolCallId, tool);
this.toolSpans.delete(toolCallId);
}
@@ -590,14 +729,28 @@ export class InAppAgentInstrumentation {
tool.failureMessage ??
getToolFailureMessage(undefined, output);
const toolCallApproval = this.toolCallApprovals.get(toolCallId);
- const body: ToolObservationBody = {
+ const executionTimes = this.toolExecutionTimes.get(toolCallId);
+ const rawStartTime = executionTimes?.startTime ?? tool.startTime;
+ const endTime =
+ toolCallApproval === "rejected"
+ ? rawStartTime
+ : (executionTimes?.endTime ?? new Date());
+ // Keep causal order: tool execution cannot start before the generation that
+ // requested it finished (or the last known generation end).
+ const startTime =
+ this.lastLlmGenerationEndTime &&
+ rawStartTime.getTime() < this.lastLlmGenerationEndTime.getTime()
+ ? this.lastLlmGenerationEndTime
+ : rawStartTime;
+ const body: ObservationBody = {
id: toolCallId,
- traceId: this.agentRun.traceId,
- parentObservationId: this.agentRun.observationId,
+ traceId: this.traceId,
+ parentObservationId: this.rootObservationId,
name: tool.name,
- startTime: tool.startTime,
- endTime: new Date(),
- completionStartTime: tool.startTime,
+ startTime: startTime.getTime() > endTime.getTime() ? endTime : startTime,
+ endTime,
+ completionStartTime:
+ startTime.getTime() > endTime.getTime() ? endTime : startTime,
input,
output,
...(failureMessage
@@ -616,38 +769,236 @@ export class InAppAgentInstrumentation {
this.recordToolCall(toolCallId, tool, output);
this.toolCallApprovals.delete(toolCallId);
+ this.toolExecutionTimes.delete(toolCallId);
- (
- this.langfuse as unknown as {
- enqueue: (type: string, body: ToolObservationBody) => void;
- }
- ).enqueue("tool-create", body);
+ this.enqueueObservation("tool-create", body);
}
- // Model is only set alongside provided usage; on its own it would make
- // ingestion tokenize the agent-turn input/output into estimated usage.
- private getAgentRunUsage() {
- if (!this.usageDetails) {
- return {};
+ private emitRootAgentObservation(params: {
+ metadata: Record<string, unknown>;
+ level?: "ERROR";
+ statusMessage?: string;
+ }) {
+ this.enqueueObservation("agent-create", {
+ id: this.rootObservationId,
+ traceId: this.traceId,
+ name: IN_APP_AGENT_TURN_NAME,
+ startTime: this.traceStartTime,
+ endTime: new Date(),
+ input: this.getTraceInput(),
+ output: this.getAgentRunOutput(),
+ metadata: params.metadata,
+ ...(this.prompt
+ ? {
+ promptName: this.prompt.name,
+ promptVersion: this.prompt.version,
+ }
+ : {}),
+ ...(params.level ? { level: params.level } : {}),
+ ...(params.statusMessage ? { statusMessage: params.statusMessage } : {}),
+ });
+ }
+
+ private emitLlmGeneration(
+ step: OpenLlmStep,
+ event: unknown,
+ options?: {
+ level?: "ERROR";
+ statusMessage?: string;
+ },
+ ) {
+ const eventRecord = isRecord(event) ? event : undefined;
+ const providerFinish = step.providerFinish;
+ const usageDetails =
+ getStepUsageDetails(eventRecord?.usage) ??
+ getStepUsageDetails(providerFinish?.usage);
+ const finishReason =
+ getStepFinishReason(eventRecord) ?? getStepFinishReason(providerFinish);
+ const model =
+ getModelIdFromStepEvent(eventRecord) ?? this.model ?? undefined;
+ // Mastra can delay onStepFinish until after requested tools execute. The
+ // raw provider finish is the actual model boundary and keeps the following
+ // tool's execute() window intact instead of clamping it to zero duration.
+ const endTime = step.providerFinishTime ?? new Date();
+ this.lastLlmGenerationEndTime = endTime;
+ const id = getInAppAgentLlmCallObservationId(this.runId, step.stepNumber);
+
+ this.enqueueObservation("generation-create", {
+ id,
+ traceId: this.traceId,
+ parentObservationId: this.rootObservationId,
+ name: IN_APP_AGENT_LLM_CALL_NAME,
+ startTime: step.startTime,
+ endTime,
+ completionStartTime: step.completionStartTime ?? step.startTime,
+ input: this.getLlmStepInput(step.inputMessageCount),
+ output: this.getLlmStepOutput(step, eventRecord),
+ // Model only travels with usage so Langfuse does not invent tokenizer-
+ // estimated costs for generations that never reported tokens.
+ ...(usageDetails
+ ? {
+ usageDetails,
+ ...(model ? { model } : {}),
+ }
+ : {}),
+ ...(options?.level ? { level: options.level } : {}),
+ ...(options?.statusMessage
+ ? { statusMessage: options.statusMessage }
+ : {}),
+ metadata: {
+ ...(finishReason ? { finish_reason: finishReason } : {}),
+ ...(options?.statusMessage ? { error: options.statusMessage } : {}),
+ },
+ });
+ }
+
+ private getLlmStepInput(inputMessageCount: number) {
+ const base = isRecord(this.agentRunInput) ? this.agentRunInput : {};
+ const baseMessages = Array.isArray(base.messages) ? base.messages : [];
+
+ return {
+ ...base,
+ messages: [
+ ...baseMessages,
+ ...this.agentRunOutputMessages.slice(0, inputMessageCount),
+ ],
+ };
+ }
+
+ private getTraceInput() {
+ if (!this.approvalContinuation || !isRecord(this.agentRunInput)) {
+ return this.agentRunInput;
}
+ const messages = Array.isArray(this.agentRunInput.messages)
+ ? this.agentRunInput.messages.filter((message) => {
+ if (!isRecord(message)) {
+ return false;
+ }
+
+ return (
+ message.role === "developer" ||
+ message.role === "system" ||
+ message.role === "user"
+ );
+ })
+ : [];
+
return {
- usageDetails: this.usageDetails,
- ...(this.model ? { model: this.model } : {}),
+ ...this.agentRunInput,
+ messages,
};
}
+ private getLlmStepOutput(
+ step: OpenLlmStep,
+ event: Record<string, unknown> | undefined,
+ ) {
+ const text = typeof event?.text === "string" ? event.text : undefined;
+ const toolCalls = Array.isArray(event?.toolCalls)
+ ? event.toolCalls
+ : undefined;
+ if (text !== undefined || toolCalls?.length) {
+ return {
+ ...(text !== undefined ? { text } : {}),
+ ...(toolCalls?.length ? { tool_calls: toolCalls } : {}),
+ };
+ }
+
+ const response = isRecord(event?.response) ? event.response : undefined;
+ if (Array.isArray(response?.messages) && response.messages.length > 0) {
+ return { messages: response.messages };
+ }
+
+ const messages = this.agentRunOutputMessages.slice(step.inputMessageCount);
+ return messages.length > 0 ? { messages } : undefined;
+ }
+
+ private endOpenLlmStep(
+ metadata?: Record<string, unknown>,
+ statusMessage?: string,
+ ) {
+ if (!this.openLlmStep) {
+ return;
+ }
+
+ this.emitLlmGeneration(this.openLlmStep, undefined, {
+ ...(!this.openLlmStep.providerFinish && (statusMessage || metadata)
+ ? {
+ level: "ERROR" as const,
+ statusMessage:
+ statusMessage ??
+ (typeof metadata?.error === "string"
+ ? metadata.error
+ : "LLM step closed before step finish"),
+ }
+ : {}),
+ });
+ this.openLlmStep = undefined;
+ this.flushDeferredToolCompletions();
+ }
+
private endOpenToolSpans(
metadata?: Record<string, unknown>,
statusMessage?: string,
) {
+ // Approval interrupts complete the stream without a tool result. Drop those
+ // incomplete spans — the generation already records the tool_call request,
+ // and the continuation run emits the real tool observation after approval.
+ const forceClose =
+ Boolean(statusMessage) ||
+ Boolean(metadata?.aborted) ||
+ Boolean(metadata?.error);
+
for (const [toolCallId, tool] of this.toolSpans.entries()) {
+ if (tool.output === undefined && !forceClose) {
+ this.toolSpans.delete(toolCallId);
+ this.toolExecutionTimes.delete(toolCallId);
+ continue;
+ }
+
this.createToolObservation(toolCallId, tool, {
metadata,
statusMessage,
});
this.toolSpans.delete(toolCallId);
}
+
+ // endOpenLlmStep() has already drained completed tools before this method
+ // runs; keep the unconditional flush to preserve that lifecycle invariant.
+ this.flushDeferredToolCompletions();
+ }
+
+ private ensureOpenLlmStep(now: Date) {
+ if (this.openLlmStep) {
+ return;
+ }
+
+ this.openLlmStep = {
+ stepNumber: ++this.nextStepNumber,
+ startTime: now,
+ inputMessageCount: this.agentRunOutputMessages.length,
+ };
+ }
+
+ private flushDeferredToolCompletions() {
+ if (this.deferredToolCompletions.size === 0) {
+ return;
+ }
+
+ const deferred = [...this.deferredToolCompletions.entries()];
+ this.deferredToolCompletions.clear();
+ for (const [toolCallId, tool] of deferred) {
+ this.createToolObservation(toolCallId, tool);
+ }
+ }
+
+ private enqueueObservation(type: string, body: ObservationBody) {
+ (
+ this.langfuse as unknown as {
+ enqueue: (type: string, body: ObservationBody) => void;
+ }
+ ).enqueue(type, body);
}
// Reasoning that no assistant message followed (e.g. aborted or errored
@@ -691,7 +1042,6 @@ export class InAppAgentInstrumentation {
private recordAssistantText(delta: string) {
this.output += delta;
- this.completionStartTime ??= new Date();
const thinking = this.takePendingThinking();
const lastMessage = this.agentRunOutputMessages.at(-1);
@@ -720,8 +1070,6 @@ export class InAppAgentInstrumentation {
},
output: unknown,
) {
- this.completionStartTime ??= tool.startTime;
-
const toolCall: AgentRunToolCall = {
id: toolCallId,
name: tool.name,
@@ -748,18 +1096,121 @@ export class InAppAgentInstrumentation {
}
private getAgentRunOutput() {
- if (this.agentRunOutputMessages.length === 0) {
+ const priorMessages =
+ this.approvalContinuation && isRecord(this.agentRunInput)
+ ? getPriorAgentRunOutputMessages(this.agentRunInput.messages)
+ : [];
+ const messages = [...priorMessages, ...this.agentRunOutputMessages];
+
+ if (messages.length === 0) {
return undefined;
}
+ const priorToolCalls = priorMessages.flatMap(
+ (message) => message.tool_calls ?? [],
+ );
+ const toolCalls = [...priorToolCalls, ...this.agentRunToolCalls];
+
return {
- messages: this.agentRunOutputMessages,
+ messages,
...(this.output ? { text: this.output } : {}),
- ...(this.agentRunToolCalls.length > 0
- ? { tool_calls: this.agentRunToolCalls }
- : {}),
+ ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}),
};
}
+
+ private emitApprovalWaitObservation() {
+ const continuation = this.approvalContinuation;
+ if (!continuation) {
+ return;
+ }
+
+ const requestedAt =
+ continuation.approvalRequestedAt ??
+ continuation.approvalDecidedAt ??
+ this.agentRunStartTime;
+ const decidedAt = continuation.approvalDecidedAt ?? this.agentRunStartTime;
+ const endTime =
+ decidedAt.getTime() < requestedAt.getTime() ? requestedAt : decidedAt;
+
+ this.enqueueObservation("span-create", {
+ id: `${this.runId}-approval-wait`,
+ traceId: this.traceId,
+ parentObservationId: this.rootObservationId,
+ name: `waiting for user approval for tool ${continuation.toolName}`,
+ startTime: requestedAt,
+ endTime,
+ input: {
+ tool_call_id: continuation.toolCallId,
+ tool_name: continuation.toolName,
+ },
+ output: `User ${continuation.status} tool ${continuation.toolName}`,
+ metadata: continuation.metadata,
+ });
+ }
+}
+
+function getApprovalContinuation(
+ input: AgUiRunAgentInput,
+): ApprovalContinuation | undefined {
+ const forwardedProps = ResumeForwardedPropsSchema.safeParse(
+ input.forwardedProps,
+ );
+
+ if (!forwardedProps.success) {
+ return undefined;
+ }
+
+ const {
+ approved,
+ approvalRequest,
+ continuationNumber = 1,
+ rootRunId = approvalRequest.runId,
+ traceStartedAt,
+ approvalRequestedAt,
+ approvalDecidedAt,
+ } = forwardedProps.data.command.resume;
+ const status = approved ? "approved" : "rejected";
+
+ return {
+ status,
+ continuationNumber,
+ rootRunId,
+ toolCallId: approvalRequest.toolCallId,
+ toolName: approvalRequest.toolName,
+ ...(traceStartedAt ? { traceStartedAt: new Date(traceStartedAt) } : {}),
+ ...(approvalRequestedAt
+ ? { approvalRequestedAt: new Date(approvalRequestedAt) }
+ : {}),
+ ...(approvalDecidedAt
+ ? { approvalDecidedAt: new Date(approvalDecidedAt) }
+ : {}),
+ metadata: {
+ continuation_type: "tool_approval",
+ continuation_number: continuationNumber,
+ parent_run_id: approvalRequest.runId,
+ approval_status: status,
+ approval_tool_call_id: approvalRequest.toolCallId,
+ approval_tool_name: approvalRequest.toolName,
+ ...(approvalRequestedAt
+ ? { approval_requested_at: approvalRequestedAt }
+ : {}),
+ ...(approvalDecidedAt ? { approval_decided_at: approvalDecidedAt } : {}),
+ },
+ };
+}
+
+function getPriorAgentRunOutputMessages(
+ messages: unknown,
+): AgentRunChatMessage[] {
+ if (!Array.isArray(messages)) {
+ return [];
+ }
+
+ return messages.filter(
+ (message): message is AgentRunChatMessage =>
+ isRecord(message) &&
+ (message.role === "assistant" || message.role === "tool"),
+ );
}
function getAgentRunInput(input: AgUiRunAgentInput): unknown {
@@ -970,6 +1421,98 @@ function getFiniteNumber(value: unknown): number | undefined {
: undefined;
}
+function getStepUsageDetails(
+ usage: unknown,
+): Record<string, number> | undefined {
+ if (!isRecord(usage)) {
+ return undefined;
+ }
+
+ const nestedInputTokens = isRecord(usage.inputTokens)
+ ? usage.inputTokens
+ : undefined;
+ const nestedOutputTokens = isRecord(usage.outputTokens)
+ ? usage.outputTokens
+ : undefined;
+ const inputTokens =
+ getFiniteNumber(usage.inputTokens) ??
+ getFiniteNumber(nestedInputTokens?.total);
+ const outputTokens =
+ getFiniteNumber(usage.outputTokens) ??
+ getFiniteNumber(nestedOutputTokens?.total);
+ if (inputTokens === undefined && outputTokens === undefined) {
+ return undefined;
+ }
+
+ const cacheReadTokens =
+ getFiniteNumber(usage.cachedInputTokens) ??
+ getFiniteNumber(nestedInputTokens?.cacheRead) ??
+ 0;
+ const cacheWriteTokens =
+ getFiniteNumber(usage.cacheCreationInputTokens) ??
+ getFiniteNumber(nestedInputTokens?.cacheWrite) ??
+ 0;
+ const uncachedInputTokens = getFiniteNumber(nestedInputTokens?.noCache);
+
+ // Mastra's inputTokens includes cache reads/writes, while Langfuse model
+ // prices bill `input` as non-cached input tokens with separate cache keys.
+ return {
+ input:
+ uncachedInputTokens ??
+ Math.max(0, (inputTokens ?? 0) - cacheReadTokens - cacheWriteTokens),
+ output: outputTokens ?? 0,
+ cache_read_input_tokens: cacheReadTokens,
+ cache_creation_input_tokens: cacheWriteTokens,
+ total:
+ getFiniteNumber(usage.totalTokens) ??
+ (inputTokens ?? 0) + (outputTokens ?? 0),
+ };
+}
+
+function getStepFinishReason(
+ event: Record<string, unknown> | undefined,
+): string | undefined {
+ if (!event) {
+ return undefined;
+ }
+
+ const finishReason = event.finishReason;
+ if (typeof finishReason === "string") {
+ return finishReason;
+ }
+
+ if (isRecord(finishReason)) {
+ return (
+ getStringValue(finishReason.unified) ?? getStringValue(finishReason.raw)
+ );
+ }
+
+ return isRecord(event.stepResult)
+ ? getStringValue(event.stepResult.reason)
+ : undefined;
+}
+
+function getModelIdFromStepEvent(
+ event: Record<string, unknown> | undefined,
+): string | undefined {
+ if (!event) {
+ return undefined;
+ }
+
+ if (isRecord(event.model)) {
+ const modelId = getStringValue(event.model.modelId);
+ if (modelId) {
+ return modelId;
+ }
+ }
+
+ if (isRecord(event.response)) {
+ return getStringValue(event.response.modelId);
+ }
+
+ return undefined;
+}
+
function getSerializableToolParameters(tool: Record<string, unknown>): unknown {
const parameters =
tool.parameters ??
diff --git a/packages/shared/src/in-app-agent/server/runLifecycle.ts b/packages/shared/src/in-app-agent/server/runLifecycle.ts
index 702f2abf1e..d0d7c4bcbb 100644
--- a/packages/shared/src/in-app-agent/server/runLifecycle.ts
+++ b/packages/shared/src/in-app-agent/server/runLifecycle.ts
@@ -219,7 +219,13 @@ export async function decideToolApproval(params: {
projectId: params.projectId,
conversationId: params.conversationId,
},
- select: { status: true, finishedAt: true, request: true },
+ select: {
+ status: true,
+ finishedAt: true,
+ claimedAt: true,
+ createdAt: true,
+ request: true,
+ },
});
if (
@@ -271,6 +277,21 @@ export async function decideToolApproval(params: {
const parentRequest = InAppAgentRunRequestSchema.safeParse(
parentRun.request,
);
+ // Preserve tracing lineage across durable approval continuation runs.
+ const continuationNumber =
+ parentRequest.success && parentRequest.data.kind === "approvalDecision"
+ ? (parentRequest.data.continuationNumber ?? 1) + 1
+ : 1;
+ const rootRunId =
+ parentRequest.success && parentRequest.data.kind === "approvalDecision"
+ ? (parentRequest.data.rootRunId ?? params.parentRunId)
+ : params.parentRunId;
+ const traceStartedAt =
+ parentRequest.success &&
+ parentRequest.data.kind === "approvalDecision" &&
+ parentRequest.data.traceStartedAt
+ ? parentRequest.data.traceStartedAt
+ : (parentRun.claimedAt ?? parentRun.createdAt).toISOString();
// Persist the grant in the same transaction as the exactly-once decision CAS.
if (params.alwaysAllowToolName) {
@@ -325,6 +346,12 @@ export async function decideToolApproval(params: {
request: {
kind: "approvalDecision",
parentRunId: params.parentRunId,
+ rootRunId,
+ traceStartedAt,
+ ...(parentRun.finishedAt
+ ? { approvalRequestedAt: parentRun.finishedAt.toISOString() }
+ : {}),
+ continuationNumber,
toolCallId: params.toolCallId,
approved: params.approved,
context: parentRequest.success ? parentRequest.data.context : [],
diff --git a/packages/shared/src/in-app-agent/server/tools.ts b/packages/shared/src/in-app-agent/server/tools.ts
index 86b641d6da..6dfc293622 100644
--- a/packages/shared/src/in-app-agent/server/tools.ts
+++ b/packages/shared/src/in-app-agent/server/tools.ts
@@ -776,7 +776,7 @@ export function withOptionalSilentMcpOutput(params: {
) as Record<string, Tool>;
}
-function getToolCallId(context: unknown) {
+export function getToolCallId(context: unknown) {
if (
typeof context !== "object" ||
context === null ||
diff --git a/web/src/__tests__/server/in-app-agent-background-run.servertest.ts b/web/src/__tests__/server/in-app-agent-background-run.servertest.ts
index cbb2ba6ce1..cb0a13f128 100644
--- a/web/src/__tests__/server/in-app-agent-background-run.servertest.ts
+++ b/web/src/__tests__/server/in-app-agent-background-run.servertest.ts
@@ -29,6 +29,7 @@ import { inAppAgentRouter } from "@/src/features/in-app-agent/server/router";
import { createInnerTRPCContext } from "@/src/server/api/trpc";
import type * as SharedServerModule from "@langfuse/shared/src/server";
+import type * as PersistenceModule from "@langfuse/shared/in-app-agent/server/persistence";
const enqueuedJobs: Array<{ name: string; payload: unknown; jobId?: string }> =
[];
@@ -38,6 +39,22 @@ const rateLimitMocks = vi.hoisted(() => ({
rateLimitRequest: vi.fn(),
}));
+const persistenceMocks = vi.hoisted(() => ({
+ maybeInferAndPersistConversationTitle: vi.fn().mockResolvedValue(undefined),
+}));
+
+vi.mock("@langfuse/shared/in-app-agent/server/persistence", async () => {
+ const actual = await vi.importActual<typeof PersistenceModule>(
+ "@langfuse/shared/in-app-agent/server/persistence",
+ );
+
+ return {
+ ...actual,
+ maybeInferAndPersistConversationTitle:
+ persistenceMocks.maybeInferAndPersistConversationTitle,
+ };
+});
+
vi.mock("@langfuse/shared/src/server", async () => {
const actual = await vi.importActual<typeof SharedServerModule>(
"@langfuse/shared/src/server",
@@ -85,6 +102,7 @@ describe("in-app agent background runs", () => {
(env as any).LANGFUSE_AWS_BEDROCK_MODEL = "test-model";
enqueuedJobs.length = 0;
enqueueShouldFail = false;
+ persistenceMocks.maybeInferAndPersistConversationTitle.mockClear();
rateLimitMocks.rateLimitRequest.mockResolvedValue({
isRateLimited: () => false,
res: undefined,
@@ -92,6 +110,7 @@ describe("in-app agent background runs", () => {
});
afterEach(() => {
+ vi.useRealTimers();
(env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;
(env as any).LANGFUSE_AWS_BEDROCK_MODEL = originalBedrockModel;
(sharedEnv as any).LANGFUSE_IN_APP_AGENT_MAX_ACTIVE_RUNS_PER_USER =
@@ -289,6 +308,9 @@ describe("in-app agent background runs", () => {
}),
}),
]);
+ expect(
+ persistenceMocks.maybeInferAndPersistConversationTitle,
+ ).toHaveBeenCalledOnce();
});
it("rejects a second submit while a run is active without duplicating anything", async () => {
@@ -576,6 +598,8 @@ describe("in-app agent background runs", () => {
});
it("decides an approval exactly once and reads the tool args server-side", async () => {
+ vi.useFakeTimers({ toFake: ["Date"] });
+ vi.setSystemTime(new Date("2026-08-14T10:01:00.000Z"));
const { caller, projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
const context = [
@@ -585,12 +609,14 @@ describe("in-app agent background runs", () => {
},
{ description: "user_name", value: "Agent User" },
];
+ const parkedAt = new Date("2026-08-14T10:00:05.000Z");
const parkedRunId = await parkRunForApproval({
projectId,
conversationId: conversation.id,
userId,
toolCallId: "tool-call-1",
context,
+ parkedAt,
});
const { runId: continuationRunId } = await caller.decideToolApproval({
@@ -615,6 +641,10 @@ describe("in-app agent background runs", () => {
expect(continuation.request).toMatchObject({
kind: "approvalDecision",
parentRunId: parkedRunId,
+ rootRunId: parkedRunId,
+ traceStartedAt: parkedRun.createdAt.toISOString(),
+ approvalRequestedAt: parkedAt.toISOString(),
+ continuationNumber: 1,
toolCallId: "tool-call-1",
approved: true,
context,
@@ -655,6 +685,8 @@ describe("in-app agent background runs", () => {
});
it("preserves context through an approved then rejected chained continuation", async () => {
+ vi.useFakeTimers({ toFake: ["Date"] });
+ vi.setSystemTime(new Date("2026-08-14T10:01:00.000Z"));
const { caller, projectId, userId } = await createCaller();
const conversation = await createConversation({ projectId, userId });
const context = [
@@ -681,13 +713,26 @@ describe("in-app agent background runs", () => {
approved: true,
});
+ const approvedContinuation = await prisma.inAppAgentRun.findFirstOrThrow({
+ where: { id: approvedContinuationId, projectId },
+ });
+ expect(approvedContinuation.request).toMatchObject({
+ kind: "approvalDecision",
+ parentRunId: initialRunId,
+ rootRunId: initialRunId,
+ traceStartedAt: expect.any(String),
+ approvalRequestedAt: expect.any(String),
+ continuationNumber: 1,
+ });
+
+ const secondParkedAt = new Date("2026-08-14T10:03:00.000Z");
await prisma.inAppAgentRun.update({
where: {
id_projectId: { id: approvedContinuationId, projectId },
},
data: {
status: InAppAgentRunStatus.AWAITING_APPROVAL,
- finishedAt: new Date(),
+ finishedAt: secondParkedAt,
},
});
await prisma.inAppAgentEvent.create({
@@ -710,6 +755,7 @@ describe("in-app agent background runs", () => {
},
},
});
+ vi.setSystemTime(new Date("2026-08-14T10:04:00.000Z"));
const { runId: rejectedContinuationId } = await caller.decideToolApproval({
projectId,
@@ -725,6 +771,11 @@ describe("in-app agent background runs", () => {
expect(rejectedContinuation.request).toMatchObject({
kind: "approvalDecision",
parentRunId: approvedContinuationId,
+ rootRunId: initialRunId,
+ traceStartedAt: (approvedContinuation.request as Record<string, unknown>)
+ .traceStartedAt,
+ approvalRequestedAt: secondParkedAt.toISOString(),
+ continuationNumber: 2,
toolCallId: "tool-call-2",
approved: false,
context,
diff --git a/web/src/__tests__/server/in-app-agent-stream.servertest.ts b/web/src/__tests__/server/in-app-agent-stream.servertest.ts
index cba3980229..0bf7b6714a 100644
--- a/web/src/__tests__/server/in-app-agent-stream.servertest.ts
+++ b/web/src/__tests__/server/in-app-agent-stream.servertest.ts
@@ -76,12 +76,31 @@ const adapterEvents = vi.hoisted(() => ({
}),
}));
+const bedrockMocks = vi.hoisted(() => ({
+ streamParts: [] as unknown[],
+ doGenerate: vi.fn(),
+ doStream: vi.fn(async () => ({
+ stream: new ReadableStream({
+ start(controller) {
+ for (const part of bedrockMocks.streamParts) {
+ controller.enqueue(part);
+ }
+ controller.close();
+ },
+ }),
+ })),
+}));
+
const instrumentationMocks = vi.hoisted(() => {
const instrumentation = {
recordEvents: vi.fn(),
recordAvailableTools: vi.fn(),
recordToolCallApproval: vi.fn(),
+ recordToolExecutionStart: vi.fn(),
+ recordToolExecutionEnd: vi.fn(),
+ recordModelCallFinish: vi.fn(),
recordStepFinish: vi.fn(),
+ recordStreamChunk: vi.fn(),
end: vi.fn(),
endWithError: vi.fn(),
flush: vi.fn(),
@@ -206,7 +225,14 @@ vi.mock("@ag-ui/mastra", () => ({
}));
vi.mock("ai-sdk-amazon-bedrock-v4", () => ({
- createAmazonBedrock: vi.fn(() => vi.fn(() => ({}))),
+ createAmazonBedrock: vi.fn(() => (modelId: string) => ({
+ specificationVersion: "v3",
+ provider: "amazon-bedrock",
+ modelId,
+ supportedUrls: {},
+ doGenerate: bedrockMocks.doGenerate,
+ doStream: bedrockMocks.doStream,
+ })),
}));
vi.mock("@aws-sdk/credential-providers", () => ({
@@ -467,6 +493,7 @@ describe("IN_APP_AGENT_LANGFUSE_MCP_TOOL_POLICIES", () => {
describe("createAgUiStream", () => {
beforeEach(() => {
vi.clearAllMocks();
+ bedrockMocks.streamParts = [];
promptMocks.getPrompt.mockResolvedValue({
name: "in-app-agent-system-prompt",
version: 2,
@@ -474,6 +501,95 @@ describe("createAgUiStream", () => {
});
});
+ const initializeBasicTracedAgent = async (runId: string) => {
+ const { createAgUiStream } =
+ await import("@langfuse/shared/in-app-agent/server/agent");
+ const input = {
+ threadId: "conversation-1",
+ runId,
+ messages: [
+ {
+ id: "user-message-1",
+ role: "user" as const,
+ content: "hello",
+ },
+ ],
+ tools: [],
+ context: [],
+ state: null,
+ forwardedProps: {},
+ };
+ adapterEvents.items = [
+ {
+ type: EventType.RUN_STARTED,
+ threadId: input.threadId,
+ runId: input.runId,
+ },
+ {
+ type: EventType.RUN_FINISHED,
+ threadId: input.threadId,
+ runId: input.runId,
+ },
+ ];
+
+ const stream = await createAgUiStream({
+ input,
+ signal: new AbortController().signal,
+ options: {
+ awsBedrock: { modelId: "test-model" },
+ langfuseMcp: {
+ url: "https://example.com/api/public/mcp",
+ publicKey: "pk",
+ secretKey: "sk",
+ toolPolicy: defaultInAppAgentToolPolicy,
+ },
+ redirectAction: { projectId: "project-1", isV4Enabled: false },
+ langfuseClient: {
+ getPrompt: promptMocks.getPrompt,
+ } as unknown as Langfuse,
+ useLocalPrompt: false,
+ langfuseTracing: createTestTracingConfig(),
+ },
+ });
+ await readStream(stream);
+ };
+
+ it("forwards model stream parts when finish tracing throws", async () => {
+ instrumentationMocks.instrumentation.recordModelCallFinish.mockImplementationOnce(
+ () => {
+ throw new Error("tracing failed");
+ },
+ );
+ await initializeBasicTracedAgent("run-provider-finish-tracing-error");
+
+ const textPart = { type: "text-delta", id: "text-1", delta: "hello" };
+ const finishPart = {
+ type: "finish",
+ usage: {
+ inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 },
+ outputTokens: { total: 2, text: 2, reasoning: 0 },
+ },
+ finishReason: { unified: "stop", raw: "end_turn" },
+ };
+ bedrockMocks.streamParts = [textPart, finishPart];
+ const model = vi.mocked(Agent).mock.calls.at(-1)?.[0]?.model as unknown as {
+ doStream: (options: unknown) => Promise<{
+ stream: ReadableStream<unknown>;
+ }>;
+ };
+
+ const modelResult = await model.doStream({});
+ const forwardedParts: unknown[] = [];
+ for await (const part of modelResult.stream) {
+ forwardedParts.push(part);
+ }
+
+ expect(forwardedParts).toEqual([textPart, finishPart]);
+ expect(
+ instrumentationMocks.instrumentation.recordModelCallFinish,
+ ).toHaveBeenCalledWith(finishPart);
+ });
+
it("serializes valid events including adapter snapshots and reasoning messages", async () => {
const { createAgUiStream } =
await import("@langfuse/shared/in-app-agent/server/agent");
@@ -829,6 +945,16 @@ describe("createAgUiStream", () => {
expect(
instrumentationMocks.instrumentation.recordStepFinish,
).toHaveBeenCalledWith({ usage: { inputTokens: 10, outputTokens: 5 } });
+ const onChunk = (
+ agentConfig?.defaultOptions as
+ | { onChunk?: (chunk: unknown) => void }
+ | undefined
+ )?.onChunk;
+ expect(onChunk).toEqual(expect.any(Function));
+ onChunk?.({ type: "step-start", payload: {} });
+ expect(
+ instrumentationMocks.instrumentation.recordStreamChunk,
+ ).toHaveBeenCalledWith({ type: "step-start", payload: {} });
expect(
instrumentationMocks.instrumentation.recordEvents.mock.calls.flatMap(
([events]) => (events as AgUiEvent[]).map((event) => event.type),
@@ -1336,6 +1462,24 @@ describe("createAgUiStream", () => {
toolCallId: "tool-call-1",
status: "approved",
});
+ expect(
+ instrumentationMocks.instrumentation.recordToolExecutionStart,
+ ).toHaveBeenCalledWith("tool-call-1");
+ expect(
+ instrumentationMocks.instrumentation.recordToolExecutionEnd,
+ ).toHaveBeenCalledWith("tool-call-1");
+ expect(
+ instrumentationMocks.instrumentation.recordToolExecutionStart.mock
+ .invocationCallOrder[0],
+ ).toBeLessThan(
+ adapterEvents.createScoreConfigExecute.mock.invocationCallOrder[0]!,
+ );
+ expect(
+ adapterEvents.createScoreConfigExecute.mock.invocationCallOrder[0],
+ ).toBeLessThan(
+ instrumentationMocks.instrumentation.recordToolExecutionEnd.mock
+ .invocationCallOrder[0]!,
+ );
expect(adapterEvents.createScoreConfigExecute).toHaveBeenCalledWith(
{
@@ -1354,6 +1498,73 @@ describe("createAgUiStream", () => {
);
});
+ it("preserves successful and failed tool outcomes when timing tracing throws", async () => {
+ const tracingError = new Error("tool timing tracing failed");
+ instrumentationMocks.instrumentation.recordToolExecutionStart
+ .mockImplementationOnce(() => {
+ throw tracingError;
+ })
+ .mockImplementationOnce(() => {
+ throw tracingError;
+ });
+ instrumentationMocks.instrumentation.recordToolExecutionEnd
+ .mockImplementationOnce(() => {
+ throw tracingError;
+ })
+ .mockImplementationOnce(() => {
+ throw tracingError;
+ });
+ await initializeBasicTracedAgent("run-tool-timing-tracing-error");
+
+ const tool = getAgentTools(
+ vi.mocked(Agent).mock.calls.at(-1)?.[0],
+ )?.langfuse_createScoreConfig;
+ const execute = tool?.execute;
+ expect(execute).toBeTypeOf("function");
+ const toolInput = {
+ name: "readiness",
+ dataType: "NUMERIC",
+ numericMinValue: 0,
+ numericMaxValue: 1,
+ };
+ const toolContext = {
+ abortSignal: new AbortController().signal,
+ agent: {
+ toolCallId: "tool-call-1",
+ threadId: "conversation-1",
+ },
+ };
+
+ await expect(execute?.(toolInput, toolContext)).resolves.toEqual({
+ id: "score-config-1",
+ name: "readiness",
+ dataType: "NUMERIC",
+ });
+
+ const originalToolError = new Error("original tool failure");
+ adapterEvents.createScoreConfigExecute.mockRejectedValueOnce(
+ originalToolError,
+ );
+ await expect(execute?.(toolInput, toolContext)).rejects.toBe(
+ originalToolError,
+ );
+
+ const startOrder =
+ instrumentationMocks.instrumentation.recordToolExecutionStart.mock
+ .invocationCallOrder;
+ const executeOrder =
+ adapterEvents.createScoreConfigExecute.mock.invocationCallOrder;
+ const endOrder =
+ instrumentationMocks.instrumentation.recordToolExecutionEnd.mock
+ .invocationCallOrder;
+ expect(startOrder).toHaveLength(2);
+ expect(endOrder).toHaveLength(2);
+ expect(startOrder[0]).toBeLessThan(executeOrder[0]!);
+ expect(executeOrder[0]).toBeLessThan(endOrder[0]!);
+ expect(startOrder[1]).toBeLessThan(executeOrder[1]!);
+ expect(executeOrder[1]).toBeLessThan(endOrder[1]!);
+ });
+
it("persists raw silent output for approved tools while resuming with redacted content", async () => {
const { createAgUiStream } =
await import("@langfuse/shared/in-app-agent/server/agent");
@@ -1481,11 +1692,30 @@ describe("createAgUiStream", () => {
},
langfuseClient,
useLocalPrompt: false,
+ langfuseTracing: createTestTracingConfig(),
},
});
await readStream(stream);
expect(onError).not.toHaveBeenCalled();
+ expect(
+ instrumentationMocks.instrumentation.recordToolExecutionStart,
+ ).toHaveBeenCalledWith("tool-call-1");
+ expect(
+ instrumentationMocks.instrumentation.recordToolExecutionEnd,
+ ).toHaveBeenCalledWith("tool-call-1");
+ expect(
+ instrumentationMocks.instrumentation.recordToolExecutionStart.mock
+ .invocationCallOrder[0],
+ ).toBeLessThan(
+ adapterEvents.createScoreConfigExecute.mock.invocationCallOrder[0]!,
+ );
+ expect(
+ adapterEvents.createScoreConfigExecute.mock.invocationCallOrder[0],
+ ).toBeLessThan(
+ instrumentationMocks.instrumentation.recordToolExecutionEnd.mock
+ .invocationCallOrder[0]!,
+ );
const resumedMessages =
(
adapterEvents.inputs[0] as {
diff --git a/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts b/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts
index 9d768af2e7..e55b563398 100644
--- a/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts
+++ b/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts
@@ -3,6 +3,7 @@ import { EventType } from "@ag-ui/core";
import {
getInAppAgentInstrumentationObservationId,
getInAppAgentInstrumentationTraceId,
+ getInAppAgentLlmCallObservationId,
} from "@langfuse/shared/in-app-agent";
import type { AgUiRunAgentInput } from "@langfuse/shared/in-app-agent";
import { InAppAgentInstrumentation } from "@langfuse/shared/in-app-agent/server/instrumentation";
@@ -12,14 +13,8 @@ const traceId = getInAppAgentInstrumentationTraceId(runId);
const agentRunObservationId = getInAppAgentInstrumentationObservationId(runId);
const mocks = vi.hoisted(() => {
- const agentGeneration = {
- observationId: "run-1",
- traceId: "run-1-trace",
- update: vi.fn(),
- end: vi.fn(),
- };
const trace = {
- generation: vi.fn(() => agentGeneration),
+ generation: vi.fn(),
update: vi.fn(),
};
const handler = {
@@ -30,7 +25,6 @@ const mocks = vi.hoisted(() => {
};
return {
- agentGeneration,
trace,
handler,
processTracedEvents: vi.fn(async () => undefined),
@@ -153,13 +147,7 @@ describe("InAppAgentInstrumentation", () => {
input: expectedAgentRunInput,
}),
);
- expect(mocks.trace.generation).toHaveBeenCalledWith(
- expect.objectContaining({
- id: agentRunObservationId,
- name: "agent-turn",
- input: expectedAgentRunInput,
- }),
- );
+ expect(mocks.trace.generation).not.toHaveBeenCalled();
expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
"tool-create",
expect.objectContaining({
@@ -179,10 +167,15 @@ describe("InAppAgentInstrumentation", () => {
}),
);
expect(
- mocks.handler.langfuse.enqueue.mock.calls[0]?.[1].metadata,
+ mocks.handler.langfuse.enqueue.mock.calls.find(
+ ([type]) => type === "tool-create",
+ )?.[1].metadata,
).not.toHaveProperty("toolCallApproval");
- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "agent-create",
expect.objectContaining({
+ id: agentRunObservationId,
+ traceId,
name: "agent-turn",
input: expectedAgentRunInput,
output: {
@@ -202,9 +195,13 @@ describe("InAppAgentInstrumentation", () => {
text: "hi there",
tool_calls: [toolCall],
},
- completionStartTime: expect.any(Date),
}),
);
+ const agentCreateBody = mocks.handler.langfuse.enqueue.mock.calls.find(
+ ([type]) => type === "agent-create",
+ )?.[1];
+ expect(agentCreateBody).not.toHaveProperty("usageDetails");
+ expect(agentCreateBody).not.toHaveProperty("model");
expect(mocks.handler.langfuse.enqueue).not.toHaveBeenCalledWith(
"span-create",
expect.anything(),
@@ -233,15 +230,59 @@ describe("InAppAgentInstrumentation", () => {
);
});
- it.each(["approved", "rejected"] as const)(
- "records manual tool approval metadata as %s",
- (status) => {
+ it("records approved manual tool approval metadata", () => {
+ const instrumentation = createInstrumentation();
+
+ instrumentation.recordToolCallApproval({
+ toolCallId: "tool-1",
+ status: "approved",
+ });
+ instrumentation.recordEvents([
+ {
+ type: EventType.TOOL_CALL_START,
+ toolCallId: "tool-1",
+ toolCallName: "createScoreConfig",
+ parentMessageId: "tool-1-approval-tool-call",
+ },
+ {
+ type: EventType.TOOL_CALL_ARGS,
+ toolCallId: "tool-1",
+ delta: '{"name":"readiness"}',
+ },
+ {
+ type: EventType.TOOL_CALL_END,
+ toolCallId: "tool-1",
+ },
+ {
+ type: EventType.TOOL_CALL_RESULT,
+ toolCallId: "tool-1",
+ content: "ok",
+ },
+ ]);
+
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "tool-create",
+ expect.objectContaining({
+ metadata: expect.objectContaining({
+ toolCallId: "tool-1",
+ parentMessageId: "tool-1-approval-tool-call",
+ toolCallApproval: "approved",
+ }),
+ }),
+ );
+ });
+
+ it("records rejected approvals without fabricating tool execution duration", () => {
+ vi.useFakeTimers();
+ try {
const instrumentation = createInstrumentation();
+ const rejectedAt = new Date("2026-01-01T00:00:01.000Z");
instrumentation.recordToolCallApproval({
toolCallId: "tool-1",
- status,
+ status: "rejected",
});
+ vi.setSystemTime(rejectedAt);
instrumentation.recordEvents([
{
type: EventType.TOOL_CALL_START,
@@ -258,27 +299,34 @@ describe("InAppAgentInstrumentation", () => {
type: EventType.TOOL_CALL_END,
toolCallId: "tool-1",
},
+ ]);
+ vi.setSystemTime(new Date("2026-01-01T00:00:05.000Z"));
+ instrumentation.recordEvents([
{
type: EventType.TOOL_CALL_RESULT,
toolCallId: "tool-1",
- content: "ok",
+ content: "rejected",
},
]);
expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
"tool-create",
expect.objectContaining({
+ startTime: rejectedAt,
+ endTime: rejectedAt,
metadata: expect.objectContaining({
toolCallId: "tool-1",
parentMessageId: "tool-1-approval-tool-call",
- toolCallApproval: status,
+ toolCallApproval: "rejected",
}),
}),
);
- },
- );
+ } finally {
+ vi.useRealTimers();
+ }
+ });
- it("records run failures on the agent generation", () => {
+ it("records run failures on the root agent observation", () => {
const instrumentation = createInstrumentation();
const toolCall = {
id: "tool-1",
@@ -313,8 +361,10 @@ describe("InAppAgentInstrumentation", () => {
}),
}),
);
- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "agent-create",
expect.objectContaining({
+ id: agentRunObservationId,
name: "agent-turn",
input: expectedAgentRunInput,
output: {
@@ -327,7 +377,6 @@ describe("InAppAgentInstrumentation", () => {
],
tool_calls: [toolCall],
},
- completionStartTime: expect.any(Date),
level: "ERROR",
statusMessage: "agent failed",
metadata: expect.objectContaining({ error: "agent failed" }),
@@ -335,7 +384,7 @@ describe("InAppAgentInstrumentation", () => {
);
});
- it("records available tools & skills on the agent generation input", () => {
+ it("records available tools & skills on the agent observation input", () => {
const instrumentation = createInstrumentation();
instrumentation.recordAvailableTools({
@@ -368,7 +417,8 @@ describe("InAppAgentInstrumentation", () => {
]);
instrumentation.end();
- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "agent-create",
expect.objectContaining({
name: "agent-turn",
input: {
@@ -443,7 +493,9 @@ describe("InAppAgentInstrumentation", () => {
},
]);
- const toolCreateBody = mocks.handler.langfuse.enqueue.mock.calls[0]?.[1];
+ const toolCreateBody = mocks.handler.langfuse.enqueue.mock.calls.find(
+ ([type]) => type === "tool-create",
+ )?.[1];
expect(toolCreateBody).toEqual(
expect.objectContaining({
@@ -596,7 +648,9 @@ describe("InAppAgentInstrumentation", () => {
},
]);
- const toolCreateBody = mocks.handler.langfuse.enqueue.mock.calls[0]?.[1];
+ const toolCreateBody = mocks.handler.langfuse.enqueue.mock.calls.find(
+ ([type]) => type === "tool-create",
+ )?.[1];
expect(toolCreateBody).toEqual(
expect.objectContaining({
output: expectedOutput,
@@ -607,7 +661,7 @@ describe("InAppAgentInstrumentation", () => {
},
);
- it("sets static prompt metadata on the trace and agent generation", () => {
+ it("sets static prompt metadata on the trace and root agent observation", () => {
const instrumentation = createInstrumentation(undefined, {
name: "in-app-agent-system-prompt",
version: 3,
@@ -641,26 +695,14 @@ describe("InAppAgentInstrumentation", () => {
},
}),
);
- expect(mocks.trace.generation).toHaveBeenCalledWith({
- id: agentRunObservationId,
- name: "agent-turn",
- input: expectedAgentRunInput,
- metadata: {
- langfuse_project_id: "project-1",
- langfuse_user_email: "[email protected]",
- langfuse_user_project_role: "ADMIN",
- langfuse_user_is_admin: true,
- prompt_name: "in-app-agent-system-prompt",
- prompt_version: 3,
- },
- promptName: "in-app-agent-system-prompt",
- promptVersion: 3,
- });
+ expect(mocks.trace.generation).not.toHaveBeenCalled();
instrumentation.end();
- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "agent-create",
expect.objectContaining({
+ id: agentRunObservationId,
name: "agent-turn",
input: expectedAgentRunInput,
promptName: "in-app-agent-system-prompt",
@@ -682,7 +724,8 @@ describe("InAppAgentInstrumentation", () => {
},
]);
- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "agent-create",
expect.objectContaining({
name: "agent-turn",
input: expectedAgentRunInput,
@@ -690,7 +733,6 @@ describe("InAppAgentInstrumentation", () => {
messages: [{ role: "assistant", content: "second turn output" }],
text: "second turn output",
},
- completionStartTime: expect.any(Date),
}),
);
expect(mocks.trace.update).toHaveBeenCalledWith(
@@ -704,7 +746,7 @@ describe("InAppAgentInstrumentation", () => {
);
});
- it("records AG-UI context in the agent generation input", () => {
+ it("records AG-UI context in the agent turn input", () => {
createInstrumentation({
context: [
{
@@ -723,20 +765,7 @@ describe("InAppAgentInstrumentation", () => {
],
});
- expect(mocks.trace.generation).toHaveBeenCalledWith(
- expect.objectContaining({
- name: "agent-turn",
- input: {
- messages: [{ role: "user", content: "hello" }],
- context: {
- current_url:
- "https://cloud.langfuse.com/project/project-1/traces?filter=value",
- browser_languages: ["en-US", "en", "de"],
- current_trace: { id: "trace-1" },
- },
- },
- }),
- );
+ expect(mocks.trace.generation).not.toHaveBeenCalled();
expect(mocks.handler.langfuse.trace).toHaveBeenCalledWith(
expect.objectContaining({
input: {
@@ -752,7 +781,7 @@ describe("InAppAgentInstrumentation", () => {
);
});
- it("records only the current turn messages in the agent generation input", () => {
+ it("records only the current turn messages in the agent turn input", () => {
createInstrumentation({
messages: [
{
@@ -795,7 +824,7 @@ describe("InAppAgentInstrumentation", () => {
],
});
- expect(mocks.trace.generation).toHaveBeenCalledWith(
+ expect(mocks.handler.langfuse.trace).toHaveBeenCalledWith(
expect.objectContaining({
name: "agent-turn",
input: {
@@ -805,6 +834,211 @@ describe("InAppAgentInstrumentation", () => {
);
});
+ it.each([
+ { approved: true, status: "approved" as const },
+ { approved: false, status: "rejected" as const },
+ ])(
+ "appends a $status approval continuation to the parent trace with a wait span",
+ ({ approved, status }) => {
+ const traceStartedAt = "2026-08-14T10:00:00.000Z";
+ const approvalRequestedAt = "2026-08-14T10:00:05.000Z";
+ const approvalDecidedAt = "2026-08-14T10:02:05.000Z";
+ const instrumentation = createInstrumentation({
+ forwardedProps: {
+ command: {
+ resume: {
+ approved,
+ continuationNumber: 2,
+ rootRunId: "root-run-1",
+ traceStartedAt,
+ approvalRequestedAt,
+ approvalDecidedAt,
+ approvalRequest: {
+ type: "tool_approval_request",
+ toolCallId: "tool-1",
+ toolName: "langfuse_createTextPrompt",
+ args: { name: "FOO" },
+ runId: "parent-run-1",
+ },
+ },
+ },
+ },
+ });
+
+ instrumentation.recordToolCallApproval({
+ toolCallId: "tool-1",
+ status,
+ });
+ instrumentation.recordEvents([
+ {
+ type: EventType.TOOL_CALL_START,
+ toolCallId: "tool-1",
+ toolCallName: "langfuse_createTextPrompt",
+ },
+ {
+ type: EventType.TOOL_CALL_ARGS,
+ toolCallId: "tool-1",
+ delta: '{"name":"FOO"}',
+ },
+ {
+ type: EventType.TOOL_CALL_END,
+ toolCallId: "tool-1",
+ },
+ {
+ type: EventType.TOOL_CALL_RESULT,
+ toolCallId: "tool-1",
+ content: '{"name":"FOO","version":1}',
+ },
+ ]);
+ instrumentation.recordStreamChunk({ type: "step-start", payload: {} });
+ instrumentation.recordStepFinish({
+ usage: { inputTokens: 100, outputTokens: 10, totalTokens: 110 },
+ });
+ instrumentation.end({});
+
+ const parentTraceId = getInAppAgentInstrumentationTraceId("root-run-1");
+ const parentObservationId =
+ getInAppAgentInstrumentationObservationId("root-run-1");
+ const approvalMetadata = {
+ continuation_type: "tool_approval",
+ continuation_number: 2,
+ parent_run_id: "parent-run-1",
+ approval_status: status,
+ approval_tool_call_id: "tool-1",
+ approval_tool_name: "langfuse_createTextPrompt",
+ approval_requested_at: approvalRequestedAt,
+ approval_decided_at: approvalDecidedAt,
+ };
+
+ expect(mocks.handler.langfuse.trace).toHaveBeenCalledWith(
+ expect.objectContaining({
+ id: parentTraceId,
+ name: "agent-turn",
+ sessionId: "conversation-1",
+ timestamp: new Date(traceStartedAt),
+ input: expectedAgentRunInput,
+ tags: ["in-app-agent"],
+ metadata: expect.objectContaining({
+ approval_continuation_count: 2,
+ }),
+ }),
+ );
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "agent-create",
+ expect.objectContaining({
+ id: parentObservationId,
+ traceId: parentTraceId,
+ name: "agent-turn",
+ startTime: new Date(traceStartedAt),
+ input: expectedAgentRunInput,
+ metadata: expect.objectContaining({
+ approval_continuation_count: 2,
+ }),
+ }),
+ );
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "span-create",
+ expect.objectContaining({
+ id: `${runId}-approval-wait`,
+ traceId: parentTraceId,
+ parentObservationId,
+ name: "waiting for user approval for tool langfuse_createTextPrompt",
+ startTime: new Date(approvalRequestedAt),
+ endTime: new Date(approvalDecidedAt),
+ output: `User ${status} tool langfuse_createTextPrompt`,
+ metadata: approvalMetadata,
+ }),
+ );
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "generation-create",
+ expect.objectContaining({
+ traceId: parentTraceId,
+ parentObservationId,
+ input: expect.objectContaining({
+ messages: [
+ { role: "user", content: "hello" },
+ expect.objectContaining({
+ role: "assistant",
+ tool_calls: [
+ expect.objectContaining({
+ id: "tool-1",
+ name: "langfuse_createTextPrompt",
+ }),
+ ],
+ }),
+ {
+ role: "tool",
+ tool_call_id: "tool-1",
+ content: { name: "FOO", version: 1 },
+ },
+ ],
+ }),
+ }),
+ );
+ expect(mocks.trace.update).toHaveBeenCalledWith(
+ expect.objectContaining({
+ input: expectedAgentRunInput,
+ metadata: expect.objectContaining({
+ approval_continuation_count: 2,
+ }),
+ }),
+ );
+ },
+ );
+
+ it("preserves earlier assistant output when a continuation updates the root trace", () => {
+ const instrumentation = createInstrumentation({
+ messages: [
+ { id: "message-1", role: "user", content: "hello" },
+ {
+ id: "assistant-before-approval",
+ role: "assistant",
+ content: "I need your approval.",
+ },
+ ],
+ forwardedProps: {
+ command: {
+ resume: {
+ approved: true,
+ rootRunId: "root-run-1",
+ traceStartedAt: "2026-08-14T10:00:00.000Z",
+ approvalRequestedAt: "2026-08-14T10:00:05.000Z",
+ approvalDecidedAt: "2026-08-14T10:02:05.000Z",
+ approvalRequest: {
+ type: "tool_approval_request",
+ toolCallId: "tool-1",
+ toolName: "langfuse_createTextPrompt",
+ runId: "parent-run-1",
+ },
+ },
+ },
+ },
+ });
+
+ instrumentation.recordEvents([
+ {
+ type: EventType.TEXT_MESSAGE_CONTENT,
+ delta: "The tool completed.",
+ },
+ ]);
+ instrumentation.end({});
+
+ const finalRootObservation = mocks.handler.langfuse.enqueue.mock.calls
+ .filter(([type]) => type === "agent-create")
+ .at(-1)?.[1];
+ expect(finalRootObservation).toEqual(
+ expect.objectContaining({
+ input: expectedAgentRunInput,
+ output: expect.objectContaining({
+ messages: [
+ { role: "assistant", content: "I need your approval." },
+ { role: "assistant", content: "The tool completed." },
+ ],
+ }),
+ }),
+ );
+ });
+
it("compacts text message chunks before recording output", () => {
const instrumentation = createInstrumentation();
@@ -825,7 +1059,8 @@ describe("InAppAgentInstrumentation", () => {
},
]);
- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "agent-create",
expect.objectContaining({
name: "agent-turn",
input: expectedAgentRunInput,
@@ -833,7 +1068,6 @@ describe("InAppAgentInstrumentation", () => {
messages: [{ role: "assistant", content: "chunk output" }],
text: "chunk output",
},
- completionStartTime: expect.any(Date),
}),
);
});
@@ -908,7 +1142,8 @@ describe("InAppAgentInstrumentation", () => {
},
]);
- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "agent-create",
expect.objectContaining({
name: "agent-turn",
input: expectedAgentRunInput,
@@ -930,13 +1165,12 @@ describe("InAppAgentInstrumentation", () => {
text: "Done",
tool_calls: [toolCall],
},
- completionStartTime: expect.any(Date),
}),
);
- const updateArg = mocks.agentGeneration.update.mock.calls.at(-1)?.[0] as {
- metadata?: Record<string, unknown>;
- };
- expect(updateArg.metadata).not.toHaveProperty("reasoning");
+ const agentCreateBody = mocks.handler.langfuse.enqueue.mock.calls.find(
+ ([type]) => type === "agent-create",
+ )?.[1] as { metadata?: Record<string, unknown> };
+ expect(agentCreateBody.metadata).not.toHaveProperty("reasoning");
});
it("records reasoning without a following assistant message as its own thinking message", () => {
@@ -953,7 +1187,8 @@ describe("InAppAgentInstrumentation", () => {
},
]);
- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "agent-create",
expect.objectContaining({
name: "agent-turn",
output: {
@@ -987,9 +1222,53 @@ describe("InAppAgentInstrumentation", () => {
);
});
- it("aggregates step usage and model onto the agent generation", () => {
+ it("emits one generation per LLM step with usage on children only", () => {
const instrumentation = createInstrumentation();
+ const modelName = "eu.anthropic.claude-sonnet-4-5-20250929-v1:0";
+
+ // Root agent observation must exist before children so tools/generations
+ // do not orphan onto the trace.
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "agent-create",
+ expect.objectContaining({
+ id: agentRunObservationId,
+ name: "agent-turn",
+ }),
+ );
+ instrumentation.recordStreamChunk({
+ type: "step-start",
+ payload: {},
+ });
+ instrumentation.recordStreamChunk({
+ type: "tool-call",
+ payload: { toolCallId: "tool-1", toolName: "listObservations" },
+ });
+ instrumentation.recordEvents([
+ {
+ type: EventType.TOOL_CALL_START,
+ toolCallId: "tool-1",
+ toolCallName: "listObservations",
+ },
+ {
+ type: EventType.TOOL_CALL_ARGS,
+ toolCallId: "tool-1",
+ delta: '{"limit":10}',
+ },
+ {
+ type: EventType.TOOL_CALL_END,
+ toolCallId: "tool-1",
+ },
+ {
+ type: EventType.TOOL_CALL_RESULT,
+ toolCallId: "tool-1",
+ content: '{"ok":true}',
+ },
+ ]);
+ instrumentation.recordStreamChunk({
+ type: "tool-result",
+ payload: { toolCallId: "tool-1", toolName: "listObservations" },
+ });
instrumentation.recordStepFinish({
usage: {
inputTokens: 1100,
@@ -998,7 +1277,42 @@ describe("InAppAgentInstrumentation", () => {
cachedInputTokens: 800,
cacheCreationInputTokens: 100,
},
+ finishReason: "tool-calls",
+ text: "",
+ toolCalls: [
+ {
+ toolCallId: "tool-1",
+ toolName: "listObservations",
+ args: { limit: 10 },
+ },
+ ],
+ response: {
+ messages: [
+ {
+ role: "assistant",
+ content: "",
+ tool_calls: [{ id: "tool-1" }],
+ },
+ {
+ role: "tool",
+ tool_call_id: "tool-1",
+ content: { ok: true },
+ },
+ ],
+ },
+ });
+
+ instrumentation.recordStreamChunk({ type: "step-start", payload: {} });
+ instrumentation.recordStreamChunk({
+ type: "text-delta",
+ payload: { text: "Done" },
});
+ instrumentation.recordEvents([
+ {
+ type: EventType.TEXT_MESSAGE_CONTENT,
+ delta: "Done",
+ },
+ ]);
instrumentation.recordStepFinish({
usage: {
inputTokens: 1200,
@@ -1006,37 +1320,332 @@ describe("InAppAgentInstrumentation", () => {
totalTokens: 1230,
cachedInputTokens: 1000,
},
+ finishReason: "stop",
+ text: "Done",
});
instrumentation.end({});
- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(
+ const generationCreates = mocks.handler.langfuse.enqueue.mock.calls.filter(
+ ([type]) => type === "generation-create",
+ );
+ expect(generationCreates).toHaveLength(2);
+
+ expect(generationCreates[0]?.[1]).toEqual(
expect.objectContaining({
- model: "eu.anthropic.claude-sonnet-4-5-20250929-v1:0",
+ id: getInAppAgentLlmCallObservationId(runId, 1),
+ parentObservationId: agentRunObservationId,
+ model: modelName,
usageDetails: {
- // Mastra's inputTokens includes cache reads/writes; the priced
- // `input` key must only contain non-cached input tokens.
- input: 400,
- output: 80,
- cache_read_input_tokens: 1800,
+ input: 200,
+ output: 50,
+ cache_read_input_tokens: 800,
cache_creation_input_tokens: 100,
- total: 2380,
+ total: 1150,
+ },
+ metadata: expect.objectContaining({
+ finish_reason: "tool-calls",
+ }),
+ }),
+ );
+ expect(generationCreates[0]?.[1].input.messages).toHaveLength(1);
+ expect(generationCreates[0]?.[1].output).toEqual({
+ text: "",
+ tool_calls: [
+ {
+ toolCallId: "tool-1",
+ toolName: "listObservations",
+ args: { limit: 10 },
+ },
+ ],
+ });
+
+ expect(generationCreates[1]?.[1]).toEqual(
+ expect.objectContaining({
+ id: getInAppAgentLlmCallObservationId(runId, 2),
+ parentObservationId: agentRunObservationId,
+ model: modelName,
+ usageDetails: {
+ input: 200,
+ output: 30,
+ cache_read_input_tokens: 1000,
+ cache_creation_input_tokens: 0,
+ total: 1230,
},
+ metadata: expect.objectContaining({
+ finish_reason: "stop",
+ }),
}),
);
+ expect(generationCreates[1]?.[1].input.messages.length).toBeGreaterThan(
+ generationCreates[0]?.[1].input.messages.length,
+ );
+
+ const agentCreates = mocks.handler.langfuse.enqueue.mock.calls.filter(
+ ([type]) => type === "agent-create",
+ );
+ const finalAgentCreate = agentCreates.at(-1)?.[1];
+ expect(finalAgentCreate).toEqual(
+ expect.objectContaining({
+ id: agentRunObservationId,
+ name: "agent-turn",
+ }),
+ );
+ expect(finalAgentCreate).not.toHaveProperty("usageDetails");
+ expect(finalAgentCreate).not.toHaveProperty("model");
});
- it("records aggregated usage when the run fails", () => {
+ it("emits tools after the generation and clamps start to generation end", () => {
const instrumentation = createInstrumentation();
+ const stepStart = new Date("2026-01-01T00:00:00.000Z");
+ const toolCallChunkA = new Date("2026-01-01T00:00:01.000Z");
+ const toolCallChunkB = new Date("2026-01-01T00:00:02.000Z");
+ const agUiToolAStart = new Date("2026-01-01T00:00:01.500Z");
+ const agUiToolBStart = new Date("2026-01-01T00:00:02.500Z");
+ const toolAStart = new Date("2026-01-01T00:00:02.900Z");
+ const toolBStart = new Date("2026-01-01T00:00:03.200Z");
+ const toolAEnd = new Date("2026-01-01T00:00:03.000Z");
+ const toolBEnd = new Date("2026-01-01T00:00:04.000Z");
+ const providerFinish = new Date("2026-01-01T00:00:02.800Z");
+ const stepFinish = new Date("2026-01-01T00:00:04.200Z");
+
+ vi.setSystemTime(stepStart);
+ instrumentation.recordStreamChunk({ type: "step-start", payload: {} });
+
+ // tool-call chunks are request-time during the LLM stream — must not become
+ // tool observation startTimes (that sorted tools above the generation).
+ vi.setSystemTime(toolCallChunkA);
+ instrumentation.recordStreamChunk({
+ type: "tool-call",
+ payload: { toolCallId: "tool-a", toolName: "listObservations" },
+ });
+ vi.setSystemTime(toolCallChunkB);
+ instrumentation.recordStreamChunk({
+ type: "tool-call",
+ payload: { toolCallId: "tool-b", toolName: "getTrace" },
+ });
+ vi.setSystemTime(agUiToolAStart);
+ instrumentation.recordEvents([
+ {
+ type: EventType.TOOL_CALL_START,
+ toolCallId: "tool-a",
+ toolCallName: "listObservations",
+ },
+ {
+ type: EventType.TOOL_CALL_ARGS,
+ toolCallId: "tool-a",
+ delta: "{}",
+ },
+ {
+ type: EventType.TOOL_CALL_END,
+ toolCallId: "tool-a",
+ },
+ ]);
+ vi.setSystemTime(agUiToolBStart);
+ instrumentation.recordEvents([
+ {
+ type: EventType.TOOL_CALL_START,
+ toolCallId: "tool-b",
+ toolCallName: "getTrace",
+ },
+ {
+ type: EventType.TOOL_CALL_ARGS,
+ toolCallId: "tool-b",
+ delta: "{}",
+ },
+ {
+ type: EventType.TOOL_CALL_END,
+ toolCallId: "tool-b",
+ },
+ ]);
+
+ // The model stream finishes before tool execution, but Mastra only invokes
+ // onStepFinish after the tools have resolved.
+ vi.setSystemTime(providerFinish);
+ instrumentation.recordModelCallFinish({
+ usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
+ finishReason: { unified: "tool-calls", raw: "tool_use" },
+ });
+
+ vi.setSystemTime(toolAStart);
+ instrumentation.recordToolExecutionStart("tool-a");
+ vi.setSystemTime(toolAEnd);
+ instrumentation.recordToolExecutionEnd("tool-a");
+ instrumentation.recordStreamChunk({
+ type: "tool-result",
+ payload: { toolCallId: "tool-a", toolName: "listObservations" },
+ });
+ instrumentation.recordEvents([
+ {
+ type: EventType.TOOL_CALL_RESULT,
+ toolCallId: "tool-a",
+ content: '{"a":1}',
+ },
+ ]);
+
+ vi.setSystemTime(toolBStart);
+ instrumentation.recordToolExecutionStart("tool-b");
+ vi.setSystemTime(toolBEnd);
+ instrumentation.recordToolExecutionEnd("tool-b");
+ instrumentation.recordStreamChunk({
+ type: "tool-result",
+ payload: { toolCallId: "tool-b", toolName: "getTrace" },
+ });
+ instrumentation.recordEvents([
+ {
+ type: EventType.TOOL_CALL_RESULT,
+ toolCallId: "tool-b",
+ content: '{"b":2}',
+ },
+ ]);
+
+ // Tools completed during the open LLM step — deferred until step-finish.
+ expect(
+ mocks.handler.langfuse.enqueue.mock.calls.filter(
+ ([type]) => type === "tool-create",
+ ),
+ ).toHaveLength(0);
+
+ vi.setSystemTime(stepFinish);
instrumentation.recordStepFinish({
- usage: { inputTokens: 100, outputTokens: 10, totalTokens: 110 },
+ usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
+ finishReason: "tool-calls",
});
- instrumentation.endWithError(new Error("agent failed"));
- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(
+ const enqueueOrder = mocks.handler.langfuse.enqueue.mock.calls
+ .filter(
+ ([type]) => type === "generation-create" || type === "tool-create",
+ )
+ .map(([type, body]) => ({ type, id: body.id }));
+ expect(enqueueOrder.map((entry) => entry.type)).toEqual([
+ "generation-create",
+ "tool-create",
+ "tool-create",
+ ]);
+
+ const generationCreate = mocks.handler.langfuse.enqueue.mock.calls.find(
+ ([type]) => type === "generation-create",
+ )?.[1];
+ expect(generationCreate).toEqual(
+ expect.objectContaining({ endTime: providerFinish }),
+ );
+
+ const toolCreates = mocks.handler.langfuse.enqueue.mock.calls.filter(
+ ([type]) => type === "tool-create",
+ );
+ expect(toolCreates).toHaveLength(2);
+ // Actual execute() timing wins over the delayed AG-UI event timestamps.
+ expect(toolCreates[0]?.[1]).toEqual(
+ expect.objectContaining({
+ id: "tool-a",
+ parentObservationId: agentRunObservationId,
+ startTime: toolAStart,
+ endTime: toolAEnd,
+ }),
+ );
+ expect(toolCreates[1]?.[1]).toEqual(
+ expect.objectContaining({
+ id: "tool-b",
+ parentObservationId: agentRunObservationId,
+ startTime: toolBStart,
+ endTime: toolBEnd,
+ }),
+ );
+
+ vi.useRealTimers();
+ });
+
+ it("opens an LLM step on first model delta when step-start is missing", () => {
+ const instrumentation = createInstrumentation();
+ const firstDelta = new Date("2026-01-01T00:00:10.000Z");
+ const stepFinish = new Date("2026-01-01T00:00:12.000Z");
+
+ vi.setSystemTime(firstDelta);
+ instrumentation.recordStreamChunk({
+ type: "reasoning-delta",
+ payload: { text: "thinking" },
+ });
+ vi.setSystemTime(stepFinish);
+ instrumentation.recordStepFinish({
+ usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
+ finishReason: "stop",
+ });
+
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "generation-create",
+ expect.objectContaining({
+ startTime: firstDelta,
+ endTime: stepFinish,
+ }),
+ );
+
+ vi.useRealTimers();
+ });
+
+ it("does not force-close incomplete tools on a normal end (approval interrupt)", () => {
+ const instrumentation = createInstrumentation();
+
+ instrumentation.recordEvents([
+ {
+ type: EventType.TOOL_CALL_START,
+ toolCallId: "tool-awaiting-approval",
+ toolCallName: "createAnnotationQueueItem",
+ },
+ {
+ type: EventType.TOOL_CALL_ARGS,
+ toolCallId: "tool-awaiting-approval",
+ delta: '{"queueId":"q1"}',
+ },
+ {
+ type: EventType.TOOL_CALL_END,
+ toolCallId: "tool-awaiting-approval",
+ },
+ ]);
+ instrumentation.end({});
+
+ expect(mocks.handler.langfuse.enqueue).not.toHaveBeenCalledWith(
+ "tool-create",
+ expect.objectContaining({ id: "tool-awaiting-approval" }),
+ );
+ });
+
+ it("closes an open LLM step as ERROR when the run is aborted", () => {
+ const instrumentation = createInstrumentation();
+
+ instrumentation.recordStreamChunk({ type: "step-start", payload: {} });
+ instrumentation.end({ aborted: true });
+
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "generation-create",
expect.objectContaining({
+ id: getInAppAgentLlmCallObservationId(runId, 1),
+ parentObservationId: agentRunObservationId,
level: "ERROR",
- model: "eu.anthropic.claude-sonnet-4-5-20250929-v1:0",
+ statusMessage: "aborted",
+ }),
+ );
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "agent-create",
+ expect.objectContaining({
+ id: agentRunObservationId,
+ metadata: expect.objectContaining({ aborted: true }),
+ }),
+ );
+ });
+
+ it("emits a generation when step finish arrives without a prior step-start", () => {
+ const instrumentation = createInstrumentation();
+
+ instrumentation.recordStepFinish({
+ usage: { inputTokens: 100, outputTokens: 10, totalTokens: 110 },
+ finishReason: "stop",
+ });
+ instrumentation.end({});
+
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "generation-create",
+ expect.objectContaining({
+ id: getInAppAgentLlmCallObservationId(runId, 1),
usageDetails: {
input: 100,
output: 10,
@@ -1044,21 +1653,99 @@ describe("InAppAgentInstrumentation", () => {
cache_creation_input_tokens: 0,
total: 110,
},
+ model: "eu.anthropic.claude-sonnet-4-5-20250929-v1:0",
}),
);
});
- it("omits usage and model when no step reported usage", () => {
+ it("uses provider finish usage when approval ends before step finish", () => {
const instrumentation = createInstrumentation();
- instrumentation.recordStepFinish(undefined);
- instrumentation.recordStepFinish({ usage: { foo: 1 } });
+ instrumentation.recordStreamChunk({ type: "step-start", payload: {} });
+ instrumentation.recordModelCallFinish({
+ usage: {
+ inputTokens: {
+ total: 1_100,
+ noCache: 200,
+ cacheRead: 800,
+ cacheWrite: 100,
+ },
+ outputTokens: { total: 50, text: 50, reasoning: 0 },
+ },
+ finishReason: { unified: "tool-calls", raw: "tool_use" },
+ });
instrumentation.end({});
- const agentGenerationBody = mocks.agentGeneration.update.mock.calls[0]?.[0];
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "generation-create",
+ expect.objectContaining({
+ id: getInAppAgentLlmCallObservationId(runId, 1),
+ model: "eu.anthropic.claude-sonnet-4-5-20250929-v1:0",
+ usageDetails: {
+ input: 200,
+ output: 50,
+ cache_read_input_tokens: 800,
+ cache_creation_input_tokens: 100,
+ total: 1_150,
+ },
+ metadata: expect.objectContaining({
+ finish_reason: "tool-calls",
+ }),
+ }),
+ );
+ });
+
+ it("keeps a provider-finished generation successful when the run later fails", () => {
+ const instrumentation = createInstrumentation();
- expect(agentGenerationBody).not.toHaveProperty("usageDetails");
- expect(agentGenerationBody).not.toHaveProperty("model");
+ instrumentation.recordStreamChunk({ type: "step-start", payload: {} });
+ instrumentation.recordEvents([
+ {
+ type: EventType.TEXT_MESSAGE_CONTENT,
+ delta: "Calling the tool",
+ },
+ ]);
+ instrumentation.recordModelCallFinish({
+ usage: {
+ inputTokens: {
+ total: 1_100,
+ noCache: 200,
+ cacheRead: 800,
+ cacheWrite: 100,
+ },
+ outputTokens: { total: 50, text: 50, reasoning: 0 },
+ },
+ finishReason: { unified: "tool-calls", raw: "tool_use" },
+ });
+ instrumentation.endWithError(new Error("tool execution failed"));
+
+ const generation = mocks.handler.langfuse.enqueue.mock.calls.find(
+ ([type]) => type === "generation-create",
+ )?.[1];
+ expect(generation).toEqual(
+ expect.objectContaining({
+ id: getInAppAgentLlmCallObservationId(runId, 1),
+ model: "eu.anthropic.claude-sonnet-4-5-20250929-v1:0",
+ usageDetails: expect.objectContaining({
+ input: 200,
+ output: 50,
+ total: 1_150,
+ }),
+ output: {
+ messages: [{ role: "assistant", content: "Calling the tool" }],
+ },
+ }),
+ );
+ expect(generation).not.toHaveProperty("level");
+ expect(generation).not.toHaveProperty("statusMessage");
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "agent-create",
+ expect.objectContaining({
+ id: agentRunObservationId,
+ level: "ERROR",
+ statusMessage: "tool execution failed",
+ }),
+ );
});
it("ignores step finish events after instrumentation ended", () => {
@@ -1069,8 +1756,9 @@ describe("InAppAgentInstrumentation", () => {
usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
});
- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(
- expect.not.objectContaining({ usageDetails: expect.anything() }),
+ expect(mocks.handler.langfuse.enqueue).not.toHaveBeenCalledWith(
+ "generation-create",
+ expect.anything(),
);
});
});
diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts
index ea29795b1d..70f4265527 100644
--- a/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts
+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts
@@ -1,11 +1,12 @@
import { randomUUID } from "crypto";
-import { describe, expect, it, vi } from "vitest";
+import { beforeEach, describe, expect, it, vi } from "vitest";
import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
import {
createInAppAgentConversationId,
createInAppAgentRunId,
+ ResumeForwardedPropsSchema,
} from "@langfuse/shared/in-app-agent";
vi.hoisted(() => {
@@ -54,6 +55,7 @@ const scenarioRef = vi.hoisted(() => ({
current: undefined as AgentScenario | undefined,
failApiKeyDelete: false,
apiKeyDeleteCalls: 0,
+ titleInferenceCalls: 0,
}));
vi.mock("@langfuse/shared/in-app-agent/server", async (importOriginal) => {
@@ -65,6 +67,9 @@ vi.mock("@langfuse/shared/in-app-agent/server", async (importOriginal) => {
return {
...actual,
IN_APP_AGENT_HEARTBEAT_INTERVAL_MS: 50,
+ maybeInferAndPersistConversationTitle: async () => {
+ scenarioRef.titleInferenceCalls += 1;
+ },
createAgUiStream: async (params: {
input: never;
signal: AbortSignal;
@@ -203,6 +208,10 @@ const getInAppAgentApiKeys = (projectId: string) =>
async function seedApprovedContinuation(opts?: {
context?: Array<{ description: string; value: string }>;
alwaysAllowedTools?: string[];
+ continuationNumber?: number;
+ rootRunId?: string;
+ traceStartedAt?: string;
+ approvalRequestedAt?: string;
}) {
const seeded = await seedBackgroundRun({
alwaysAllowedTools: opts?.alwaysAllowedTools,
@@ -234,8 +243,18 @@ async function seedApprovedContinuation(opts?: {
request: {
kind: "approvalDecision",
parentRunId: parentRun.id,
+ ...(opts?.rootRunId ? { rootRunId: opts.rootRunId } : {}),
+ ...(opts?.traceStartedAt
+ ? { traceStartedAt: opts.traceStartedAt }
+ : {}),
+ ...(opts?.approvalRequestedAt
+ ? { approvalRequestedAt: opts.approvalRequestedAt }
+ : {}),
toolCallId: "tc-1",
approved: true,
+ ...(opts?.continuationNumber
+ ? { continuationNumber: opts.continuationNumber }
+ : {}),
...(opts?.context ? { context: opts.context } : {}),
},
},
@@ -245,7 +264,24 @@ async function seedApprovedContinuation(opts?: {
}
describe("executeInAppAgentRun", () => {
+ beforeEach(() => {
+ scenarioRef.titleInferenceCalls = 0;
+ });
+
+ it("does not regenerate the conversation title after executing a user-message run", async () => {
+ const { projectId, run } = await seedBackgroundRun();
+
+ scenarioRef.current = completingScenario;
+
+ await executeInAppAgentRun({ projectId, runId: run.id });
+
+ expect(scenarioRef.titleInferenceCalls).toBe(0);
+ });
+
it("passes persisted continuation context to the agent input", async () => {
+ const rootRunId = "root-run-1";
+ const traceStartedAt = "2026-08-14T10:00:00.000Z";
+ const approvalRequestedAt = "2026-08-14T10:00:05.000Z";
const context = [
{
description: "current_url",
@@ -253,10 +289,25 @@ describe("executeInAppAgentRun", () => {
},
{ description: "browser_languages", value: '["de-DE"]' },
];
- const { projectId, run } = await seedApprovedContinuation({ context });
+ const { projectId, run } = await seedApprovedContinuation({
+ context,
+ continuationNumber: 3,
+ rootRunId,
+ traceStartedAt,
+ approvalRequestedAt,
+ });
scenarioRef.current = async ({ input, options }) => {
expect(input.context).toEqual(context);
+ const resume = ResumeForwardedPropsSchema.parse(input.forwardedProps)
+ .command.resume;
+ expect(resume).toMatchObject({
+ continuationNumber: 3,
+ rootRunId,
+ traceStartedAt,
+ approvalRequestedAt,
+ approvalDecidedAt: run.createdAt.toISOString(),
+ });
await options.onComplete();
await options.onFinish();
};
@@ -322,7 +373,6 @@ describe("executeInAppAgentRun", () => {
expect(finished.claimedAt).not.toBeNull();
expect(finished.heartbeatAt).not.toBeNull();
expect(finished.errorCode).toBeNull();
-
// Key was minted and linked during the run, deleted and unlinked after.
expect(keysDuringRun).toBe(1);
expect(await getInAppAgentApiKeys(projectId)).toHaveLength(0);
diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.ts
index 3819d71a4a..b03fc7ebf0 100644
--- a/worker/src/features/in-app-agent/executeInAppAgentRun.ts
+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.ts
@@ -39,7 +39,6 @@ import {
createInAppAgentToolPolicy,
getInAppAgentMcpAllowedToolNames,
getInAppAgentRegistryToolName,
- maybeInferAndPersistConversationTitle,
parseInAppAgentInterruptEvent,
shouldFlushPersistedEvent,
toPersistableAgentEvent,
@@ -257,6 +256,17 @@ export async function executeInAppAgentRun(params: {
command: {
resume: {
approved: request.approved,
+ continuationNumber: request.continuationNumber ?? 1,
+ ...(request.rootRunId
+ ? { rootRunId: request.rootRunId }
+ : {}),
+ ...(request.traceStartedAt
+ ? { traceStartedAt: request.traceStartedAt }
+ : {}),
+ ...(request.approvalRequestedAt
+ ? { approvalRequestedAt: request.approvalRequestedAt }
+ : {}),
+ approvalDecidedAt: run.createdAt.toISOString(),
approvalRequest,
},
},
@@ -437,19 +447,6 @@ export async function executeInAppAgentRun(params: {
? { status: InAppAgentRunStatus.AWAITING_APPROVAL }
: { status: InAppAgentRunStatus.SUCCEEDED },
);
- await maybeInferAndPersistConversationTitle({
- prisma,
- projectId,
- conversationId: conversation.id,
- userId: run.triggeredByUserId!,
- aiTelemetryEnabled: project.organization.aiTelemetryEnabled,
- }).catch((error) =>
- logger.error("Failed to infer in-app agent conversation title", {
- error,
- projectId,
- runId,
- }),
- );
},
onAbort: async () => {
const reason = abortController.signal.reason as AbortReason;