🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(in-app-agent): trace complete agent turns across approvals (#16037)
Lysander Kiesel2ef787e7e2fc
diff --git a/packages/shared/src/features/inAppAgent/types.ts b/packages/shared/src/features/inAppAgent/types.tsindex fd5724e6e7..dbff305eab 100644--- a/packages/shared/src/features/inAppAgent/types.ts+++ b/packages/shared/src/features/inAppAgent/types.ts@@ -79,8 +79,16 @@ export const InAppAgentRunRequestSchema = z.discriminatedUnion("kind", [}),z.object({kind: z.literal("approvalDecision"),- /** Provenance for debugging only; no code path queries lineage. */+ /** Immediate parent run used to derive durable approval-chain lineage. */parentRunId: z.string(),+ /** Original user-message run whose trace owns the complete approval chain. */+ rootRunId: z.string().optional(),+ /** Stable root trace timestamp propagated across durable continuations. */+ traceStartedAt: z.iso.datetime({ offset: true }).optional(),+ /** Time at which the parent run parked and began waiting for approval. */+ approvalRequestedAt: z.iso.datetime({ offset: true }).optional(),+ /** One-based position in the current user turn's approval continuation chain. */+ continuationNumber: z.number().int().positive().optional(),toolCallId: z.string(),approved: z.boolean(),/** Inherited sanitized context; defaults for legacy continuation rows. */diff --git a/packages/shared/src/in-app-agent/constants.ts b/packages/shared/src/in-app-agent/constants.tsindex 2d9cf98d69..3f738b0262 100644--- a/packages/shared/src/in-app-agent/constants.ts+++ b/packages/shared/src/in-app-agent/constants.ts@@ -29,3 +29,10 @@ export const getInAppAgentInstrumentationObservationId = (runId: string) =>// level telemetry and feedback aggregate on the same per-turn trace.export const getInAppAgentInstrumentationTraceId = (runId: string) =>`${runId}-trace`;++// Per-LLM-call generation ids stay deterministic from the run so retries and+// late flushes can address the same observation.+export const getInAppAgentLlmCallObservationId = (+ runId: string,+ stepNumber: number,+) => `${runId}-llm-${stepNumber}`;diff --git a/packages/shared/src/in-app-agent/schema.ts b/packages/shared/src/in-app-agent/schema.tsindex c07d96ade0..b277458789 100644--- a/packages/shared/src/in-app-agent/schema.ts+++ b/packages/shared/src/in-app-agent/schema.ts@@ -239,6 +239,11 @@ export const ResumeForwardedPropsSchema = z.object({command: z.object({resume: z.object({approved: z.boolean(),+ continuationNumber: z.number().int().positive().optional(),+ rootRunId: z.string().min(1).optional(),+ traceStartedAt: z.iso.datetime({ offset: true }).optional(),+ approvalRequestedAt: z.iso.datetime({ offset: true }).optional(),+ approvalDecidedAt: z.iso.datetime({ offset: true }).optional(),approvalRequest: InAppAgentToolApprovalRequestSchema,}),}),diff --git a/packages/shared/src/in-app-agent/server/agent.ts b/packages/shared/src/in-app-agent/server/agent.tsindex 3403c300fa..e5485b6dca 100644--- a/packages/shared/src/in-app-agent/server/agent.ts+++ b/packages/shared/src/in-app-agent/server/agent.ts@@ -33,6 +33,7 @@ import {filterInAppAgentAvailableLangfuseMcpTools,getInAppAgentMcpAllowedToolNames,getInAppAgentRegistryToolName,+ getToolCallId,hasCallableExecute,type CompletedInAppAgentMcpToolCall,type InAppAgentToolPolicy,@@ -237,10 +238,63 @@ export async function createAgUiStream(params: {: undefined,model: params.options.awsBedrock.modelId,});- instrumentation?.recordAvailableSkills?.(LANGFUSE_IN_APP_AGENT_SKILLS);+ const recordInstrumentation = (+ operation: string,+ callback: (+ activeInstrumentation: NonNullable<typeof instrumentation>,+ ) => void,+ ) => {+ if (!instrumentation) {+ return;+ }++ try {+ callback(instrumentation);+ } catch (error) {+ logger.warn("Failed to record in-app agent Langfuse tracing", {+ error,+ operation,+ runId: params.input.runId,+ threadId: params.input.threadId,+ });+ }+ };+ recordInstrumentation("recordAvailableSkills", (instrumentation) =>+ instrumentation.recordAvailableSkills?.(LANGFUSE_IN_APP_AGENT_SKILLS),+ );const onStepFinish = instrumentation? (event: unknown) => {- instrumentation.recordStepFinish?.(event);+ recordInstrumentation("recordStepFinish", (instrumentation) =>+ instrumentation.recordStepFinish?.(event),+ );+ }+ : undefined;+ const onChunk = instrumentation+ ? (chunk: unknown) => {+ recordInstrumentation("recordStreamChunk", (instrumentation) =>+ instrumentation.recordStreamChunk?.(chunk),+ );+ }+ : undefined;+ const onModelCallFinish = instrumentation+ ? (event: unknown) => {+ recordInstrumentation("recordModelCallFinish", (instrumentation) =>+ instrumentation.recordModelCallFinish?.(event),+ );+ }+ : undefined;+ const onToolExecutionStart = instrumentation+ ? (toolCallId: string) => {+ recordInstrumentation("recordToolExecutionStart", (instrumentation) =>+ instrumentation.recordToolExecutionStart?.(toolCallId),+ );+ }+ : undefined;+ const onToolExecutionEnd = instrumentation+ ? (toolCallId: string) => {+ recordInstrumentation("recordToolExecutionEnd", (instrumentation) =>+ instrumentation.recordToolExecutionEnd?.(toolCallId),+ );}: undefined;@@ -324,8 +378,12 @@ export async function createAgUiStream(params: {return;}- instrumentation?.endWithError(error);- instrumentation?.flush();+ recordInstrumentation("endWithError", (instrumentation) =>+ instrumentation.endWithError(error),+ );+ recordInstrumentation("flush", (instrumentation) =>+ instrumentation.flush(),+ );ending = true;closed = true;shouldEnqueue = false;@@ -429,8 +487,12 @@ export async function createAgUiStream(params: {return;}- instrumentation?.end({ aborted: true });- instrumentation?.flush();+ recordInstrumentation("end", (instrumentation) =>+ instrumentation.end({ aborted: true }),+ );+ recordInstrumentation("flush", (instrumentation) =>+ instrumentation.flush(),+ );ending = true;shouldEnqueue = false;removeAbortHandler();@@ -489,8 +551,14 @@ export async function createAgUiStream(params: {awsProfile,instructions,onToolsAvailable: (tools) =>- instrumentation?.recordAvailableTools?.(tools),+ recordInstrumentation("recordAvailableTools", (instrumentation) =>+ instrumentation.recordAvailableTools?.(tools),+ ),onStepFinish,+ onChunk,+ onModelCallFinish,+ onToolExecutionStart,+ onToolExecutionEnd,}).then(async (initialAdapter) => {if (ending || closed || params.signal.aborted) {@@ -559,6 +627,10 @@ export async function createAgUiStream(params: {awsProfile,instructions,onStepFinish,+ onChunk,+ onModelCallFinish,+ onToolExecutionStart,+ onToolExecutionEnd,});if (ending || closed || params.signal.aborted) {@@ -595,7 +667,9 @@ export async function createAgUiStream(params: {params.input,);- instrumentation?.recordEvents(agUiEvents);+ recordInstrumentation("recordEvents", (instrumentation) =>+ instrumentation.recordEvents(agUiEvents),+ );for (const agUiEvent of agUiEvents) {if (@@ -616,10 +690,16 @@ export async function createAgUiStream(params: {agUiEvent.type === EventType.RUN_STARTED &&pendingSyntheticEvents.length > 0) {- instrumentation?.recordToolCallApproval(- runInput.toolCallApproval,+ recordInstrumentation(+ "recordToolCallApproval",+ (instrumentation) =>+ instrumentation.recordToolCallApproval(+ runInput.toolCallApproval,+ ),+ );+ recordInstrumentation("recordEvents", (instrumentation) =>+ instrumentation.recordEvents(pendingSyntheticEvents),);- instrumentation?.recordEvents(pendingSyntheticEvents);for (const syntheticEvent of pendingSyntheticEvents) {enqueueEvent(syntheticEvent);}@@ -639,7 +719,9 @@ export async function createAgUiStream(params: {if (streamedRunError !== null) {closeController(() => {- instrumentation?.flush();+ recordInstrumentation("flush", (instrumentation) =>+ instrumentation.flush(),+ );return handleStreamedRunError();});return;@@ -652,11 +734,17 @@ export async function createAgUiStream(params: {});const runErrorEvent = createRunErrorEvent(params.input, error);- instrumentation?.recordEvents([runErrorEvent]);+ recordInstrumentation("recordEvents", (instrumentation) =>+ instrumentation.recordEvents([runErrorEvent]),+ );enqueueEvent(runErrorEvent, () =>params.options.onError?.(error),);- closeController(() => instrumentation?.flush());+ closeController(() =>+ recordInstrumentation("flush", (instrumentation) =>+ instrumentation.flush(),+ ),+ );},complete() {if (ending || closed) {@@ -671,12 +759,18 @@ export async function createAgUiStream(params: {closeController(streamedRunError === null? () => {- instrumentation?.end({});- instrumentation?.flush();+ recordInstrumentation("end", (instrumentation) =>+ instrumentation.end({}),+ );+ recordInstrumentation("flush", (instrumentation) =>+ instrumentation.flush(),+ );return params.options.onComplete?.();}: () => {- instrumentation?.flush();+ recordInstrumentation("flush", (instrumentation) =>+ instrumentation.flush(),+ );return handleStreamedRunError();},);@@ -700,9 +794,15 @@ export async function createAgUiStream(params: {});const runErrorEvent = createRunErrorEvent(params.input, error);- instrumentation?.recordEvents([runErrorEvent]);+ recordInstrumentation("recordEvents", (instrumentation) =>+ instrumentation.recordEvents([runErrorEvent]),+ );enqueueEvent(runErrorEvent, () => params.options.onError?.(error));- closeController(() => instrumentation?.flush());+ closeController(() =>+ recordInstrumentation("flush", (instrumentation) =>+ instrumentation.flush(),+ ),+ );});},cancel() {@@ -711,8 +811,12 @@ export async function createAgUiStream(params: {}ending = true;- instrumentation?.end({ aborted: true });- instrumentation?.flush();+ recordInstrumentation("end", (instrumentation) =>+ instrumentation.end({ aborted: true }),+ );+ recordInstrumentation("flush", (instrumentation) =>+ instrumentation.flush(),+ );shouldEnqueue = false;removeAbortHandler();interruptAdapter?.();@@ -751,6 +855,85 @@ type ExecutableInAppAgentTool = {toModelOutput?: (output: unknown) => unknown | PromiseLike<unknown>;};+type BedrockLanguageModel = ReturnType<ReturnType<typeof createAmazonBedrock>>;++function withModelCallFinish(+ model: BedrockLanguageModel,+ onFinish?: (event: unknown) => void,+): BedrockLanguageModel {+ if (!onFinish) {+ return model;+ }++ return {+ specificationVersion: model.specificationVersion,+ provider: model.provider,+ modelId: model.modelId,+ supportedUrls: model.supportedUrls,+ doGenerate: (options) => model.doGenerate(options),+ doStream: async (options) => {+ const result = await model.doStream(options);++ return {+ ...result,+ stream: result.stream.pipeThrough(+ new TransformStream({+ transform(part, controller) {+ if (part.type === "finish") {+ onFinish(part);+ }+ controller.enqueue(part);+ },+ }),+ ),+ };+ },+ };+}++function withToolExecutionTiming<TTool>(params: {+ tools: Record<string, TTool>;+ onStart?: (toolCallId: string) => void;+ onEnd?: (toolCallId: string) => void;+}): Record<string, TTool> {+ if (!params.onStart && !params.onEnd) {+ return params.tools;+ }++ return Object.fromEntries(+ Object.entries(params.tools).map(([toolName, tool]) => {+ if (!hasCallableExecute(tool)) {+ return [toolName, tool];+ }++ const execute = tool.execute.bind(tool) as (+ inputData: unknown,+ context: unknown,+ ) => Promise<unknown>;++ return [+ toolName,+ {+ ...tool,+ execute: async (inputData: unknown, context: unknown) => {+ const toolCallId = getToolCallId(context);+ if (!toolCallId) {+ return execute(inputData, context);+ }++ params.onStart?.(toolCallId);+ try {+ return await execute(inputData, context);+ } finally {+ params.onEnd?.(toolCallId);+ }+ },+ } as TTool,+ ];+ }),+ );+}+async function createMastraAdapter(params: {input: AgUiRunAgentInput;signal: AbortSignal;@@ -760,6 +943,10 @@ async function createMastraAdapter(params: {instructions: string;onToolsAvailable?: (tools: Record<string, unknown>) => void;onStepFinish?: (event: unknown) => void;+ onChunk?: (chunk: unknown) => void;+ onModelCallFinish?: (event: unknown) => void;+ onToolExecutionStart?: (toolCallId: string) => void;+ onToolExecutionEnd?: (toolCallId: string) => void;}) {const bedrock = createAmazonBedrock({...(params.options.awsBedrock.region@@ -842,39 +1029,43 @@ async function createMastraAdapter(params: {// agent.stream(..., { toolsets }) call. Keep Mastra's per-request MCP// discovery, then prefix tool names for constructor-based tools so the// model sees the same names that later appear in AG-UI tool-call events.- const tools = withInAppAgentToolApproval(- {- ...withOptionalSilentMcpOutput({- tools: withLangfuseMcpRateLimitWait({- tools: prefixToolsetTools(- "langfuse",- filterInAppAgentAvailableLangfuseMcpTools({- tools: toolsets.langfuse,- policy: params.options.langfuseMcp.toolPolicy,- }),- ),- signal: params.signal,- runId: params.input.runId,- threadId: params.input.threadId,+ const tools = withToolExecutionTiming({+ tools: withInAppAgentToolApproval(+ {+ ...withOptionalSilentMcpOutput({+ tools: withLangfuseMcpRateLimitWait({+ tools: prefixToolsetTools(+ "langfuse",+ filterInAppAgentAvailableLangfuseMcpTools({+ tools: toolsets.langfuse,+ policy: params.options.langfuseMcp.toolPolicy,+ }),+ ),+ signal: params.signal,+ runId: params.input.runId,+ threadId: params.input.threadId,+ }),+ sandbox: params.options.sandbox,+ onToolCallCompleted: params.options.onMcpToolCallCompleted,}),- sandbox: params.options.sandbox,- onToolCallCompleted: params.options.onMcpToolCallCompleted,- }),- ...withOptionalSilentMcpOutput({- tools: prefixToolsetTools("langfuseDocs", toolsets.langfuseDocs),- sandbox: params.options.sandbox,- onToolCallCompleted: params.options.onMcpToolCallCompleted,- }),- [IN_APP_AGENT_REDIRECT_TOOL_NAME]: createRedirectActionTool({- projectId: params.options.redirectAction.projectId,- isV4Enabled: params.options.redirectAction.isV4Enabled,- }),- ...(params.options.sandbox- ? createSandboxTools(params.options.sandbox)- : {}),- },- params.options.langfuseMcp.toolPolicy,- );+ ...withOptionalSilentMcpOutput({+ tools: prefixToolsetTools("langfuseDocs", toolsets.langfuseDocs),+ sandbox: params.options.sandbox,+ onToolCallCompleted: params.options.onMcpToolCallCompleted,+ }),+ [IN_APP_AGENT_REDIRECT_TOOL_NAME]: createRedirectActionTool({+ projectId: params.options.redirectAction.projectId,+ isV4Enabled: params.options.redirectAction.isV4Enabled,+ }),+ ...(params.options.sandbox+ ? createSandboxTools(params.options.sandbox)+ : {}),+ },+ params.options.langfuseMcp.toolPolicy,+ ),+ onStart: params.onToolExecutionStart,+ onEnd: params.onToolExecutionEnd,+ });params.onToolsAvailable?.(tools);const reasoningProviderOptions = getBedrockReasoningProviderOptions(@@ -887,14 +1078,18 @@ async function createMastraAdapter(params: {// instruction channel so the model receives the same higher-priority// guidance on resumed runs.let developerGuidance: string | undefined;+ const model = withModelCallFinish(+ bedrock(+ params.options.awsBedrock.modelId as Parameters<typeof bedrock>[0],+ ),+ params.onModelCallFinish,+ );const agent = new Agent({id: "langfuse-in-app-assistant",name: ASSISTANT_TITLE,instructions: () =>[params.instructions, developerGuidance].filter(Boolean).join("\n\n"),- model: bedrock(- params.options.awsBedrock.modelId as Parameters<typeof bedrock>[0],- ),+ model,skills: LANGFUSE_IN_APP_AGENT_SKILLS,tools,maxRetries: 2,@@ -907,6 +1102,8 @@ async function createMastraAdapter(params: {// Fires once per LLM call with that call's token usage; the AG-UI// event stream itself never carries usage....(params.onStepFinish ? { onStepFinish: params.onStepFinish } : {}),+ // Tool execution windows (and optional step-start timing bookmarks).+ ...(params.onChunk ? { onChunk: params.onChunk } : {}),...(reasoningProviderOptions? { providerOptions: reasoningProviderOptions }: {}),diff --git a/packages/shared/src/in-app-agent/server/instrumentation.ts b/packages/shared/src/in-app-agent/server/instrumentation.tsindex cec9a4d940..1b163f206a 100644--- a/packages/shared/src/in-app-agent/server/instrumentation.ts+++ b/packages/shared/src/in-app-agent/server/instrumentation.ts@@ -4,8 +4,14 @@ import { getInternalTracingHandler, logger } from "../../server";import {getInAppAgentInstrumentationObservationId,getInAppAgentInstrumentationTraceId,+ getInAppAgentLlmCallObservationId,} from "../constants";-import type { AgUiEvent, AgUiMessage, AgUiRunAgentInput } from "../schema";+import {+ ResumeForwardedPropsSchema,+ type AgUiEvent,+ type AgUiMessage,+ type AgUiRunAgentInput,+} from "../schema";import { compactTextMessageChunks } from "./eventCompaction";import {getToolFailureMessage,@@ -43,11 +49,11 @@ export type InAppAgentPromptMetadata = {};const IN_APP_AGENT_TURN_NAME = "agent-turn";+const IN_APP_AGENT_LLM_CALL_NAME = "invoke-model";type InternalTracingHandler = ReturnType<typeof getInternalTracingHandler>;type InAppAgentTrace = ReturnType<InternalTracingHandler["handler"]["langfuse"]["trace"]>;-type InAppAgentGeneration = ReturnType<InAppAgentTrace["generation"]>;type InAppAgentLangfuse = InternalTracingHandler["handler"]["langfuse"];type AgentRunToolCall = {id: string;@@ -82,19 +88,23 @@ type AgentRunChatMessage = {tool_calls?: AgentRunToolCall[];tool_call_id?: string;};-type ToolObservationBody = {+type ObservationBody = {id: string;traceId: string;- parentObservationId: string | null;+ parentObservationId?: string | null;name: string;startTime: Date;endTime: Date;- completionStartTime: Date;+ completionStartTime?: Date;input?: unknown;output?: unknown;- level?: "ERROR";- statusMessage?: string;+ model?: string;+ usageDetails?: Record<string, number>;+ level?: "DEFAULT" | "ERROR";+ statusMessage?: string | null;metadata?: Record<string, unknown>;+ promptName?: string;+ promptVersion?: number;};type ToolCallApprovalStatus = "approved" | "rejected";type AgentRunToolSpan = {@@ -106,6 +116,32 @@ type AgentRunToolSpan = {failureMessage?: string;parentMessageId?: string;};+// Lightweight bookmark for the in-flight Mastra step. Generations are emitted+// from onStepFinish (which carries usage); this only supplies start timing and+// the input-message boundary for context-growth visibility.+type OpenLlmStep = {+ stepNumber: number;+ startTime: Date;+ inputMessageCount: number;+ completionStartTime?: Date;+ providerFinish?: Record<string, unknown>;+ providerFinishTime?: Date;+};+type ToolExecutionTimes = {+ startTime?: Date;+ endTime?: Date;+};+type ApprovalContinuation = {+ status: ToolCallApprovalStatus;+ continuationNumber: number;+ rootRunId: string;+ toolCallId: string;+ toolName: string;+ traceStartedAt?: Date;+ approvalRequestedAt?: Date;+ approvalDecidedAt?: Date;+ metadata: Record<string, unknown>;+};export function createInAppAgentInstrumentation({input,@@ -140,24 +176,39 @@ export class InAppAgentInstrumentation {private readonly processTracedEvents: () => Promise<void>;private readonly langfuse: InAppAgentLangfuse;private readonly trace: InAppAgentTrace;- private readonly agentRun: InAppAgentGeneration;+ private readonly runId: string;+ private readonly traceId: string;+ private readonly rootObservationId: string;+ private readonly traceStartTime: Date;+ private readonly agentRunStartTime: Date;private agentRunInput: unknown;+ private readonly approvalContinuation?: ApprovalContinuation;private readonly prompt?: InAppAgentPromptMetadata;private readonly toolSpans = new Map<string, AgentRunToolSpan>();private readonly toolCallApprovals = new Map<string,ToolCallApprovalStatus>();+ private readonly toolExecutionTimes = new Map<string, ToolExecutionTimes>();private readonly metadata: Record<string, unknown>;private readonly agentRunOutputMessages: AgentRunChatMessage[] = [];private readonly agentRunToolCalls: AgentRunToolCall[] = [];private output = "";private currentReasoning = "";private pendingThinking: AgentRunThinkingPart[] = [];- private completionStartTime?: Date;private ended = false;private readonly model?: string;- private usageDetails?: Record<string, number>;+ private nextStepNumber = 0;+ private openLlmStep?: OpenLlmStep;+ // Tools that finished while an LLM step was still open. Emit after the+ // generation so the tree sorts model → tools (tool-call chunks arrive during+ // the LLM stream, before onStepFinish).+ private readonly deferredToolCompletions = new Map<+ string,+ AgentRunToolSpan+ >();+ // Clamp tool execution windows to start at/after the last generation end.+ private lastLlmGenerationEndTime?: Date;constructor(params: {input: AgUiRunAgentInput;@@ -172,6 +223,7 @@ export class InAppAgentInstrumentation {prompt?: InAppAgentPromptMetadata;model?: string;}) {+ this.approvalContinuation = getApprovalContinuation(params.input);this.metadata = {...params.metadata,...(params.userEmail ? { langfuse_user_email: params.userEmail } : {}),@@ -185,14 +237,28 @@ export class InAppAgentInstrumentation {prompt_version: params.prompt.version,}: {}),+ ...(this.approvalContinuation+ ? {+ approval_continuation_count:+ this.approvalContinuation.continuationNumber,+ }+ : {}),};this.agentRunInput = getAgentRunInput(params.input);this.prompt = params.prompt;this.model = params.model;+ this.runId = params.runId;+ const rootRunId = this.approvalContinuation?.rootRunId ?? params.runId;+ this.traceId = getInAppAgentInstrumentationTraceId(rootRunId);+ this.rootObservationId =+ getInAppAgentInstrumentationObservationId(rootRunId);+ this.agentRunStartTime = new Date();+ this.traceStartTime =+ this.approvalContinuation?.traceStartedAt ?? this.agentRunStartTime;const traceSinkParams = {targetProjectId: params.targetProjectId,- traceId: getInAppAgentInstrumentationTraceId(params.runId),+ traceId: this.traceId,traceName: IN_APP_AGENT_TURN_NAME,environment: params.environment,userId: params.userId,@@ -205,26 +271,21 @@ export class InAppAgentInstrumentation {this.langfuse = handler.langfuse;this.trace = this.langfuse.trace({- id: getInAppAgentInstrumentationTraceId(params.runId),+ id: this.traceId,name: IN_APP_AGENT_TURN_NAME,userId: params.userId,sessionId: params.input.threadId,- input: this.agentRunInput,+ timestamp: this.traceStartTime,+ input: this.getTraceInput(),metadata: this.metadata,tags: ["in-app-agent"],});- this.agentRun = this.trace.generation({- id: getInAppAgentInstrumentationObservationId(params.input.runId),- name: IN_APP_AGENT_TURN_NAME,- input: this.agentRunInput,- metadata: this.metadata,- ...(params.prompt- ? {- promptName: params.prompt.name,- promptVersion: params.prompt.version,- }- : {}),- });++ // Create the root observation immediately so tool/generation children+ // emitted during the run have a parent that already exists (avoids+ // orphans that attach to the trace as siblings of the late agent).+ this.emitRootAgentObservation({ metadata: this.metadata });+ this.emitApprovalWaitObservation();}recordEvents(events: AgUiEvent[]) {@@ -267,48 +328,140 @@ export class InAppAgentInstrumentation {this.toolCallApprovals.set(approval.toolCallId, approval.status);}- // Receives Mastra's per-LLM-call onStepFinish event and accumulates its- // token usage onto the agent-turn generation, since the AG-UI event stream- // itself never carries usage.- recordStepFinish(event: unknown) {+ recordToolExecutionStart(toolCallId: string) {if (this.ended) {return;}- const usage = isRecord(event) ? event.usage : undefined;- if (!isRecord(usage)) {+ const times = this.toolExecutionTimes.get(toolCallId) ?? {};+ times.startTime ??= new Date();+ this.toolExecutionTimes.set(toolCallId, times);+ }++ recordToolExecutionEnd(toolCallId: string) {+ if (this.ended) {return;}- const inputTokens = getFiniteNumber(usage.inputTokens);- const outputTokens = getFiniteNumber(usage.outputTokens);- if (inputTokens === undefined && outputTokens === undefined) {+ const times = this.toolExecutionTimes.get(toolCallId) ?? {};+ times.endTime ??= new Date();+ this.toolExecutionTimes.set(toolCallId, times);+ }++ // Bedrock's V3 stream emits exact usage before Mastra executes tools. Keep+ // it on the open step as a fallback because approval suspension can end the+ // run without Mastra ever invoking onStepFinish.+ recordModelCallFinish(event: unknown) {+ if (!isRecord(event)) {return;}- const cacheReadTokens = getFiniteNumber(usage.cachedInputTokens) ?? 0;- const cacheWriteTokens =- getFiniteNumber(usage.cacheCreationInputTokens) ?? 0;- const totals = (this.usageDetails ??= {- input: 0,- output: 0,- cache_read_input_tokens: 0,- cache_creation_input_tokens: 0,- total: 0,- });+ if (this.ended) {+ return;+ }- // Mastra's inputTokens includes cache reads/writes, while Langfuse model- // prices bill `input` as non-cached input tokens with separate cache keys.- totals.input += Math.max(- 0,- (inputTokens ?? 0) - cacheReadTokens - cacheWriteTokens,- );- totals.output += outputTokens ?? 0;- totals.cache_read_input_tokens += cacheReadTokens;- totals.cache_creation_input_tokens += cacheWriteTokens;- totals.total +=- getFiniteNumber(usage.totalTokens) ??- (inputTokens ?? 0) + (outputTokens ?? 0);+ const now = new Date();+ this.ensureOpenLlmStep(now);+ if (this.openLlmStep) {+ this.openLlmStep.providerFinish = event;+ this.openLlmStep.providerFinishTime = now;+ }+ }++ // Mastra stream chunks: step-start (or first model delta) bookmarks the LLM+ // window; tool-result ends tool execution. Do not treat tool-call chunks as+ // execution start — they fire while the model is still streaming.+ // Usage and output still come from recordStepFinish.+ recordStreamChunk(chunk: unknown) {+ if (this.ended || !isRecord(chunk)) {+ return;+ }++ const type = chunk.type;+ if (typeof type !== "string") {+ return;+ }++ const payload = isRecord(chunk.payload) ? chunk.payload : {};+ const now = new Date();++ if (type === "step-start") {+ // A prior step should have been closed by onStepFinish. If not, emit it+ // as an error so we never silently drop a generation.+ if (this.openLlmStep) {+ const providerFinish = this.openLlmStep.providerFinish;+ this.emitLlmGeneration(+ this.openLlmStep,+ undefined,+ providerFinish+ ? undefined+ : {+ level: "ERROR",+ statusMessage: "LLM step closed by a subsequent step-start",+ },+ );+ this.flushDeferredToolCompletions();+ }++ this.openLlmStep = {+ stepNumber: ++this.nextStepNumber,+ startTime: now,+ inputMessageCount: this.agentRunOutputMessages.length,+ };+ return;+ }++ // tool-call means "model requested this tool", not "tool started".+ // Ensure an LLM step is open so missing step-start still gets a real window.+ if (type === "tool-call") {+ this.ensureOpenLlmStep(now);+ return;+ }++ if (type === "tool-result" || type === "tool-error") {+ const toolCallId = getStringValue(payload.toolCallId);+ if (!toolCallId) {+ return;+ }++ const times = this.toolExecutionTimes.get(toolCallId) ?? {};+ times.endTime ??= now;+ this.toolExecutionTimes.set(toolCallId, times);+ return;+ }++ if (+ type === "text-delta" ||+ type === "reasoning-delta" ||+ type === "reasoning"+ ) {+ this.ensureOpenLlmStep(now);+ if (this.openLlmStep) {+ this.openLlmStep.completionStartTime ??= now;+ }+ }+ }++ // Receives Mastra's per-LLM-call onStepFinish event and emits one generation+ // with that call's usage/model. The AG-UI event stream itself never carries usage.+ recordStepFinish(event: unknown) {+ if (this.ended) {+ return;+ }++ const step =+ this.openLlmStep ??+ ({+ stepNumber: ++this.nextStepNumber,+ // Prefer a non-zero window when step-start never arrived: backdate to+ // the last generation end (or run start) so tools do not sort above us.+ startTime: this.lastLlmGenerationEndTime ?? this.agentRunStartTime,+ inputMessageCount: this.agentRunOutputMessages.length,+ } satisfies OpenLlmStep);++ this.emitLlmGeneration(step, event);+ this.openLlmStep = undefined;+ this.flushDeferredToolCompletions();}recordAvailableSkills(skills: unknown[]) {@@ -334,35 +487,22 @@ export class InAppAgentInstrumentation {}const message = error instanceof Error ? error.message : String(error);+ this.endOpenLlmStep({ error: message }, message);this.endOpenToolSpans({ error: message }, message);this.flushTrailingThinking();- this.agentRun.update({- name: IN_APP_AGENT_TURN_NAME,- input: this.agentRunInput,- output: this.getAgentRunOutput(),- ...(this.completionStartTime- ? { completionStartTime: this.completionStartTime }- : {}),- ...this.getAgentRunUsage(),+ this.emitRootAgentObservation({level: "ERROR",statusMessage: message,metadata: {...this.metadata,error: message,},- ...(this.prompt- ? {- promptName: this.prompt.name,- promptVersion: this.prompt.version,- }- : {}),});this.trace.update({- input: this.agentRunInput,+ input: this.getTraceInput(),output: this.getAgentRunOutput(),metadata: { ...this.metadata, error: message },});- this.agentRun.end();this.ended = true;}@@ -371,6 +511,11 @@ export class InAppAgentInstrumentation {return;}+ if (params?.aborted) {+ this.endOpenLlmStep({ aborted: true }, "aborted");+ } else {+ this.endOpenLlmStep();+ }this.endOpenToolSpans(params?.aborted ? { aborted: true } : undefined);this.flushTrailingThinking();const metadata = {@@ -378,28 +523,12 @@ export class InAppAgentInstrumentation {...(params?.aborted ? { aborted: true } : {}),...(params?.result ? { result: params.result } : {}),};- this.agentRun.update({- name: IN_APP_AGENT_TURN_NAME,- input: this.agentRunInput,- output: this.getAgentRunOutput(),- ...(this.completionStartTime- ? { completionStartTime: this.completionStartTime }- : {}),- ...this.getAgentRunUsage(),- metadata,- ...(this.prompt- ? {- promptName: this.prompt.name,- promptVersion: this.prompt.version,- }- : {}),- });+ this.emitRootAgentObservation({ metadata });this.trace.update({- input: this.agentRunInput,+ input: this.getTraceInput(),output: this.getAgentRunOutput(),metadata,});- this.agentRun.end();this.ended = true;}@@ -513,6 +642,8 @@ export class InAppAgentInstrumentation {const name =typeof event.toolCallName === "string" ? event.toolCallName : "tool-call";+ // Execution start = AG-UI tool-call start (or later clamp to generation end).+ // Mastra tool-call chunks are request-time and must not set this.this.toolSpans.set(event.toolCallId, {name,startTime: new Date(),@@ -570,6 +701,14 @@ export class InAppAgentInstrumentation {return;}+ // Tool-call args/results often arrive while the LLM step is still open.+ // Hold the observation until step-finish so the tree sorts generation → tools.+ if (this.openLlmStep) {+ this.deferredToolCompletions.set(toolCallId, tool);+ this.toolSpans.delete(toolCallId);+ return;+ }+this.createToolObservation(toolCallId, tool);this.toolSpans.delete(toolCallId);}@@ -590,14 +729,28 @@ export class InAppAgentInstrumentation {tool.failureMessage ??getToolFailureMessage(undefined, output);const toolCallApproval = this.toolCallApprovals.get(toolCallId);- const body: ToolObservationBody = {+ const executionTimes = this.toolExecutionTimes.get(toolCallId);+ const rawStartTime = executionTimes?.startTime ?? tool.startTime;+ const endTime =+ toolCallApproval === "rejected"+ ? rawStartTime+ : (executionTimes?.endTime ?? new Date());+ // Keep causal order: tool execution cannot start before the generation that+ // requested it finished (or the last known generation end).+ const startTime =+ this.lastLlmGenerationEndTime &&+ rawStartTime.getTime() < this.lastLlmGenerationEndTime.getTime()+ ? this.lastLlmGenerationEndTime+ : rawStartTime;+ const body: ObservationBody = {id: toolCallId,- traceId: this.agentRun.traceId,- parentObservationId: this.agentRun.observationId,+ traceId: this.traceId,+ parentObservationId: this.rootObservationId,name: tool.name,- startTime: tool.startTime,- endTime: new Date(),- completionStartTime: tool.startTime,+ startTime: startTime.getTime() > endTime.getTime() ? endTime : startTime,+ endTime,+ completionStartTime:+ startTime.getTime() > endTime.getTime() ? endTime : startTime,input,output,...(failureMessage@@ -616,38 +769,236 @@ export class InAppAgentInstrumentation {this.recordToolCall(toolCallId, tool, output);this.toolCallApprovals.delete(toolCallId);+ this.toolExecutionTimes.delete(toolCallId);- (- this.langfuse as unknown as {- enqueue: (type: string, body: ToolObservationBody) => void;- }- ).enqueue("tool-create", body);+ this.enqueueObservation("tool-create", body);}- // Model is only set alongside provided usage; on its own it would make- // ingestion tokenize the agent-turn input/output into estimated usage.- private getAgentRunUsage() {- if (!this.usageDetails) {- return {};+ private emitRootAgentObservation(params: {+ metadata: Record<string, unknown>;+ level?: "ERROR";+ statusMessage?: string;+ }) {+ this.enqueueObservation("agent-create", {+ id: this.rootObservationId,+ traceId: this.traceId,+ name: IN_APP_AGENT_TURN_NAME,+ startTime: this.traceStartTime,+ endTime: new Date(),+ input: this.getTraceInput(),+ output: this.getAgentRunOutput(),+ metadata: params.metadata,+ ...(this.prompt+ ? {+ promptName: this.prompt.name,+ promptVersion: this.prompt.version,+ }+ : {}),+ ...(params.level ? { level: params.level } : {}),+ ...(params.statusMessage ? { statusMessage: params.statusMessage } : {}),+ });+ }++ private emitLlmGeneration(+ step: OpenLlmStep,+ event: unknown,+ options?: {+ level?: "ERROR";+ statusMessage?: string;+ },+ ) {+ const eventRecord = isRecord(event) ? event : undefined;+ const providerFinish = step.providerFinish;+ const usageDetails =+ getStepUsageDetails(eventRecord?.usage) ??+ getStepUsageDetails(providerFinish?.usage);+ const finishReason =+ getStepFinishReason(eventRecord) ?? getStepFinishReason(providerFinish);+ const model =+ getModelIdFromStepEvent(eventRecord) ?? this.model ?? undefined;+ // Mastra can delay onStepFinish until after requested tools execute. The+ // raw provider finish is the actual model boundary and keeps the following+ // tool's execute() window intact instead of clamping it to zero duration.+ const endTime = step.providerFinishTime ?? new Date();+ this.lastLlmGenerationEndTime = endTime;+ const id = getInAppAgentLlmCallObservationId(this.runId, step.stepNumber);++ this.enqueueObservation("generation-create", {+ id,+ traceId: this.traceId,+ parentObservationId: this.rootObservationId,+ name: IN_APP_AGENT_LLM_CALL_NAME,+ startTime: step.startTime,+ endTime,+ completionStartTime: step.completionStartTime ?? step.startTime,+ input: this.getLlmStepInput(step.inputMessageCount),+ output: this.getLlmStepOutput(step, eventRecord),+ // Model only travels with usage so Langfuse does not invent tokenizer-+ // estimated costs for generations that never reported tokens.+ ...(usageDetails+ ? {+ usageDetails,+ ...(model ? { model } : {}),+ }+ : {}),+ ...(options?.level ? { level: options.level } : {}),+ ...(options?.statusMessage+ ? { statusMessage: options.statusMessage }+ : {}),+ metadata: {+ ...(finishReason ? { finish_reason: finishReason } : {}),+ ...(options?.statusMessage ? { error: options.statusMessage } : {}),+ },+ });+ }++ private getLlmStepInput(inputMessageCount: number) {+ const base = isRecord(this.agentRunInput) ? this.agentRunInput : {};+ const baseMessages = Array.isArray(base.messages) ? base.messages : [];++ return {+ ...base,+ messages: [+ ...baseMessages,+ ...this.agentRunOutputMessages.slice(0, inputMessageCount),+ ],+ };+ }++ private getTraceInput() {+ if (!this.approvalContinuation || !isRecord(this.agentRunInput)) {+ return this.agentRunInput;}+ const messages = Array.isArray(this.agentRunInput.messages)+ ? this.agentRunInput.messages.filter((message) => {+ if (!isRecord(message)) {+ return false;+ }++ return (+ message.role === "developer" ||+ message.role === "system" ||+ message.role === "user"+ );+ })+ : [];+return {- usageDetails: this.usageDetails,- ...(this.model ? { model: this.model } : {}),+ ...this.agentRunInput,+ messages,};}+ private getLlmStepOutput(+ step: OpenLlmStep,+ event: Record<string, unknown> | undefined,+ ) {+ const text = typeof event?.text === "string" ? event.text : undefined;+ const toolCalls = Array.isArray(event?.toolCalls)+ ? event.toolCalls+ : undefined;+ if (text !== undefined || toolCalls?.length) {+ return {+ ...(text !== undefined ? { text } : {}),+ ...(toolCalls?.length ? { tool_calls: toolCalls } : {}),+ };+ }++ const response = isRecord(event?.response) ? event.response : undefined;+ if (Array.isArray(response?.messages) && response.messages.length > 0) {+ return { messages: response.messages };+ }++ const messages = this.agentRunOutputMessages.slice(step.inputMessageCount);+ return messages.length > 0 ? { messages } : undefined;+ }++ private endOpenLlmStep(+ metadata?: Record<string, unknown>,+ statusMessage?: string,+ ) {+ if (!this.openLlmStep) {+ return;+ }++ this.emitLlmGeneration(this.openLlmStep, undefined, {+ ...(!this.openLlmStep.providerFinish && (statusMessage || metadata)+ ? {+ level: "ERROR" as const,+ statusMessage:+ statusMessage ??+ (typeof metadata?.error === "string"+ ? metadata.error+ : "LLM step closed before step finish"),+ }+ : {}),+ });+ this.openLlmStep = undefined;+ this.flushDeferredToolCompletions();+ }+private endOpenToolSpans(metadata?: Record<string, unknown>,statusMessage?: string,) {+ // Approval interrupts complete the stream without a tool result. Drop those+ // incomplete spans — the generation already records the tool_call request,+ // and the continuation run emits the real tool observation after approval.+ const forceClose =+ Boolean(statusMessage) ||+ Boolean(metadata?.aborted) ||+ Boolean(metadata?.error);+for (const [toolCallId, tool] of this.toolSpans.entries()) {+ if (tool.output === undefined && !forceClose) {+ this.toolSpans.delete(toolCallId);+ this.toolExecutionTimes.delete(toolCallId);+ continue;+ }+this.createToolObservation(toolCallId, tool, {metadata,statusMessage,});this.toolSpans.delete(toolCallId);}++ // endOpenLlmStep() has already drained completed tools before this method+ // runs; keep the unconditional flush to preserve that lifecycle invariant.+ this.flushDeferredToolCompletions();+ }++ private ensureOpenLlmStep(now: Date) {+ if (this.openLlmStep) {+ return;+ }++ this.openLlmStep = {+ stepNumber: ++this.nextStepNumber,+ startTime: now,+ inputMessageCount: this.agentRunOutputMessages.length,+ };+ }++ private flushDeferredToolCompletions() {+ if (this.deferredToolCompletions.size === 0) {+ return;+ }++ const deferred = [...this.deferredToolCompletions.entries()];+ this.deferredToolCompletions.clear();+ for (const [toolCallId, tool] of deferred) {+ this.createToolObservation(toolCallId, tool);+ }+ }++ private enqueueObservation(type: string, body: ObservationBody) {+ (+ this.langfuse as unknown as {+ enqueue: (type: string, body: ObservationBody) => void;+ }+ ).enqueue(type, body);}// Reasoning that no assistant message followed (e.g. aborted or errored@@ -691,7 +1042,6 @@ export class InAppAgentInstrumentation {private recordAssistantText(delta: string) {this.output += delta;- this.completionStartTime ??= new Date();const thinking = this.takePendingThinking();const lastMessage = this.agentRunOutputMessages.at(-1);@@ -720,8 +1070,6 @@ export class InAppAgentInstrumentation {},output: unknown,) {- this.completionStartTime ??= tool.startTime;-const toolCall: AgentRunToolCall = {id: toolCallId,name: tool.name,@@ -748,18 +1096,121 @@ export class InAppAgentInstrumentation {}private getAgentRunOutput() {- if (this.agentRunOutputMessages.length === 0) {+ const priorMessages =+ this.approvalContinuation && isRecord(this.agentRunInput)+ ? getPriorAgentRunOutputMessages(this.agentRunInput.messages)+ : [];+ const messages = [...priorMessages, ...this.agentRunOutputMessages];++ if (messages.length === 0) {return undefined;}+ const priorToolCalls = priorMessages.flatMap(+ (message) => message.tool_calls ?? [],+ );+ const toolCalls = [...priorToolCalls, ...this.agentRunToolCalls];+return {- messages: this.agentRunOutputMessages,+ messages,...(this.output ? { text: this.output } : {}),- ...(this.agentRunToolCalls.length > 0- ? { tool_calls: this.agentRunToolCalls }- : {}),+ ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}),};}++ private emitApprovalWaitObservation() {+ const continuation = this.approvalContinuation;+ if (!continuation) {+ return;+ }++ const requestedAt =+ continuation.approvalRequestedAt ??+ continuation.approvalDecidedAt ??+ this.agentRunStartTime;+ const decidedAt = continuation.approvalDecidedAt ?? this.agentRunStartTime;+ const endTime =+ decidedAt.getTime() < requestedAt.getTime() ? requestedAt : decidedAt;++ this.enqueueObservation("span-create", {+ id: `${this.runId}-approval-wait`,+ traceId: this.traceId,+ parentObservationId: this.rootObservationId,+ name: `waiting for user approval for tool ${continuation.toolName}`,+ startTime: requestedAt,+ endTime,+ input: {+ tool_call_id: continuation.toolCallId,+ tool_name: continuation.toolName,+ },+ output: `User ${continuation.status} tool ${continuation.toolName}`,+ metadata: continuation.metadata,+ });+ }+}++function getApprovalContinuation(+ input: AgUiRunAgentInput,+): ApprovalContinuation | undefined {+ const forwardedProps = ResumeForwardedPropsSchema.safeParse(+ input.forwardedProps,+ );++ if (!forwardedProps.success) {+ return undefined;+ }++ const {+ approved,+ approvalRequest,+ continuationNumber = 1,+ rootRunId = approvalRequest.runId,+ traceStartedAt,+ approvalRequestedAt,+ approvalDecidedAt,+ } = forwardedProps.data.command.resume;+ const status = approved ? "approved" : "rejected";++ return {+ status,+ continuationNumber,+ rootRunId,+ toolCallId: approvalRequest.toolCallId,+ toolName: approvalRequest.toolName,+ ...(traceStartedAt ? { traceStartedAt: new Date(traceStartedAt) } : {}),+ ...(approvalRequestedAt+ ? { approvalRequestedAt: new Date(approvalRequestedAt) }+ : {}),+ ...(approvalDecidedAt+ ? { approvalDecidedAt: new Date(approvalDecidedAt) }+ : {}),+ metadata: {+ continuation_type: "tool_approval",+ continuation_number: continuationNumber,+ parent_run_id: approvalRequest.runId,+ approval_status: status,+ approval_tool_call_id: approvalRequest.toolCallId,+ approval_tool_name: approvalRequest.toolName,+ ...(approvalRequestedAt+ ? { approval_requested_at: approvalRequestedAt }+ : {}),+ ...(approvalDecidedAt ? { approval_decided_at: approvalDecidedAt } : {}),+ },+ };+}++function getPriorAgentRunOutputMessages(+ messages: unknown,+): AgentRunChatMessage[] {+ if (!Array.isArray(messages)) {+ return [];+ }++ return messages.filter(+ (message): message is AgentRunChatMessage =>+ isRecord(message) &&+ (message.role === "assistant" || message.role === "tool"),+ );}function getAgentRunInput(input: AgUiRunAgentInput): unknown {@@ -970,6 +1421,98 @@ function getFiniteNumber(value: unknown): number | undefined {: undefined;}+function getStepUsageDetails(+ usage: unknown,+): Record<string, number> | undefined {+ if (!isRecord(usage)) {+ return undefined;+ }++ const nestedInputTokens = isRecord(usage.inputTokens)+ ? usage.inputTokens+ : undefined;+ const nestedOutputTokens = isRecord(usage.outputTokens)+ ? usage.outputTokens+ : undefined;+ const inputTokens =+ getFiniteNumber(usage.inputTokens) ??+ getFiniteNumber(nestedInputTokens?.total);+ const outputTokens =+ getFiniteNumber(usage.outputTokens) ??+ getFiniteNumber(nestedOutputTokens?.total);+ if (inputTokens === undefined && outputTokens === undefined) {+ return undefined;+ }++ const cacheReadTokens =+ getFiniteNumber(usage.cachedInputTokens) ??+ getFiniteNumber(nestedInputTokens?.cacheRead) ??+ 0;+ const cacheWriteTokens =+ getFiniteNumber(usage.cacheCreationInputTokens) ??+ getFiniteNumber(nestedInputTokens?.cacheWrite) ??+ 0;+ const uncachedInputTokens = getFiniteNumber(nestedInputTokens?.noCache);++ // Mastra's inputTokens includes cache reads/writes, while Langfuse model+ // prices bill `input` as non-cached input tokens with separate cache keys.+ return {+ input:+ uncachedInputTokens ??+ Math.max(0, (inputTokens ?? 0) - cacheReadTokens - cacheWriteTokens),+ output: outputTokens ?? 0,+ cache_read_input_tokens: cacheReadTokens,+ cache_creation_input_tokens: cacheWriteTokens,+ total:+ getFiniteNumber(usage.totalTokens) ??+ (inputTokens ?? 0) + (outputTokens ?? 0),+ };+}++function getStepFinishReason(+ event: Record<string, unknown> | undefined,+): string | undefined {+ if (!event) {+ return undefined;+ }++ const finishReason = event.finishReason;+ if (typeof finishReason === "string") {+ return finishReason;+ }++ if (isRecord(finishReason)) {+ return (+ getStringValue(finishReason.unified) ?? getStringValue(finishReason.raw)+ );+ }++ return isRecord(event.stepResult)+ ? getStringValue(event.stepResult.reason)+ : undefined;+}++function getModelIdFromStepEvent(+ event: Record<string, unknown> | undefined,+): string | undefined {+ if (!event) {+ return undefined;+ }++ if (isRecord(event.model)) {+ const modelId = getStringValue(event.model.modelId);+ if (modelId) {+ return modelId;+ }+ }++ if (isRecord(event.response)) {+ return getStringValue(event.response.modelId);+ }++ return undefined;+}+function getSerializableToolParameters(tool: Record<string, unknown>): unknown {const parameters =tool.parameters ??diff --git a/packages/shared/src/in-app-agent/server/runLifecycle.ts b/packages/shared/src/in-app-agent/server/runLifecycle.tsindex 702f2abf1e..d0d7c4bcbb 100644--- a/packages/shared/src/in-app-agent/server/runLifecycle.ts+++ b/packages/shared/src/in-app-agent/server/runLifecycle.ts@@ -219,7 +219,13 @@ export async function decideToolApproval(params: {projectId: params.projectId,conversationId: params.conversationId,},- select: { status: true, finishedAt: true, request: true },+ select: {+ status: true,+ finishedAt: true,+ claimedAt: true,+ createdAt: true,+ request: true,+ },});if (@@ -271,6 +277,21 @@ export async function decideToolApproval(params: {const parentRequest = InAppAgentRunRequestSchema.safeParse(parentRun.request,);+ // Preserve tracing lineage across durable approval continuation runs.+ const continuationNumber =+ parentRequest.success && parentRequest.data.kind === "approvalDecision"+ ? (parentRequest.data.continuationNumber ?? 1) + 1+ : 1;+ const rootRunId =+ parentRequest.success && parentRequest.data.kind === "approvalDecision"+ ? (parentRequest.data.rootRunId ?? params.parentRunId)+ : params.parentRunId;+ const traceStartedAt =+ parentRequest.success &&+ parentRequest.data.kind === "approvalDecision" &&+ parentRequest.data.traceStartedAt+ ? parentRequest.data.traceStartedAt+ : (parentRun.claimedAt ?? parentRun.createdAt).toISOString();// Persist the grant in the same transaction as the exactly-once decision CAS.if (params.alwaysAllowToolName) {@@ -325,6 +346,12 @@ export async function decideToolApproval(params: {request: {kind: "approvalDecision",parentRunId: params.parentRunId,+ rootRunId,+ traceStartedAt,+ ...(parentRun.finishedAt+ ? { approvalRequestedAt: parentRun.finishedAt.toISOString() }+ : {}),+ continuationNumber,toolCallId: params.toolCallId,approved: params.approved,context: parentRequest.success ? parentRequest.data.context : [],diff --git a/packages/shared/src/in-app-agent/server/tools.ts b/packages/shared/src/in-app-agent/server/tools.tsindex 86b641d6da..6dfc293622 100644--- a/packages/shared/src/in-app-agent/server/tools.ts+++ b/packages/shared/src/in-app-agent/server/tools.ts@@ -776,7 +776,7 @@ export function withOptionalSilentMcpOutput(params: {) as Record<string, Tool>;}-function getToolCallId(context: unknown) {+export function getToolCallId(context: unknown) {if (typeof context !== "object" ||context === null ||diff --git a/web/src/__tests__/server/in-app-agent-background-run.servertest.ts b/web/src/__tests__/server/in-app-agent-background-run.servertest.tsindex cbb2ba6ce1..cb0a13f128 100644--- a/web/src/__tests__/server/in-app-agent-background-run.servertest.ts+++ b/web/src/__tests__/server/in-app-agent-background-run.servertest.ts@@ -29,6 +29,7 @@ import { inAppAgentRouter } from "@/src/features/in-app-agent/server/router";import { createInnerTRPCContext } from "@/src/server/api/trpc";import type * as SharedServerModule from "@langfuse/shared/src/server";+import type * as PersistenceModule from "@langfuse/shared/in-app-agent/server/persistence";const enqueuedJobs: Array<{ name: string; payload: unknown; jobId?: string }> =[];@@ -38,6 +39,22 @@ const rateLimitMocks = vi.hoisted(() => ({rateLimitRequest: vi.fn(),}));+const persistenceMocks = vi.hoisted(() => ({+ maybeInferAndPersistConversationTitle: vi.fn().mockResolvedValue(undefined),+}));++vi.mock("@langfuse/shared/in-app-agent/server/persistence", async () => {+ const actual = await vi.importActual<typeof PersistenceModule>(+ "@langfuse/shared/in-app-agent/server/persistence",+ );++ return {+ ...actual,+ maybeInferAndPersistConversationTitle:+ persistenceMocks.maybeInferAndPersistConversationTitle,+ };+});+vi.mock("@langfuse/shared/src/server", async () => {const actual = await vi.importActual<typeof SharedServerModule>("@langfuse/shared/src/server",@@ -85,6 +102,7 @@ describe("in-app agent background runs", () => {(env as any).LANGFUSE_AWS_BEDROCK_MODEL = "test-model";enqueuedJobs.length = 0;enqueueShouldFail = false;+ persistenceMocks.maybeInferAndPersistConversationTitle.mockClear();rateLimitMocks.rateLimitRequest.mockResolvedValue({isRateLimited: () => false,res: undefined,@@ -92,6 +110,7 @@ describe("in-app agent background runs", () => {});afterEach(() => {+ vi.useRealTimers();(env as any).NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = originalCloudRegion;(env as any).LANGFUSE_AWS_BEDROCK_MODEL = originalBedrockModel;(sharedEnv as any).LANGFUSE_IN_APP_AGENT_MAX_ACTIVE_RUNS_PER_USER =@@ -289,6 +308,9 @@ describe("in-app agent background runs", () => {}),}),]);+ expect(+ persistenceMocks.maybeInferAndPersistConversationTitle,+ ).toHaveBeenCalledOnce();});it("rejects a second submit while a run is active without duplicating anything", async () => {@@ -576,6 +598,8 @@ describe("in-app agent background runs", () => {});it("decides an approval exactly once and reads the tool args server-side", async () => {+ vi.useFakeTimers({ toFake: ["Date"] });+ vi.setSystemTime(new Date("2026-08-14T10:01:00.000Z"));const { caller, projectId, userId } = await createCaller();const conversation = await createConversation({ projectId, userId });const context = [@@ -585,12 +609,14 @@ describe("in-app agent background runs", () => {},{ description: "user_name", value: "Agent User" },];+ const parkedAt = new Date("2026-08-14T10:00:05.000Z");const parkedRunId = await parkRunForApproval({projectId,conversationId: conversation.id,userId,toolCallId: "tool-call-1",context,+ parkedAt,});const { runId: continuationRunId } = await caller.decideToolApproval({@@ -615,6 +641,10 @@ describe("in-app agent background runs", () => {expect(continuation.request).toMatchObject({kind: "approvalDecision",parentRunId: parkedRunId,+ rootRunId: parkedRunId,+ traceStartedAt: parkedRun.createdAt.toISOString(),+ approvalRequestedAt: parkedAt.toISOString(),+ continuationNumber: 1,toolCallId: "tool-call-1",approved: true,context,@@ -655,6 +685,8 @@ describe("in-app agent background runs", () => {});it("preserves context through an approved then rejected chained continuation", async () => {+ vi.useFakeTimers({ toFake: ["Date"] });+ vi.setSystemTime(new Date("2026-08-14T10:01:00.000Z"));const { caller, projectId, userId } = await createCaller();const conversation = await createConversation({ projectId, userId });const context = [@@ -681,13 +713,26 @@ describe("in-app agent background runs", () => {approved: true,});+ const approvedContinuation = await prisma.inAppAgentRun.findFirstOrThrow({+ where: { id: approvedContinuationId, projectId },+ });+ expect(approvedContinuation.request).toMatchObject({+ kind: "approvalDecision",+ parentRunId: initialRunId,+ rootRunId: initialRunId,+ traceStartedAt: expect.any(String),+ approvalRequestedAt: expect.any(String),+ continuationNumber: 1,+ });++ const secondParkedAt = new Date("2026-08-14T10:03:00.000Z");await prisma.inAppAgentRun.update({where: {id_projectId: { id: approvedContinuationId, projectId },},data: {status: InAppAgentRunStatus.AWAITING_APPROVAL,- finishedAt: new Date(),+ finishedAt: secondParkedAt,},});await prisma.inAppAgentEvent.create({@@ -710,6 +755,7 @@ describe("in-app agent background runs", () => {},},});+ vi.setSystemTime(new Date("2026-08-14T10:04:00.000Z"));const { runId: rejectedContinuationId } = await caller.decideToolApproval({projectId,@@ -725,6 +771,11 @@ describe("in-app agent background runs", () => {expect(rejectedContinuation.request).toMatchObject({kind: "approvalDecision",parentRunId: approvedContinuationId,+ rootRunId: initialRunId,+ traceStartedAt: (approvedContinuation.request as Record<string, unknown>)+ .traceStartedAt,+ approvalRequestedAt: secondParkedAt.toISOString(),+ continuationNumber: 2,toolCallId: "tool-call-2",approved: false,context,diff --git a/web/src/__tests__/server/in-app-agent-stream.servertest.ts b/web/src/__tests__/server/in-app-agent-stream.servertest.tsindex cba3980229..0bf7b6714a 100644--- a/web/src/__tests__/server/in-app-agent-stream.servertest.ts+++ b/web/src/__tests__/server/in-app-agent-stream.servertest.ts@@ -76,12 +76,31 @@ const adapterEvents = vi.hoisted(() => ({}),}));+const bedrockMocks = vi.hoisted(() => ({+ streamParts: [] as unknown[],+ doGenerate: vi.fn(),+ doStream: vi.fn(async () => ({+ stream: new ReadableStream({+ start(controller) {+ for (const part of bedrockMocks.streamParts) {+ controller.enqueue(part);+ }+ controller.close();+ },+ }),+ })),+}));+const instrumentationMocks = vi.hoisted(() => {const instrumentation = {recordEvents: vi.fn(),recordAvailableTools: vi.fn(),recordToolCallApproval: vi.fn(),+ recordToolExecutionStart: vi.fn(),+ recordToolExecutionEnd: vi.fn(),+ recordModelCallFinish: vi.fn(),recordStepFinish: vi.fn(),+ recordStreamChunk: vi.fn(),end: vi.fn(),endWithError: vi.fn(),flush: vi.fn(),@@ -206,7 +225,14 @@ vi.mock("@ag-ui/mastra", () => ({}));vi.mock("ai-sdk-amazon-bedrock-v4", () => ({- createAmazonBedrock: vi.fn(() => vi.fn(() => ({}))),+ createAmazonBedrock: vi.fn(() => (modelId: string) => ({+ specificationVersion: "v3",+ provider: "amazon-bedrock",+ modelId,+ supportedUrls: {},+ doGenerate: bedrockMocks.doGenerate,+ doStream: bedrockMocks.doStream,+ })),}));vi.mock("@aws-sdk/credential-providers", () => ({@@ -467,6 +493,7 @@ describe("IN_APP_AGENT_LANGFUSE_MCP_TOOL_POLICIES", () => {describe("createAgUiStream", () => {beforeEach(() => {vi.clearAllMocks();+ bedrockMocks.streamParts = [];promptMocks.getPrompt.mockResolvedValue({name: "in-app-agent-system-prompt",version: 2,@@ -474,6 +501,95 @@ describe("createAgUiStream", () => {});});+ const initializeBasicTracedAgent = async (runId: string) => {+ const { createAgUiStream } =+ await import("@langfuse/shared/in-app-agent/server/agent");+ const input = {+ threadId: "conversation-1",+ runId,+ messages: [+ {+ id: "user-message-1",+ role: "user" as const,+ content: "hello",+ },+ ],+ tools: [],+ context: [],+ state: null,+ forwardedProps: {},+ };+ adapterEvents.items = [+ {+ type: EventType.RUN_STARTED,+ threadId: input.threadId,+ runId: input.runId,+ },+ {+ type: EventType.RUN_FINISHED,+ threadId: input.threadId,+ runId: input.runId,+ },+ ];++ const stream = await createAgUiStream({+ input,+ signal: new AbortController().signal,+ options: {+ awsBedrock: { modelId: "test-model" },+ langfuseMcp: {+ url: "https://example.com/api/public/mcp",+ publicKey: "pk",+ secretKey: "sk",+ toolPolicy: defaultInAppAgentToolPolicy,+ },+ redirectAction: { projectId: "project-1", isV4Enabled: false },+ langfuseClient: {+ getPrompt: promptMocks.getPrompt,+ } as unknown as Langfuse,+ useLocalPrompt: false,+ langfuseTracing: createTestTracingConfig(),+ },+ });+ await readStream(stream);+ };++ it("forwards model stream parts when finish tracing throws", async () => {+ instrumentationMocks.instrumentation.recordModelCallFinish.mockImplementationOnce(+ () => {+ throw new Error("tracing failed");+ },+ );+ await initializeBasicTracedAgent("run-provider-finish-tracing-error");++ const textPart = { type: "text-delta", id: "text-1", delta: "hello" };+ const finishPart = {+ type: "finish",+ usage: {+ inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 },+ outputTokens: { total: 2, text: 2, reasoning: 0 },+ },+ finishReason: { unified: "stop", raw: "end_turn" },+ };+ bedrockMocks.streamParts = [textPart, finishPart];+ const model = vi.mocked(Agent).mock.calls.at(-1)?.[0]?.model as unknown as {+ doStream: (options: unknown) => Promise<{+ stream: ReadableStream<unknown>;+ }>;+ };++ const modelResult = await model.doStream({});+ const forwardedParts: unknown[] = [];+ for await (const part of modelResult.stream) {+ forwardedParts.push(part);+ }++ expect(forwardedParts).toEqual([textPart, finishPart]);+ expect(+ instrumentationMocks.instrumentation.recordModelCallFinish,+ ).toHaveBeenCalledWith(finishPart);+ });+it("serializes valid events including adapter snapshots and reasoning messages", async () => {const { createAgUiStream } =await import("@langfuse/shared/in-app-agent/server/agent");@@ -829,6 +945,16 @@ describe("createAgUiStream", () => {expect(instrumentationMocks.instrumentation.recordStepFinish,).toHaveBeenCalledWith({ usage: { inputTokens: 10, outputTokens: 5 } });+ const onChunk = (+ agentConfig?.defaultOptions as+ | { onChunk?: (chunk: unknown) => void }+ | undefined+ )?.onChunk;+ expect(onChunk).toEqual(expect.any(Function));+ onChunk?.({ type: "step-start", payload: {} });+ expect(+ instrumentationMocks.instrumentation.recordStreamChunk,+ ).toHaveBeenCalledWith({ type: "step-start", payload: {} });expect(instrumentationMocks.instrumentation.recordEvents.mock.calls.flatMap(([events]) => (events as AgUiEvent[]).map((event) => event.type),@@ -1336,6 +1462,24 @@ describe("createAgUiStream", () => {toolCallId: "tool-call-1",status: "approved",});+ expect(+ instrumentationMocks.instrumentation.recordToolExecutionStart,+ ).toHaveBeenCalledWith("tool-call-1");+ expect(+ instrumentationMocks.instrumentation.recordToolExecutionEnd,+ ).toHaveBeenCalledWith("tool-call-1");+ expect(+ instrumentationMocks.instrumentation.recordToolExecutionStart.mock+ .invocationCallOrder[0],+ ).toBeLessThan(+ adapterEvents.createScoreConfigExecute.mock.invocationCallOrder[0]!,+ );+ expect(+ adapterEvents.createScoreConfigExecute.mock.invocationCallOrder[0],+ ).toBeLessThan(+ instrumentationMocks.instrumentation.recordToolExecutionEnd.mock+ .invocationCallOrder[0]!,+ );expect(adapterEvents.createScoreConfigExecute).toHaveBeenCalledWith({@@ -1354,6 +1498,73 @@ describe("createAgUiStream", () => {);});+ it("preserves successful and failed tool outcomes when timing tracing throws", async () => {+ const tracingError = new Error("tool timing tracing failed");+ instrumentationMocks.instrumentation.recordToolExecutionStart+ .mockImplementationOnce(() => {+ throw tracingError;+ })+ .mockImplementationOnce(() => {+ throw tracingError;+ });+ instrumentationMocks.instrumentation.recordToolExecutionEnd+ .mockImplementationOnce(() => {+ throw tracingError;+ })+ .mockImplementationOnce(() => {+ throw tracingError;+ });+ await initializeBasicTracedAgent("run-tool-timing-tracing-error");++ const tool = getAgentTools(+ vi.mocked(Agent).mock.calls.at(-1)?.[0],+ )?.langfuse_createScoreConfig;+ const execute = tool?.execute;+ expect(execute).toBeTypeOf("function");+ const toolInput = {+ name: "readiness",+ dataType: "NUMERIC",+ numericMinValue: 0,+ numericMaxValue: 1,+ };+ const toolContext = {+ abortSignal: new AbortController().signal,+ agent: {+ toolCallId: "tool-call-1",+ threadId: "conversation-1",+ },+ };++ await expect(execute?.(toolInput, toolContext)).resolves.toEqual({+ id: "score-config-1",+ name: "readiness",+ dataType: "NUMERIC",+ });++ const originalToolError = new Error("original tool failure");+ adapterEvents.createScoreConfigExecute.mockRejectedValueOnce(+ originalToolError,+ );+ await expect(execute?.(toolInput, toolContext)).rejects.toBe(+ originalToolError,+ );++ const startOrder =+ instrumentationMocks.instrumentation.recordToolExecutionStart.mock+ .invocationCallOrder;+ const executeOrder =+ adapterEvents.createScoreConfigExecute.mock.invocationCallOrder;+ const endOrder =+ instrumentationMocks.instrumentation.recordToolExecutionEnd.mock+ .invocationCallOrder;+ expect(startOrder).toHaveLength(2);+ expect(endOrder).toHaveLength(2);+ expect(startOrder[0]).toBeLessThan(executeOrder[0]!);+ expect(executeOrder[0]).toBeLessThan(endOrder[0]!);+ expect(startOrder[1]).toBeLessThan(executeOrder[1]!);+ expect(executeOrder[1]).toBeLessThan(endOrder[1]!);+ });+it("persists raw silent output for approved tools while resuming with redacted content", async () => {const { createAgUiStream } =await import("@langfuse/shared/in-app-agent/server/agent");@@ -1481,11 +1692,30 @@ describe("createAgUiStream", () => {},langfuseClient,useLocalPrompt: false,+ langfuseTracing: createTestTracingConfig(),},});await readStream(stream);expect(onError).not.toHaveBeenCalled();+ expect(+ instrumentationMocks.instrumentation.recordToolExecutionStart,+ ).toHaveBeenCalledWith("tool-call-1");+ expect(+ instrumentationMocks.instrumentation.recordToolExecutionEnd,+ ).toHaveBeenCalledWith("tool-call-1");+ expect(+ instrumentationMocks.instrumentation.recordToolExecutionStart.mock+ .invocationCallOrder[0],+ ).toBeLessThan(+ adapterEvents.createScoreConfigExecute.mock.invocationCallOrder[0]!,+ );+ expect(+ adapterEvents.createScoreConfigExecute.mock.invocationCallOrder[0],+ ).toBeLessThan(+ instrumentationMocks.instrumentation.recordToolExecutionEnd.mock+ .invocationCallOrder[0]!,+ );const resumedMessages =(adapterEvents.inputs[0] as {diff --git a/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts b/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.tsindex 9d768af2e7..e55b563398 100644--- a/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts+++ b/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts@@ -3,6 +3,7 @@ import { EventType } from "@ag-ui/core";import {getInAppAgentInstrumentationObservationId,getInAppAgentInstrumentationTraceId,+ getInAppAgentLlmCallObservationId,} from "@langfuse/shared/in-app-agent";import type { AgUiRunAgentInput } from "@langfuse/shared/in-app-agent";import { InAppAgentInstrumentation } from "@langfuse/shared/in-app-agent/server/instrumentation";@@ -12,14 +13,8 @@ const traceId = getInAppAgentInstrumentationTraceId(runId);const agentRunObservationId = getInAppAgentInstrumentationObservationId(runId);const mocks = vi.hoisted(() => {- const agentGeneration = {- observationId: "run-1",- traceId: "run-1-trace",- update: vi.fn(),- end: vi.fn(),- };const trace = {- generation: vi.fn(() => agentGeneration),+ generation: vi.fn(),update: vi.fn(),};const handler = {@@ -30,7 +25,6 @@ const mocks = vi.hoisted(() => {};return {- agentGeneration,trace,handler,processTracedEvents: vi.fn(async () => undefined),@@ -153,13 +147,7 @@ describe("InAppAgentInstrumentation", () => {input: expectedAgentRunInput,}),);- expect(mocks.trace.generation).toHaveBeenCalledWith(- expect.objectContaining({- id: agentRunObservationId,- name: "agent-turn",- input: expectedAgentRunInput,- }),- );+ expect(mocks.trace.generation).not.toHaveBeenCalled();expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith("tool-create",expect.objectContaining({@@ -179,10 +167,15 @@ describe("InAppAgentInstrumentation", () => {}),);expect(- mocks.handler.langfuse.enqueue.mock.calls[0]?.[1].metadata,+ mocks.handler.langfuse.enqueue.mock.calls.find(+ ([type]) => type === "tool-create",+ )?.[1].metadata,).not.toHaveProperty("toolCallApproval");- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "agent-create",expect.objectContaining({+ id: agentRunObservationId,+ traceId,name: "agent-turn",input: expectedAgentRunInput,output: {@@ -202,9 +195,13 @@ describe("InAppAgentInstrumentation", () => {text: "hi there",tool_calls: [toolCall],},- completionStartTime: expect.any(Date),}),);+ const agentCreateBody = mocks.handler.langfuse.enqueue.mock.calls.find(+ ([type]) => type === "agent-create",+ )?.[1];+ expect(agentCreateBody).not.toHaveProperty("usageDetails");+ expect(agentCreateBody).not.toHaveProperty("model");expect(mocks.handler.langfuse.enqueue).not.toHaveBeenCalledWith("span-create",expect.anything(),@@ -233,15 +230,59 @@ describe("InAppAgentInstrumentation", () => {);});- it.each(["approved", "rejected"] as const)(- "records manual tool approval metadata as %s",- (status) => {+ it("records approved manual tool approval metadata", () => {+ const instrumentation = createInstrumentation();++ instrumentation.recordToolCallApproval({+ toolCallId: "tool-1",+ status: "approved",+ });+ instrumentation.recordEvents([+ {+ type: EventType.TOOL_CALL_START,+ toolCallId: "tool-1",+ toolCallName: "createScoreConfig",+ parentMessageId: "tool-1-approval-tool-call",+ },+ {+ type: EventType.TOOL_CALL_ARGS,+ toolCallId: "tool-1",+ delta: '{"name":"readiness"}',+ },+ {+ type: EventType.TOOL_CALL_END,+ toolCallId: "tool-1",+ },+ {+ type: EventType.TOOL_CALL_RESULT,+ toolCallId: "tool-1",+ content: "ok",+ },+ ]);++ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "tool-create",+ expect.objectContaining({+ metadata: expect.objectContaining({+ toolCallId: "tool-1",+ parentMessageId: "tool-1-approval-tool-call",+ toolCallApproval: "approved",+ }),+ }),+ );+ });++ it("records rejected approvals without fabricating tool execution duration", () => {+ vi.useFakeTimers();+ try {const instrumentation = createInstrumentation();+ const rejectedAt = new Date("2026-01-01T00:00:01.000Z");instrumentation.recordToolCallApproval({toolCallId: "tool-1",- status,+ status: "rejected",});+ vi.setSystemTime(rejectedAt);instrumentation.recordEvents([{type: EventType.TOOL_CALL_START,@@ -258,27 +299,34 @@ describe("InAppAgentInstrumentation", () => {type: EventType.TOOL_CALL_END,toolCallId: "tool-1",},+ ]);+ vi.setSystemTime(new Date("2026-01-01T00:00:05.000Z"));+ instrumentation.recordEvents([{type: EventType.TOOL_CALL_RESULT,toolCallId: "tool-1",- content: "ok",+ content: "rejected",},]);expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith("tool-create",expect.objectContaining({+ startTime: rejectedAt,+ endTime: rejectedAt,metadata: expect.objectContaining({toolCallId: "tool-1",parentMessageId: "tool-1-approval-tool-call",- toolCallApproval: status,+ toolCallApproval: "rejected",}),}),);- },- );+ } finally {+ vi.useRealTimers();+ }+ });- it("records run failures on the agent generation", () => {+ it("records run failures on the root agent observation", () => {const instrumentation = createInstrumentation();const toolCall = {id: "tool-1",@@ -313,8 +361,10 @@ describe("InAppAgentInstrumentation", () => {}),}),);- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "agent-create",expect.objectContaining({+ id: agentRunObservationId,name: "agent-turn",input: expectedAgentRunInput,output: {@@ -327,7 +377,6 @@ describe("InAppAgentInstrumentation", () => {],tool_calls: [toolCall],},- completionStartTime: expect.any(Date),level: "ERROR",statusMessage: "agent failed",metadata: expect.objectContaining({ error: "agent failed" }),@@ -335,7 +384,7 @@ describe("InAppAgentInstrumentation", () => {);});- it("records available tools & skills on the agent generation input", () => {+ it("records available tools & skills on the agent observation input", () => {const instrumentation = createInstrumentation();instrumentation.recordAvailableTools({@@ -368,7 +417,8 @@ describe("InAppAgentInstrumentation", () => {]);instrumentation.end();- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "agent-create",expect.objectContaining({name: "agent-turn",input: {@@ -443,7 +493,9 @@ describe("InAppAgentInstrumentation", () => {},]);- const toolCreateBody = mocks.handler.langfuse.enqueue.mock.calls[0]?.[1];+ const toolCreateBody = mocks.handler.langfuse.enqueue.mock.calls.find(+ ([type]) => type === "tool-create",+ )?.[1];expect(toolCreateBody).toEqual(expect.objectContaining({@@ -596,7 +648,9 @@ describe("InAppAgentInstrumentation", () => {},]);- const toolCreateBody = mocks.handler.langfuse.enqueue.mock.calls[0]?.[1];+ const toolCreateBody = mocks.handler.langfuse.enqueue.mock.calls.find(+ ([type]) => type === "tool-create",+ )?.[1];expect(toolCreateBody).toEqual(expect.objectContaining({output: expectedOutput,@@ -607,7 +661,7 @@ describe("InAppAgentInstrumentation", () => {},);- it("sets static prompt metadata on the trace and agent generation", () => {+ it("sets static prompt metadata on the trace and root agent observation", () => {const instrumentation = createInstrumentation(undefined, {name: "in-app-agent-system-prompt",version: 3,@@ -641,26 +695,14 @@ describe("InAppAgentInstrumentation", () => {},}),);- expect(mocks.trace.generation).toHaveBeenCalledWith({- id: agentRunObservationId,- name: "agent-turn",- input: expectedAgentRunInput,- metadata: {- langfuse_project_id: "project-1",- langfuse_user_email: "[email protected]",- langfuse_user_project_role: "ADMIN",- langfuse_user_is_admin: true,- prompt_name: "in-app-agent-system-prompt",- prompt_version: 3,- },- promptName: "in-app-agent-system-prompt",- promptVersion: 3,- });+ expect(mocks.trace.generation).not.toHaveBeenCalled();instrumentation.end();- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "agent-create",expect.objectContaining({+ id: agentRunObservationId,name: "agent-turn",input: expectedAgentRunInput,promptName: "in-app-agent-system-prompt",@@ -682,7 +724,8 @@ describe("InAppAgentInstrumentation", () => {},]);- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "agent-create",expect.objectContaining({name: "agent-turn",input: expectedAgentRunInput,@@ -690,7 +733,6 @@ describe("InAppAgentInstrumentation", () => {messages: [{ role: "assistant", content: "second turn output" }],text: "second turn output",},- completionStartTime: expect.any(Date),}),);expect(mocks.trace.update).toHaveBeenCalledWith(@@ -704,7 +746,7 @@ describe("InAppAgentInstrumentation", () => {);});- it("records AG-UI context in the agent generation input", () => {+ it("records AG-UI context in the agent turn input", () => {createInstrumentation({context: [{@@ -723,20 +765,7 @@ describe("InAppAgentInstrumentation", () => {],});- expect(mocks.trace.generation).toHaveBeenCalledWith(- expect.objectContaining({- name: "agent-turn",- input: {- messages: [{ role: "user", content: "hello" }],- context: {- current_url:- "https://cloud.langfuse.com/project/project-1/traces?filter=value",- browser_languages: ["en-US", "en", "de"],- current_trace: { id: "trace-1" },- },- },- }),- );+ expect(mocks.trace.generation).not.toHaveBeenCalled();expect(mocks.handler.langfuse.trace).toHaveBeenCalledWith(expect.objectContaining({input: {@@ -752,7 +781,7 @@ describe("InAppAgentInstrumentation", () => {);});- it("records only the current turn messages in the agent generation input", () => {+ it("records only the current turn messages in the agent turn input", () => {createInstrumentation({messages: [{@@ -795,7 +824,7 @@ describe("InAppAgentInstrumentation", () => {],});- expect(mocks.trace.generation).toHaveBeenCalledWith(+ expect(mocks.handler.langfuse.trace).toHaveBeenCalledWith(expect.objectContaining({name: "agent-turn",input: {@@ -805,6 +834,211 @@ describe("InAppAgentInstrumentation", () => {);});+ it.each([+ { approved: true, status: "approved" as const },+ { approved: false, status: "rejected" as const },+ ])(+ "appends a $status approval continuation to the parent trace with a wait span",+ ({ approved, status }) => {+ const traceStartedAt = "2026-08-14T10:00:00.000Z";+ const approvalRequestedAt = "2026-08-14T10:00:05.000Z";+ const approvalDecidedAt = "2026-08-14T10:02:05.000Z";+ const instrumentation = createInstrumentation({+ forwardedProps: {+ command: {+ resume: {+ approved,+ continuationNumber: 2,+ rootRunId: "root-run-1",+ traceStartedAt,+ approvalRequestedAt,+ approvalDecidedAt,+ approvalRequest: {+ type: "tool_approval_request",+ toolCallId: "tool-1",+ toolName: "langfuse_createTextPrompt",+ args: { name: "FOO" },+ runId: "parent-run-1",+ },+ },+ },+ },+ });++ instrumentation.recordToolCallApproval({+ toolCallId: "tool-1",+ status,+ });+ instrumentation.recordEvents([+ {+ type: EventType.TOOL_CALL_START,+ toolCallId: "tool-1",+ toolCallName: "langfuse_createTextPrompt",+ },+ {+ type: EventType.TOOL_CALL_ARGS,+ toolCallId: "tool-1",+ delta: '{"name":"FOO"}',+ },+ {+ type: EventType.TOOL_CALL_END,+ toolCallId: "tool-1",+ },+ {+ type: EventType.TOOL_CALL_RESULT,+ toolCallId: "tool-1",+ content: '{"name":"FOO","version":1}',+ },+ ]);+ instrumentation.recordStreamChunk({ type: "step-start", payload: {} });+ instrumentation.recordStepFinish({+ usage: { inputTokens: 100, outputTokens: 10, totalTokens: 110 },+ });+ instrumentation.end({});++ const parentTraceId = getInAppAgentInstrumentationTraceId("root-run-1");+ const parentObservationId =+ getInAppAgentInstrumentationObservationId("root-run-1");+ const approvalMetadata = {+ continuation_type: "tool_approval",+ continuation_number: 2,+ parent_run_id: "parent-run-1",+ approval_status: status,+ approval_tool_call_id: "tool-1",+ approval_tool_name: "langfuse_createTextPrompt",+ approval_requested_at: approvalRequestedAt,+ approval_decided_at: approvalDecidedAt,+ };++ expect(mocks.handler.langfuse.trace).toHaveBeenCalledWith(+ expect.objectContaining({+ id: parentTraceId,+ name: "agent-turn",+ sessionId: "conversation-1",+ timestamp: new Date(traceStartedAt),+ input: expectedAgentRunInput,+ tags: ["in-app-agent"],+ metadata: expect.objectContaining({+ approval_continuation_count: 2,+ }),+ }),+ );+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "agent-create",+ expect.objectContaining({+ id: parentObservationId,+ traceId: parentTraceId,+ name: "agent-turn",+ startTime: new Date(traceStartedAt),+ input: expectedAgentRunInput,+ metadata: expect.objectContaining({+ approval_continuation_count: 2,+ }),+ }),+ );+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "span-create",+ expect.objectContaining({+ id: `${runId}-approval-wait`,+ traceId: parentTraceId,+ parentObservationId,+ name: "waiting for user approval for tool langfuse_createTextPrompt",+ startTime: new Date(approvalRequestedAt),+ endTime: new Date(approvalDecidedAt),+ output: `User ${status} tool langfuse_createTextPrompt`,+ metadata: approvalMetadata,+ }),+ );+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "generation-create",+ expect.objectContaining({+ traceId: parentTraceId,+ parentObservationId,+ input: expect.objectContaining({+ messages: [+ { role: "user", content: "hello" },+ expect.objectContaining({+ role: "assistant",+ tool_calls: [+ expect.objectContaining({+ id: "tool-1",+ name: "langfuse_createTextPrompt",+ }),+ ],+ }),+ {+ role: "tool",+ tool_call_id: "tool-1",+ content: { name: "FOO", version: 1 },+ },+ ],+ }),+ }),+ );+ expect(mocks.trace.update).toHaveBeenCalledWith(+ expect.objectContaining({+ input: expectedAgentRunInput,+ metadata: expect.objectContaining({+ approval_continuation_count: 2,+ }),+ }),+ );+ },+ );++ it("preserves earlier assistant output when a continuation updates the root trace", () => {+ const instrumentation = createInstrumentation({+ messages: [+ { id: "message-1", role: "user", content: "hello" },+ {+ id: "assistant-before-approval",+ role: "assistant",+ content: "I need your approval.",+ },+ ],+ forwardedProps: {+ command: {+ resume: {+ approved: true,+ rootRunId: "root-run-1",+ traceStartedAt: "2026-08-14T10:00:00.000Z",+ approvalRequestedAt: "2026-08-14T10:00:05.000Z",+ approvalDecidedAt: "2026-08-14T10:02:05.000Z",+ approvalRequest: {+ type: "tool_approval_request",+ toolCallId: "tool-1",+ toolName: "langfuse_createTextPrompt",+ runId: "parent-run-1",+ },+ },+ },+ },+ });++ instrumentation.recordEvents([+ {+ type: EventType.TEXT_MESSAGE_CONTENT,+ delta: "The tool completed.",+ },+ ]);+ instrumentation.end({});++ const finalRootObservation = mocks.handler.langfuse.enqueue.mock.calls+ .filter(([type]) => type === "agent-create")+ .at(-1)?.[1];+ expect(finalRootObservation).toEqual(+ expect.objectContaining({+ input: expectedAgentRunInput,+ output: expect.objectContaining({+ messages: [+ { role: "assistant", content: "I need your approval." },+ { role: "assistant", content: "The tool completed." },+ ],+ }),+ }),+ );+ });+it("compacts text message chunks before recording output", () => {const instrumentation = createInstrumentation();@@ -825,7 +1059,8 @@ describe("InAppAgentInstrumentation", () => {},]);- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "agent-create",expect.objectContaining({name: "agent-turn",input: expectedAgentRunInput,@@ -833,7 +1068,6 @@ describe("InAppAgentInstrumentation", () => {messages: [{ role: "assistant", content: "chunk output" }],text: "chunk output",},- completionStartTime: expect.any(Date),}),);});@@ -908,7 +1142,8 @@ describe("InAppAgentInstrumentation", () => {},]);- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "agent-create",expect.objectContaining({name: "agent-turn",input: expectedAgentRunInput,@@ -930,13 +1165,12 @@ describe("InAppAgentInstrumentation", () => {text: "Done",tool_calls: [toolCall],},- completionStartTime: expect.any(Date),}),);- const updateArg = mocks.agentGeneration.update.mock.calls.at(-1)?.[0] as {- metadata?: Record<string, unknown>;- };- expect(updateArg.metadata).not.toHaveProperty("reasoning");+ const agentCreateBody = mocks.handler.langfuse.enqueue.mock.calls.find(+ ([type]) => type === "agent-create",+ )?.[1] as { metadata?: Record<string, unknown> };+ expect(agentCreateBody.metadata).not.toHaveProperty("reasoning");});it("records reasoning without a following assistant message as its own thinking message", () => {@@ -953,7 +1187,8 @@ describe("InAppAgentInstrumentation", () => {},]);- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "agent-create",expect.objectContaining({name: "agent-turn",output: {@@ -987,9 +1222,53 @@ describe("InAppAgentInstrumentation", () => {);});- it("aggregates step usage and model onto the agent generation", () => {+ it("emits one generation per LLM step with usage on children only", () => {const instrumentation = createInstrumentation();+ const modelName = "eu.anthropic.claude-sonnet-4-5-20250929-v1:0";++ // Root agent observation must exist before children so tools/generations+ // do not orphan onto the trace.+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "agent-create",+ expect.objectContaining({+ id: agentRunObservationId,+ name: "agent-turn",+ }),+ );+ instrumentation.recordStreamChunk({+ type: "step-start",+ payload: {},+ });+ instrumentation.recordStreamChunk({+ type: "tool-call",+ payload: { toolCallId: "tool-1", toolName: "listObservations" },+ });+ instrumentation.recordEvents([+ {+ type: EventType.TOOL_CALL_START,+ toolCallId: "tool-1",+ toolCallName: "listObservations",+ },+ {+ type: EventType.TOOL_CALL_ARGS,+ toolCallId: "tool-1",+ delta: '{"limit":10}',+ },+ {+ type: EventType.TOOL_CALL_END,+ toolCallId: "tool-1",+ },+ {+ type: EventType.TOOL_CALL_RESULT,+ toolCallId: "tool-1",+ content: '{"ok":true}',+ },+ ]);+ instrumentation.recordStreamChunk({+ type: "tool-result",+ payload: { toolCallId: "tool-1", toolName: "listObservations" },+ });instrumentation.recordStepFinish({usage: {inputTokens: 1100,@@ -998,7 +1277,42 @@ describe("InAppAgentInstrumentation", () => {cachedInputTokens: 800,cacheCreationInputTokens: 100,},+ finishReason: "tool-calls",+ text: "",+ toolCalls: [+ {+ toolCallId: "tool-1",+ toolName: "listObservations",+ args: { limit: 10 },+ },+ ],+ response: {+ messages: [+ {+ role: "assistant",+ content: "",+ tool_calls: [{ id: "tool-1" }],+ },+ {+ role: "tool",+ tool_call_id: "tool-1",+ content: { ok: true },+ },+ ],+ },+ });++ instrumentation.recordStreamChunk({ type: "step-start", payload: {} });+ instrumentation.recordStreamChunk({+ type: "text-delta",+ payload: { text: "Done" },});+ instrumentation.recordEvents([+ {+ type: EventType.TEXT_MESSAGE_CONTENT,+ delta: "Done",+ },+ ]);instrumentation.recordStepFinish({usage: {inputTokens: 1200,@@ -1006,37 +1320,332 @@ describe("InAppAgentInstrumentation", () => {totalTokens: 1230,cachedInputTokens: 1000,},+ finishReason: "stop",+ text: "Done",});instrumentation.end({});- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(+ const generationCreates = mocks.handler.langfuse.enqueue.mock.calls.filter(+ ([type]) => type === "generation-create",+ );+ expect(generationCreates).toHaveLength(2);++ expect(generationCreates[0]?.[1]).toEqual(expect.objectContaining({- model: "eu.anthropic.claude-sonnet-4-5-20250929-v1:0",+ id: getInAppAgentLlmCallObservationId(runId, 1),+ parentObservationId: agentRunObservationId,+ model: modelName,usageDetails: {- // Mastra's inputTokens includes cache reads/writes; the priced- // `input` key must only contain non-cached input tokens.- input: 400,- output: 80,- cache_read_input_tokens: 1800,+ input: 200,+ output: 50,+ cache_read_input_tokens: 800,cache_creation_input_tokens: 100,- total: 2380,+ total: 1150,+ },+ metadata: expect.objectContaining({+ finish_reason: "tool-calls",+ }),+ }),+ );+ expect(generationCreates[0]?.[1].input.messages).toHaveLength(1);+ expect(generationCreates[0]?.[1].output).toEqual({+ text: "",+ tool_calls: [+ {+ toolCallId: "tool-1",+ toolName: "listObservations",+ args: { limit: 10 },+ },+ ],+ });++ expect(generationCreates[1]?.[1]).toEqual(+ expect.objectContaining({+ id: getInAppAgentLlmCallObservationId(runId, 2),+ parentObservationId: agentRunObservationId,+ model: modelName,+ usageDetails: {+ input: 200,+ output: 30,+ cache_read_input_tokens: 1000,+ cache_creation_input_tokens: 0,+ total: 1230,},+ metadata: expect.objectContaining({+ finish_reason: "stop",+ }),}),);+ expect(generationCreates[1]?.[1].input.messages.length).toBeGreaterThan(+ generationCreates[0]?.[1].input.messages.length,+ );++ const agentCreates = mocks.handler.langfuse.enqueue.mock.calls.filter(+ ([type]) => type === "agent-create",+ );+ const finalAgentCreate = agentCreates.at(-1)?.[1];+ expect(finalAgentCreate).toEqual(+ expect.objectContaining({+ id: agentRunObservationId,+ name: "agent-turn",+ }),+ );+ expect(finalAgentCreate).not.toHaveProperty("usageDetails");+ expect(finalAgentCreate).not.toHaveProperty("model");});- it("records aggregated usage when the run fails", () => {+ it("emits tools after the generation and clamps start to generation end", () => {const instrumentation = createInstrumentation();+ const stepStart = new Date("2026-01-01T00:00:00.000Z");+ const toolCallChunkA = new Date("2026-01-01T00:00:01.000Z");+ const toolCallChunkB = new Date("2026-01-01T00:00:02.000Z");+ const agUiToolAStart = new Date("2026-01-01T00:00:01.500Z");+ const agUiToolBStart = new Date("2026-01-01T00:00:02.500Z");+ const toolAStart = new Date("2026-01-01T00:00:02.900Z");+ const toolBStart = new Date("2026-01-01T00:00:03.200Z");+ const toolAEnd = new Date("2026-01-01T00:00:03.000Z");+ const toolBEnd = new Date("2026-01-01T00:00:04.000Z");+ const providerFinish = new Date("2026-01-01T00:00:02.800Z");+ const stepFinish = new Date("2026-01-01T00:00:04.200Z");++ vi.setSystemTime(stepStart);+ instrumentation.recordStreamChunk({ type: "step-start", payload: {} });++ // tool-call chunks are request-time during the LLM stream — must not become+ // tool observation startTimes (that sorted tools above the generation).+ vi.setSystemTime(toolCallChunkA);+ instrumentation.recordStreamChunk({+ type: "tool-call",+ payload: { toolCallId: "tool-a", toolName: "listObservations" },+ });+ vi.setSystemTime(toolCallChunkB);+ instrumentation.recordStreamChunk({+ type: "tool-call",+ payload: { toolCallId: "tool-b", toolName: "getTrace" },+ });+ vi.setSystemTime(agUiToolAStart);+ instrumentation.recordEvents([+ {+ type: EventType.TOOL_CALL_START,+ toolCallId: "tool-a",+ toolCallName: "listObservations",+ },+ {+ type: EventType.TOOL_CALL_ARGS,+ toolCallId: "tool-a",+ delta: "{}",+ },+ {+ type: EventType.TOOL_CALL_END,+ toolCallId: "tool-a",+ },+ ]);+ vi.setSystemTime(agUiToolBStart);+ instrumentation.recordEvents([+ {+ type: EventType.TOOL_CALL_START,+ toolCallId: "tool-b",+ toolCallName: "getTrace",+ },+ {+ type: EventType.TOOL_CALL_ARGS,+ toolCallId: "tool-b",+ delta: "{}",+ },+ {+ type: EventType.TOOL_CALL_END,+ toolCallId: "tool-b",+ },+ ]);++ // The model stream finishes before tool execution, but Mastra only invokes+ // onStepFinish after the tools have resolved.+ vi.setSystemTime(providerFinish);+ instrumentation.recordModelCallFinish({+ usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },+ finishReason: { unified: "tool-calls", raw: "tool_use" },+ });++ vi.setSystemTime(toolAStart);+ instrumentation.recordToolExecutionStart("tool-a");+ vi.setSystemTime(toolAEnd);+ instrumentation.recordToolExecutionEnd("tool-a");+ instrumentation.recordStreamChunk({+ type: "tool-result",+ payload: { toolCallId: "tool-a", toolName: "listObservations" },+ });+ instrumentation.recordEvents([+ {+ type: EventType.TOOL_CALL_RESULT,+ toolCallId: "tool-a",+ content: '{"a":1}',+ },+ ]);++ vi.setSystemTime(toolBStart);+ instrumentation.recordToolExecutionStart("tool-b");+ vi.setSystemTime(toolBEnd);+ instrumentation.recordToolExecutionEnd("tool-b");+ instrumentation.recordStreamChunk({+ type: "tool-result",+ payload: { toolCallId: "tool-b", toolName: "getTrace" },+ });+ instrumentation.recordEvents([+ {+ type: EventType.TOOL_CALL_RESULT,+ toolCallId: "tool-b",+ content: '{"b":2}',+ },+ ]);++ // Tools completed during the open LLM step — deferred until step-finish.+ expect(+ mocks.handler.langfuse.enqueue.mock.calls.filter(+ ([type]) => type === "tool-create",+ ),+ ).toHaveLength(0);++ vi.setSystemTime(stepFinish);instrumentation.recordStepFinish({- usage: { inputTokens: 100, outputTokens: 10, totalTokens: 110 },+ usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },+ finishReason: "tool-calls",});- instrumentation.endWithError(new Error("agent failed"));- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(+ const enqueueOrder = mocks.handler.langfuse.enqueue.mock.calls+ .filter(+ ([type]) => type === "generation-create" || type === "tool-create",+ )+ .map(([type, body]) => ({ type, id: body.id }));+ expect(enqueueOrder.map((entry) => entry.type)).toEqual([+ "generation-create",+ "tool-create",+ "tool-create",+ ]);++ const generationCreate = mocks.handler.langfuse.enqueue.mock.calls.find(+ ([type]) => type === "generation-create",+ )?.[1];+ expect(generationCreate).toEqual(+ expect.objectContaining({ endTime: providerFinish }),+ );++ const toolCreates = mocks.handler.langfuse.enqueue.mock.calls.filter(+ ([type]) => type === "tool-create",+ );+ expect(toolCreates).toHaveLength(2);+ // Actual execute() timing wins over the delayed AG-UI event timestamps.+ expect(toolCreates[0]?.[1]).toEqual(+ expect.objectContaining({+ id: "tool-a",+ parentObservationId: agentRunObservationId,+ startTime: toolAStart,+ endTime: toolAEnd,+ }),+ );+ expect(toolCreates[1]?.[1]).toEqual(+ expect.objectContaining({+ id: "tool-b",+ parentObservationId: agentRunObservationId,+ startTime: toolBStart,+ endTime: toolBEnd,+ }),+ );++ vi.useRealTimers();+ });++ it("opens an LLM step on first model delta when step-start is missing", () => {+ const instrumentation = createInstrumentation();+ const firstDelta = new Date("2026-01-01T00:00:10.000Z");+ const stepFinish = new Date("2026-01-01T00:00:12.000Z");++ vi.setSystemTime(firstDelta);+ instrumentation.recordStreamChunk({+ type: "reasoning-delta",+ payload: { text: "thinking" },+ });+ vi.setSystemTime(stepFinish);+ instrumentation.recordStepFinish({+ usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },+ finishReason: "stop",+ });++ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "generation-create",+ expect.objectContaining({+ startTime: firstDelta,+ endTime: stepFinish,+ }),+ );++ vi.useRealTimers();+ });++ it("does not force-close incomplete tools on a normal end (approval interrupt)", () => {+ const instrumentation = createInstrumentation();++ instrumentation.recordEvents([+ {+ type: EventType.TOOL_CALL_START,+ toolCallId: "tool-awaiting-approval",+ toolCallName: "createAnnotationQueueItem",+ },+ {+ type: EventType.TOOL_CALL_ARGS,+ toolCallId: "tool-awaiting-approval",+ delta: '{"queueId":"q1"}',+ },+ {+ type: EventType.TOOL_CALL_END,+ toolCallId: "tool-awaiting-approval",+ },+ ]);+ instrumentation.end({});++ expect(mocks.handler.langfuse.enqueue).not.toHaveBeenCalledWith(+ "tool-create",+ expect.objectContaining({ id: "tool-awaiting-approval" }),+ );+ });++ it("closes an open LLM step as ERROR when the run is aborted", () => {+ const instrumentation = createInstrumentation();++ instrumentation.recordStreamChunk({ type: "step-start", payload: {} });+ instrumentation.end({ aborted: true });++ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "generation-create",expect.objectContaining({+ id: getInAppAgentLlmCallObservationId(runId, 1),+ parentObservationId: agentRunObservationId,level: "ERROR",- model: "eu.anthropic.claude-sonnet-4-5-20250929-v1:0",+ statusMessage: "aborted",+ }),+ );+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "agent-create",+ expect.objectContaining({+ id: agentRunObservationId,+ metadata: expect.objectContaining({ aborted: true }),+ }),+ );+ });++ it("emits a generation when step finish arrives without a prior step-start", () => {+ const instrumentation = createInstrumentation();++ instrumentation.recordStepFinish({+ usage: { inputTokens: 100, outputTokens: 10, totalTokens: 110 },+ finishReason: "stop",+ });+ instrumentation.end({});++ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "generation-create",+ expect.objectContaining({+ id: getInAppAgentLlmCallObservationId(runId, 1),usageDetails: {input: 100,output: 10,@@ -1044,21 +1653,99 @@ describe("InAppAgentInstrumentation", () => {cache_creation_input_tokens: 0,total: 110,},+ model: "eu.anthropic.claude-sonnet-4-5-20250929-v1:0",}),);});- it("omits usage and model when no step reported usage", () => {+ it("uses provider finish usage when approval ends before step finish", () => {const instrumentation = createInstrumentation();- instrumentation.recordStepFinish(undefined);- instrumentation.recordStepFinish({ usage: { foo: 1 } });+ instrumentation.recordStreamChunk({ type: "step-start", payload: {} });+ instrumentation.recordModelCallFinish({+ usage: {+ inputTokens: {+ total: 1_100,+ noCache: 200,+ cacheRead: 800,+ cacheWrite: 100,+ },+ outputTokens: { total: 50, text: 50, reasoning: 0 },+ },+ finishReason: { unified: "tool-calls", raw: "tool_use" },+ });instrumentation.end({});- const agentGenerationBody = mocks.agentGeneration.update.mock.calls[0]?.[0];+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "generation-create",+ expect.objectContaining({+ id: getInAppAgentLlmCallObservationId(runId, 1),+ model: "eu.anthropic.claude-sonnet-4-5-20250929-v1:0",+ usageDetails: {+ input: 200,+ output: 50,+ cache_read_input_tokens: 800,+ cache_creation_input_tokens: 100,+ total: 1_150,+ },+ metadata: expect.objectContaining({+ finish_reason: "tool-calls",+ }),+ }),+ );+ });++ it("keeps a provider-finished generation successful when the run later fails", () => {+ const instrumentation = createInstrumentation();- expect(agentGenerationBody).not.toHaveProperty("usageDetails");- expect(agentGenerationBody).not.toHaveProperty("model");+ instrumentation.recordStreamChunk({ type: "step-start", payload: {} });+ instrumentation.recordEvents([+ {+ type: EventType.TEXT_MESSAGE_CONTENT,+ delta: "Calling the tool",+ },+ ]);+ instrumentation.recordModelCallFinish({+ usage: {+ inputTokens: {+ total: 1_100,+ noCache: 200,+ cacheRead: 800,+ cacheWrite: 100,+ },+ outputTokens: { total: 50, text: 50, reasoning: 0 },+ },+ finishReason: { unified: "tool-calls", raw: "tool_use" },+ });+ instrumentation.endWithError(new Error("tool execution failed"));++ const generation = mocks.handler.langfuse.enqueue.mock.calls.find(+ ([type]) => type === "generation-create",+ )?.[1];+ expect(generation).toEqual(+ expect.objectContaining({+ id: getInAppAgentLlmCallObservationId(runId, 1),+ model: "eu.anthropic.claude-sonnet-4-5-20250929-v1:0",+ usageDetails: expect.objectContaining({+ input: 200,+ output: 50,+ total: 1_150,+ }),+ output: {+ messages: [{ role: "assistant", content: "Calling the tool" }],+ },+ }),+ );+ expect(generation).not.toHaveProperty("level");+ expect(generation).not.toHaveProperty("statusMessage");+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "agent-create",+ expect.objectContaining({+ id: agentRunObservationId,+ level: "ERROR",+ statusMessage: "tool execution failed",+ }),+ );});it("ignores step finish events after instrumentation ended", () => {@@ -1069,8 +1756,9 @@ describe("InAppAgentInstrumentation", () => {usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },});- expect(mocks.agentGeneration.update).toHaveBeenCalledWith(- expect.not.objectContaining({ usageDetails: expect.anything() }),+ expect(mocks.handler.langfuse.enqueue).not.toHaveBeenCalledWith(+ "generation-create",+ expect.anything(),);});});diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.test.tsindex ea29795b1d..70f4265527 100644--- a/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.test.ts@@ -1,11 +1,12 @@import { randomUUID } from "crypto";-import { describe, expect, it, vi } from "vitest";+import { beforeEach, describe, expect, it, vi } from "vitest";import { createOrgProjectAndApiKey } from "@langfuse/shared/src/server";import { prisma } from "@langfuse/shared/src/db";import {createInAppAgentConversationId,createInAppAgentRunId,+ ResumeForwardedPropsSchema,} from "@langfuse/shared/in-app-agent";vi.hoisted(() => {@@ -54,6 +55,7 @@ const scenarioRef = vi.hoisted(() => ({current: undefined as AgentScenario | undefined,failApiKeyDelete: false,apiKeyDeleteCalls: 0,+ titleInferenceCalls: 0,}));vi.mock("@langfuse/shared/in-app-agent/server", async (importOriginal) => {@@ -65,6 +67,9 @@ vi.mock("@langfuse/shared/in-app-agent/server", async (importOriginal) => {return {...actual,IN_APP_AGENT_HEARTBEAT_INTERVAL_MS: 50,+ maybeInferAndPersistConversationTitle: async () => {+ scenarioRef.titleInferenceCalls += 1;+ },createAgUiStream: async (params: {input: never;signal: AbortSignal;@@ -203,6 +208,10 @@ const getInAppAgentApiKeys = (projectId: string) =>async function seedApprovedContinuation(opts?: {context?: Array<{ description: string; value: string }>;alwaysAllowedTools?: string[];+ continuationNumber?: number;+ rootRunId?: string;+ traceStartedAt?: string;+ approvalRequestedAt?: string;}) {const seeded = await seedBackgroundRun({alwaysAllowedTools: opts?.alwaysAllowedTools,@@ -234,8 +243,18 @@ async function seedApprovedContinuation(opts?: {request: {kind: "approvalDecision",parentRunId: parentRun.id,+ ...(opts?.rootRunId ? { rootRunId: opts.rootRunId } : {}),+ ...(opts?.traceStartedAt+ ? { traceStartedAt: opts.traceStartedAt }+ : {}),+ ...(opts?.approvalRequestedAt+ ? { approvalRequestedAt: opts.approvalRequestedAt }+ : {}),toolCallId: "tc-1",approved: true,+ ...(opts?.continuationNumber+ ? { continuationNumber: opts.continuationNumber }+ : {}),...(opts?.context ? { context: opts.context } : {}),},},@@ -245,7 +264,24 @@ async function seedApprovedContinuation(opts?: {}describe("executeInAppAgentRun", () => {+ beforeEach(() => {+ scenarioRef.titleInferenceCalls = 0;+ });++ it("does not regenerate the conversation title after executing a user-message run", async () => {+ const { projectId, run } = await seedBackgroundRun();++ scenarioRef.current = completingScenario;++ await executeInAppAgentRun({ projectId, runId: run.id });++ expect(scenarioRef.titleInferenceCalls).toBe(0);+ });+it("passes persisted continuation context to the agent input", async () => {+ const rootRunId = "root-run-1";+ const traceStartedAt = "2026-08-14T10:00:00.000Z";+ const approvalRequestedAt = "2026-08-14T10:00:05.000Z";const context = [{description: "current_url",@@ -253,10 +289,25 @@ describe("executeInAppAgentRun", () => {},{ description: "browser_languages", value: '["de-DE"]' },];- const { projectId, run } = await seedApprovedContinuation({ context });+ const { projectId, run } = await seedApprovedContinuation({+ context,+ continuationNumber: 3,+ rootRunId,+ traceStartedAt,+ approvalRequestedAt,+ });scenarioRef.current = async ({ input, options }) => {expect(input.context).toEqual(context);+ const resume = ResumeForwardedPropsSchema.parse(input.forwardedProps)+ .command.resume;+ expect(resume).toMatchObject({+ continuationNumber: 3,+ rootRunId,+ traceStartedAt,+ approvalRequestedAt,+ approvalDecidedAt: run.createdAt.toISOString(),+ });await options.onComplete();await options.onFinish();};@@ -322,7 +373,6 @@ describe("executeInAppAgentRun", () => {expect(finished.claimedAt).not.toBeNull();expect(finished.heartbeatAt).not.toBeNull();expect(finished.errorCode).toBeNull();-// Key was minted and linked during the run, deleted and unlinked after.expect(keysDuringRun).toBe(1);expect(await getInAppAgentApiKeys(projectId)).toHaveLength(0);diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.tsindex 3819d71a4a..b03fc7ebf0 100644--- a/worker/src/features/in-app-agent/executeInAppAgentRun.ts+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.ts@@ -39,7 +39,6 @@ import {createInAppAgentToolPolicy,getInAppAgentMcpAllowedToolNames,getInAppAgentRegistryToolName,- maybeInferAndPersistConversationTitle,parseInAppAgentInterruptEvent,shouldFlushPersistedEvent,toPersistableAgentEvent,@@ -257,6 +256,17 @@ export async function executeInAppAgentRun(params: {command: {resume: {approved: request.approved,+ continuationNumber: request.continuationNumber ?? 1,+ ...(request.rootRunId+ ? { rootRunId: request.rootRunId }+ : {}),+ ...(request.traceStartedAt+ ? { traceStartedAt: request.traceStartedAt }+ : {}),+ ...(request.approvalRequestedAt+ ? { approvalRequestedAt: request.approvalRequestedAt }+ : {}),+ approvalDecidedAt: run.createdAt.toISOString(),approvalRequest,},},@@ -437,19 +447,6 @@ export async function executeInAppAgentRun(params: {? { status: InAppAgentRunStatus.AWAITING_APPROVAL }: { status: InAppAgentRunStatus.SUCCEEDED },);- await maybeInferAndPersistConversationTitle({- prisma,- projectId,- conversationId: conversation.id,- userId: run.triggeredByUserId!,- aiTelemetryEnabled: project.organization.aiTelemetryEnabled,- }).catch((error) =>- logger.error("Failed to infer in-app agent conversation title", {- error,- projectId,- runId,- }),- );},onAbort: async () => {const reason = abortController.signal.reason as AbortReason;