🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(in-app-agent): trace model calls at stream boundary (#16231)
Lysander Kiesel63331b121598
diff --git a/packages/shared/src/in-app-agent/server/agent.ts b/packages/shared/src/in-app-agent/server/agent.tsindex e5485b6dca..b447c48f98 100644--- a/packages/shared/src/in-app-agent/server/agent.ts+++ b/packages/shared/src/in-app-agent/server/agent.ts@@ -262,24 +262,17 @@ export async function createAgUiStream(params: {recordInstrumentation("recordAvailableSkills", (instrumentation) =>instrumentation.recordAvailableSkills?.(LANGFUSE_IN_APP_AGENT_SKILLS),);- const onStepFinish = instrumentation- ? (event: unknown) => {- recordInstrumentation("recordStepFinish", (instrumentation) =>- instrumentation.recordStepFinish?.(event),+ const onModelCallStart = instrumentation+ ? (options: unknown) => {+ recordInstrumentation("recordModelCallStart", (instrumentation) =>+ instrumentation.recordModelCallStart?.(options),);}: undefined;- const onChunk = instrumentation- ? (chunk: unknown) => {- recordInstrumentation("recordStreamChunk", (instrumentation) =>- instrumentation.recordStreamChunk?.(chunk),- );- }- : undefined;- const onModelCallFinish = instrumentation- ? (event: unknown) => {- recordInstrumentation("recordModelCallFinish", (instrumentation) =>- instrumentation.recordModelCallFinish?.(event),+ const onModelStreamPart = instrumentation+ ? (part: unknown) => {+ recordInstrumentation("recordModelStreamPart", (instrumentation) =>+ instrumentation.recordModelStreamPart?.(part),);}: undefined;@@ -554,9 +547,8 @@ export async function createAgUiStream(params: {recordInstrumentation("recordAvailableTools", (instrumentation) =>instrumentation.recordAvailableTools?.(tools),),- onStepFinish,- onChunk,- onModelCallFinish,+ onModelCallStart,+ onModelStreamPart,onToolExecutionStart,onToolExecutionEnd,})@@ -626,9 +618,8 @@ export async function createAgUiStream(params: {},awsProfile,instructions,- onStepFinish,- onChunk,- onModelCallFinish,+ onModelCallStart,+ onModelStreamPart,onToolExecutionStart,onToolExecutionEnd,});@@ -857,11 +848,14 @@ type ExecutableInAppAgentTool = {type BedrockLanguageModel = ReturnType<ReturnType<typeof createAmazonBedrock>>;-function withModelCallFinish(+function withModelTracing(model: BedrockLanguageModel,- onFinish?: (event: unknown) => void,+ callbacks: {+ onStart?: (options: unknown) => void;+ onStreamPart?: (part: unknown) => void;+ },): BedrockLanguageModel {- if (!onFinish) {+ if (!callbacks.onStart && !callbacks.onStreamPart) {return model;}@@ -872,6 +866,7 @@ function withModelCallFinish(supportedUrls: model.supportedUrls,doGenerate: (options) => model.doGenerate(options),doStream: async (options) => {+ callbacks.onStart?.(options);const result = await model.doStream(options);return {@@ -879,9 +874,7 @@ function withModelCallFinish(stream: result.stream.pipeThrough(new TransformStream({transform(part, controller) {- if (part.type === "finish") {- onFinish(part);- }+ callbacks.onStreamPart?.(part);controller.enqueue(part);},}),@@ -942,9 +935,8 @@ async function createMastraAdapter(params: {awsProfile?: string;instructions: string;onToolsAvailable?: (tools: Record<string, unknown>) => void;- onStepFinish?: (event: unknown) => void;- onChunk?: (chunk: unknown) => void;- onModelCallFinish?: (event: unknown) => void;+ onModelCallStart?: (options: unknown) => void;+ onModelStreamPart?: (part: unknown) => void;onToolExecutionStart?: (toolCallId: string) => void;onToolExecutionEnd?: (toolCallId: string) => void;}) {@@ -1078,11 +1070,14 @@ async function createMastraAdapter(params: {// instruction channel so the model receives the same higher-priority// guidance on resumed runs.let developerGuidance: string | undefined;- const model = withModelCallFinish(+ const model = withModelTracing(bedrock(params.options.awsBedrock.modelId as Parameters<typeof bedrock>[0],),- params.onModelCallFinish,+ {+ onStart: params.onModelCallStart,+ onStreamPart: params.onModelStreamPart,+ },);const agent = new Agent({id: "langfuse-in-app-assistant",@@ -1099,11 +1094,6 @@ async function createMastraAdapter(params: {...(params.options.sandboxWorkspaceWasReset? { system: SANDBOX_WORKSPACE_RESET_INSTRUCTION }: {}),- // Fires once per LLM call with that call's token usage; the AG-UI- // event stream itself never carries usage.- ...(params.onStepFinish ? { onStepFinish: params.onStepFinish } : {}),- // Tool execution windows (and optional step-start timing bookmarks).- ...(params.onChunk ? { onChunk: params.onChunk } : {}),...(reasoningProviderOptions? { providerOptions: reasoningProviderOptions }: {}),diff --git a/packages/shared/src/in-app-agent/server/instrumentation.ts b/packages/shared/src/in-app-agent/server/instrumentation.tsindex 1b163f206a..0224f977d7 100644--- a/packages/shared/src/in-app-agent/server/instrumentation.ts+++ b/packages/shared/src/in-app-agent/server/instrumentation.ts@@ -1,4 +1,5 @@import { EventType } from "@ag-ui/core";+import { Buffer } from "node:buffer";import { getInternalTracingHandler, logger } from "../../server";import {@@ -116,16 +117,20 @@ type AgentRunToolSpan = {failureMessage?: string;parentMessageId?: string;};-// Lightweight bookmark for the in-flight Mastra step. Generations are emitted-// from onStepFinish (which carries usage); this only supplies start timing and-// the input-message boundary for context-growth visibility.-type OpenLlmStep = {- stepNumber: number;+type ModelToolCall = {+ toolCallId: string;+ toolName: string;+ args: unknown;+};+type OpenModelCall = {+ callNumber: number;startTime: Date;- inputMessageCount: number;+ input: unknown;completionStartTime?: Date;- providerFinish?: Record<string, unknown>;- providerFinishTime?: Date;+ textDeltas: string[];+ toolCalls: ModelToolCall[];+ modelId?: string;+ streamErrorMessage?: string;};type ToolExecutionTimes = {startTime?: Date;@@ -198,17 +203,8 @@ export class InAppAgentInstrumentation {private pendingThinking: AgentRunThinkingPart[] = [];private ended = false;private readonly model?: string;- private nextStepNumber = 0;- private openLlmStep?: OpenLlmStep;- // Tools that finished while an LLM step was still open. Emit after the- // generation so the tree sorts model → tools (tool-call chunks arrive during- // the LLM stream, before onStepFinish).- private readonly deferredToolCompletions = new Map<- string,- AgentRunToolSpan- >();- // Clamp tool execution windows to start at/after the last generation end.- private lastLlmGenerationEndTime?: Date;+ private nextModelCallNumber = 0;+ private openModelCall?: OpenModelCall;constructor(params: {input: AgUiRunAgentInput;@@ -348,120 +344,73 @@ export class InAppAgentInstrumentation {this.toolExecutionTimes.set(toolCallId, times);}- // Bedrock's V3 stream emits exact usage before Mastra executes tools. Keep- // it on the open step as a fallback because approval suspension can end the- // run without Mastra ever invoking onStepFinish.- recordModelCallFinish(event: unknown) {- if (!isRecord(event)) {- return;- }-+ recordModelCallStart(options: unknown) {if (this.ended) {return;}const now = new Date();- this.ensureOpenLlmStep(now);- if (this.openLlmStep) {- this.openLlmStep.providerFinish = event;- this.openLlmStep.providerFinishTime = now;- }+ this.openModelCall = {+ callNumber: this.openModelCall?.callNumber ?? ++this.nextModelCallNumber,+ startTime: this.openModelCall?.startTime ?? now,+ input: getModelCallInput(options),+ textDeltas: [],+ toolCalls: [],+ };}- // Mastra stream chunks: step-start (or first model delta) bookmarks the LLM- // window; tool-result ends tool execution. Do not treat tool-call chunks as- // execution start — they fire while the model is still streaming.- // Usage and output still come from recordStepFinish.- recordStreamChunk(chunk: unknown) {- if (this.ended || !isRecord(chunk)) {+ recordModelStreamPart(part: unknown) {+ if (this.ended || !this.openModelCall || !isRecord(part)) {return;}- const type = chunk.type;+ const type = part.type;if (typeof type !== "string") {return;}- const payload = isRecord(chunk.payload) ? chunk.payload : {};const now = new Date();- if (type === "step-start") {- // A prior step should have been closed by onStepFinish. If not, emit it- // as an error so we never silently drop a generation.- if (this.openLlmStep) {- const providerFinish = this.openLlmStep.providerFinish;- this.emitLlmGeneration(- this.openLlmStep,- undefined,- providerFinish- ? undefined- : {- level: "ERROR",- statusMessage: "LLM step closed by a subsequent step-start",- },- );- this.flushDeferredToolCompletions();+ if (type === "text-delta" || type === "reasoning-delta") {+ this.openModelCall.completionStartTime ??= now;+ if (type === "text-delta" && typeof part.delta === "string") {+ this.openModelCall.textDeltas.push(part.delta);}-- this.openLlmStep = {- stepNumber: ++this.nextStepNumber,- startTime: now,- inputMessageCount: this.agentRunOutputMessages.length,- };return;}- // tool-call means "model requested this tool", not "tool started".- // Ensure an LLM step is open so missing step-start still gets a real window.if (type === "tool-call") {- this.ensureOpenLlmStep(now);+ const toolCallId = getStringValue(part.toolCallId);+ const toolName = getStringValue(part.toolName);+ if (toolCallId && toolName) {+ this.openModelCall.toolCalls.push({+ toolCallId,+ toolName,+ args:+ typeof part.input === "string"+ ? parseJsonOrString(part.input)+ : part.input,+ });+ }return;}- if (type === "tool-result" || type === "tool-error") {- const toolCallId = getStringValue(payload.toolCallId);- if (!toolCallId) {- return;- }-- const times = this.toolExecutionTimes.get(toolCallId) ?? {};- times.endTime ??= now;- this.toolExecutionTimes.set(toolCallId, times);+ if (type === "error") {+ this.openModelCall.streamErrorMessage = getErrorMessage(part.error);return;}- if (- type === "text-delta" ||- type === "reasoning-delta" ||- type === "reasoning"- ) {- this.ensureOpenLlmStep(now);- if (this.openLlmStep) {- this.openLlmStep.completionStartTime ??= now;- }+ if (type === "response-metadata") {+ this.openModelCall.modelId = getStringValue(part.modelId);+ return;}- }- // Receives Mastra's per-LLM-call onStepFinish event and emits one generation- // with that call's usage/model. The AG-UI event stream itself never carries usage.- recordStepFinish(event: unknown) {- if (this.ended) {+ if (type !== "finish") {return;}- const step =- this.openLlmStep ??- ({- stepNumber: ++this.nextStepNumber,- // Prefer a non-zero window when step-start never arrived: backdate to- // the last generation end (or run start) so tools do not sort above us.- startTime: this.lastLlmGenerationEndTime ?? this.agentRunStartTime,- inputMessageCount: this.agentRunOutputMessages.length,- } satisfies OpenLlmStep);-- this.emitLlmGeneration(step, event);- this.openLlmStep = undefined;- this.flushDeferredToolCompletions();+ this.emitModelGeneration(this.openModelCall, part, now);+ this.openModelCall = undefined;}recordAvailableSkills(skills: unknown[]) {@@ -487,7 +436,7 @@ export class InAppAgentInstrumentation {}const message = error instanceof Error ? error.message : String(error);- this.endOpenLlmStep({ error: message }, message);+ this.endOpenModelCall(message);this.endOpenToolSpans({ error: message }, message);this.flushTrailingThinking();this.emitRootAgentObservation({@@ -512,9 +461,9 @@ export class InAppAgentInstrumentation {}if (params?.aborted) {- this.endOpenLlmStep({ aborted: true }, "aborted");+ this.endOpenModelCall("aborted");} else {- this.endOpenLlmStep();+ this.endOpenModelCall("Model call ended before provider finish");}this.endOpenToolSpans(params?.aborted ? { aborted: true } : undefined);this.flushTrailingThinking();@@ -701,14 +650,6 @@ export class InAppAgentInstrumentation {return;}- // Tool-call args/results often arrive while the LLM step is still open.- // Hold the observation until step-finish so the tree sorts generation → tools.- if (this.openLlmStep) {- this.deferredToolCompletions.set(toolCallId, tool);- this.toolSpans.delete(toolCallId);- return;- }-this.createToolObservation(toolCallId, tool);this.toolSpans.delete(toolCallId);}@@ -735,22 +676,16 @@ export class InAppAgentInstrumentation {toolCallApproval === "rejected"? rawStartTime: (executionTimes?.endTime ?? new Date());- // Keep causal order: tool execution cannot start before the generation that- // requested it finished (or the last known generation end).const startTime =- this.lastLlmGenerationEndTime &&- rawStartTime.getTime() < this.lastLlmGenerationEndTime.getTime()- ? this.lastLlmGenerationEndTime- : rawStartTime;+ rawStartTime.getTime() > endTime.getTime() ? endTime : rawStartTime;const body: ObservationBody = {id: toolCallId,traceId: this.traceId,parentObservationId: this.rootObservationId,name: tool.name,- startTime: startTime.getTime() > endTime.getTime() ? endTime : startTime,+ startTime,endTime,- completionStartTime:- startTime.getTime() > endTime.getTime() ? endTime : startTime,+ completionStartTime: startTime,input,output,...(failureMessage@@ -799,40 +734,42 @@ export class InAppAgentInstrumentation {});}- private emitLlmGeneration(- step: OpenLlmStep,- event: unknown,+ private emitModelGeneration(+ call: OpenModelCall,+ finish: Record<string, unknown> | undefined,+ endTime: Date,options?: {level?: "ERROR";statusMessage?: string;},) {- const eventRecord = isRecord(event) ? event : undefined;- const providerFinish = step.providerFinish;- const usageDetails =- getStepUsageDetails(eventRecord?.usage) ??- getStepUsageDetails(providerFinish?.usage);- const finishReason =- getStepFinishReason(eventRecord) ?? getStepFinishReason(providerFinish);- const model =- getModelIdFromStepEvent(eventRecord) ?? this.model ?? undefined;- // Mastra can delay onStepFinish until after requested tools execute. The- // raw provider finish is the actual model boundary and keeps the following- // tool's execute() window intact instead of clamping it to zero duration.- const endTime = step.providerFinishTime ?? new Date();- this.lastLlmGenerationEndTime = endTime;- const id = getInAppAgentLlmCallObservationId(this.runId, step.stepNumber);+ const usageDetails = getModelUsageDetails(finish?.usage);+ const finishReason = getModelFinishReason(finish);+ const model = call.modelId ?? this.model;+ const id = getInAppAgentLlmCallObservationId(this.runId, call.callNumber);+ const text = call.textDeltas.join("");+ const output =+ text || call.toolCalls.length > 0+ ? {+ ...(text ? { text } : {}),+ ...(call.toolCalls.length > 0+ ? { tool_calls: call.toolCalls }+ : {}),+ }+ : undefined;this.enqueueObservation("generation-create", {id,traceId: this.traceId,parentObservationId: this.rootObservationId,name: IN_APP_AGENT_LLM_CALL_NAME,- startTime: step.startTime,+ startTime: call.startTime,endTime,- completionStartTime: step.completionStartTime ?? step.startTime,- input: this.getLlmStepInput(step.inputMessageCount),- output: this.getLlmStepOutput(step, eventRecord),+ ...(call.completionStartTime+ ? { completionStartTime: call.completionStartTime }+ : {}),+ input: call.input,+ output,// Model only travels with usage so Langfuse does not invent tokenizer-// estimated costs for generations that never reported tokens....(usageDetails@@ -852,19 +789,6 @@ export class InAppAgentInstrumentation {});}- private getLlmStepInput(inputMessageCount: number) {- const base = isRecord(this.agentRunInput) ? this.agentRunInput : {};- const baseMessages = Array.isArray(base.messages) ? base.messages : [];-- return {- ...base,- messages: [- ...baseMessages,- ...this.agentRunOutputMessages.slice(0, inputMessageCount),- ],- };- }-private getTraceInput() {if (!this.approvalContinuation || !isRecord(this.agentRunInput)) {return this.agentRunInput;@@ -890,52 +814,17 @@ export class InAppAgentInstrumentation {};}- private getLlmStepOutput(- step: OpenLlmStep,- event: Record<string, unknown> | undefined,- ) {- const text = typeof event?.text === "string" ? event.text : undefined;- const toolCalls = Array.isArray(event?.toolCalls)- ? event.toolCalls- : undefined;- if (text !== undefined || toolCalls?.length) {- return {- ...(text !== undefined ? { text } : {}),- ...(toolCalls?.length ? { tool_calls: toolCalls } : {}),- };- }-- const response = isRecord(event?.response) ? event.response : undefined;- if (Array.isArray(response?.messages) && response.messages.length > 0) {- return { messages: response.messages };- }-- const messages = this.agentRunOutputMessages.slice(step.inputMessageCount);- return messages.length > 0 ? { messages } : undefined;- }-- private endOpenLlmStep(- metadata?: Record<string, unknown>,- statusMessage?: string,- ) {- if (!this.openLlmStep) {+ private endOpenModelCall(statusMessage: string) {+ if (!this.openModelCall) {return;}- this.emitLlmGeneration(this.openLlmStep, undefined, {- ...(!this.openLlmStep.providerFinish && (statusMessage || metadata)- ? {- level: "ERROR" as const,- statusMessage:- statusMessage ??- (typeof metadata?.error === "string"- ? metadata.error- : "LLM step closed before step finish"),- }- : {}),+ const message = this.openModelCall.streamErrorMessage ?? statusMessage;+ this.emitModelGeneration(this.openModelCall, undefined, new Date(), {+ level: "ERROR",+ statusMessage: message,});- this.openLlmStep = undefined;- this.flushDeferredToolCompletions();+ this.openModelCall = undefined;}private endOpenToolSpans(@@ -963,34 +852,6 @@ export class InAppAgentInstrumentation {});this.toolSpans.delete(toolCallId);}-- // endOpenLlmStep() has already drained completed tools before this method- // runs; keep the unconditional flush to preserve that lifecycle invariant.- this.flushDeferredToolCompletions();- }-- private ensureOpenLlmStep(now: Date) {- if (this.openLlmStep) {- return;- }-- this.openLlmStep = {- stepNumber: ++this.nextStepNumber,- startTime: now,- inputMessageCount: this.agentRunOutputMessages.length,- };- }-- private flushDeferredToolCompletions() {- if (this.deferredToolCompletions.size === 0) {- return;- }-- const deferred = [...this.deferredToolCompletions.entries()];- this.deferredToolCompletions.clear();- for (const [toolCallId, tool] of deferred) {- this.createToolObservation(toolCallId, tool);- }}private enqueueObservation(type: string, body: ObservationBody) {@@ -1421,7 +1282,31 @@ function getFiniteNumber(value: unknown): number | undefined {: undefined;}-function getStepUsageDetails(+function getModelCallInput(options: unknown): unknown {+ if (!isRecord(options)) {+ return undefined;+ }++ const input = {+ messages: options.prompt,+ maxOutputTokens: options.maxOutputTokens,+ temperature: options.temperature,+ stopSequences: options.stopSequences,+ topP: options.topP,+ topK: options.topK,+ presencePenalty: options.presencePenalty,+ frequencyPenalty: options.frequencyPenalty,+ responseFormat: options.responseFormat,+ seed: options.seed,+ tools: options.tools,+ toolChoice: options.toolChoice,+ providerOptions: options.providerOptions,+ };++ return toSerializableJson(input);+}++function getModelUsageDetails(usage: unknown,): Record<string, number> | undefined {if (!isRecord(usage)) {@@ -1454,7 +1339,7 @@ function getStepUsageDetails(0;const uncachedInputTokens = getFiniteNumber(nestedInputTokens?.noCache);- // Mastra's inputTokens includes cache reads/writes, while Langfuse model+ // Provider inputTokens includes cache reads/writes, while Langfuse model// prices bill `input` as non-cached input tokens with separate cache keys.return {input:@@ -1469,7 +1354,7 @@ function getStepUsageDetails(};}-function getStepFinishReason(+function getModelFinishReason(event: Record<string, unknown> | undefined,): string | undefined {if (!event) {@@ -1487,29 +1372,6 @@ function getStepFinishReason();}- return isRecord(event.stepResult)- ? getStringValue(event.stepResult.reason)- : undefined;-}--function getModelIdFromStepEvent(- event: Record<string, unknown> | undefined,-): string | undefined {- if (!event) {- return undefined;- }-- if (isRecord(event.model)) {- const modelId = getStringValue(event.model.modelId);- if (modelId) {- return modelId;- }- }-- if (isRecord(event.response)) {- return getStringValue(event.response.modelId);- }-return undefined;}@@ -1528,6 +1390,14 @@ function toSerializableJson(seen = new WeakSet<object>(),depth = 0,): unknown {+ if (value instanceof URL) {+ return value.toString();+ }++ if (value instanceof Uint8Array) {+ return Buffer.from(value).toString("base64");+ }+if (value === null || typeof value === "string") {return value;}@@ -1569,7 +1439,10 @@ function toSerializableJson(seen.add(value);const entries = Object.entries(value).flatMap(([key, item]) => {- const serializableItem = toSerializableJson(item, seen, depth + 1);+ const serializableItem =+ key === "data" && item instanceof Uint8Array+ ? `data:${getStringValue(value.mediaType) ?? "application/octet-stream"};base64,${Buffer.from(item).toString("base64")}`+ : toSerializableJson(item, seen, depth + 1);return serializableItem === undefined? []@@ -1592,3 +1465,11 @@ function parseJsonOrUndefined(value: string): unknown {function getStringValue(value: unknown): string | undefined {return typeof value === "string" && value.trim() ? value : undefined;}++function getErrorMessage(error: unknown): string {+ if (error instanceof Error) {+ return error.message;+ }++ return typeof error === "string" ? error : "Model stream failed";+}diff --git a/web/src/__tests__/server/in-app-agent-stream.servertest.ts b/web/src/__tests__/server/in-app-agent-stream.servertest.tsindex 0bf7b6714a..696a9547e4 100644--- a/web/src/__tests__/server/in-app-agent-stream.servertest.ts+++ b/web/src/__tests__/server/in-app-agent-stream.servertest.ts@@ -98,9 +98,8 @@ const instrumentationMocks = vi.hoisted(() => {recordToolCallApproval: vi.fn(),recordToolExecutionStart: vi.fn(),recordToolExecutionEnd: vi.fn(),- recordModelCallFinish: vi.fn(),- recordStepFinish: vi.fn(),- recordStreamChunk: vi.fn(),+ recordModelCallStart: vi.fn(),+ recordModelStreamPart: vi.fn(),end: vi.fn(),endWithError: vi.fn(),flush: vi.fn(),@@ -555,11 +554,11 @@ describe("createAgUiStream", () => {};it("forwards model stream parts when finish tracing throws", async () => {- instrumentationMocks.instrumentation.recordModelCallFinish.mockImplementationOnce(- () => {+ instrumentationMocks.instrumentation.recordModelStreamPart+ .mockImplementationOnce(() => undefined)+ .mockImplementationOnce(() => {throw new Error("tracing failed");- },- );+ });await initializeBasicTracedAgent("run-provider-finish-tracing-error");const textPart = { type: "text-delta", id: "text-1", delta: "hello" };@@ -578,7 +577,8 @@ describe("createAgUiStream", () => {}>;};- const modelResult = await model.doStream({});+ const options = { prompt: [] };+ const modelResult = await model.doStream(options);const forwardedParts: unknown[] = [];for await (const part of modelResult.stream) {forwardedParts.push(part);@@ -586,8 +586,14 @@ describe("createAgUiStream", () => {expect(forwardedParts).toEqual([textPart, finishPart]);expect(- instrumentationMocks.instrumentation.recordModelCallFinish,- ).toHaveBeenCalledWith(finishPart);+ instrumentationMocks.instrumentation.recordModelCallStart,+ ).toHaveBeenCalledWith(options);+ expect(+ instrumentationMocks.instrumentation.recordModelStreamPart,+ ).toHaveBeenNthCalledWith(1, textPart);+ expect(+ instrumentationMocks.instrumentation.recordModelStreamPart,+ ).toHaveBeenNthCalledWith(2, finishPart);});it("serializes valid events including adapter snapshots and reasoning messages", async () => {@@ -935,26 +941,6 @@ describe("createAgUiStream", () => {}),model: "eu.anthropic.claude-opus-4-8",});- const onStepFinish = (- agentConfig?.defaultOptions as- | { onStepFinish?: (event: unknown) => void }- | undefined- )?.onStepFinish;- expect(onStepFinish).toEqual(expect.any(Function));- onStepFinish?.({ usage: { inputTokens: 10, outputTokens: 5 } });- expect(- instrumentationMocks.instrumentation.recordStepFinish,- ).toHaveBeenCalledWith({ usage: { inputTokens: 10, outputTokens: 5 } });- const onChunk = (- agentConfig?.defaultOptions as- | { onChunk?: (chunk: unknown) => void }- | undefined- )?.onChunk;- expect(onChunk).toEqual(expect.any(Function));- onChunk?.({ type: "step-start", payload: {} });- expect(- instrumentationMocks.instrumentation.recordStreamChunk,- ).toHaveBeenCalledWith({ type: "step-start", payload: {} });expect(instrumentationMocks.instrumentation.recordEvents.mock.calls.flatMap(([events]) => (events as AgUiEvent[]).map((event) => event.type),diff --git a/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts b/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.tsindex e55b563398..3774dd33ca 100644--- a/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts+++ b/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts@@ -890,8 +890,29 @@ describe("InAppAgentInstrumentation", () => {content: '{"name":"FOO","version":1}',},]);- instrumentation.recordStreamChunk({ type: "step-start", payload: {} });- instrumentation.recordStepFinish({+ instrumentation.recordModelCallStart({+ prompt: [+ { role: "user", content: "hello" },+ {+ role: "assistant",+ tool_calls: [+ {+ id: "tool-1",+ name: "langfuse_createTextPrompt",+ arguments: '{"name":"FOO"}',+ type: "function",+ },+ ],+ },+ {+ role: "tool",+ tool_call_id: "tool-1",+ content: { name: "FOO", version: 1 },+ },+ ],+ });+ instrumentation.recordModelStreamPart({+ type: "finish",usage: { inputTokens: 100, outputTokens: 10, totalTokens: 110 },});instrumentation.end({});@@ -1222,7 +1243,7 @@ describe("InAppAgentInstrumentation", () => {);});- it("emits one generation per LLM step with usage on children only", () => {+ it("emits one generation per model call with usage on children only", () => {const instrumentation = createInstrumentation();const modelName = "eu.anthropic.claude-sonnet-4-5-20250929-v1:0";@@ -1236,13 +1257,25 @@ describe("InAppAgentInstrumentation", () => {}),);- instrumentation.recordStreamChunk({- type: "step-start",- payload: {},+ instrumentation.recordModelCallStart({+ prompt: [{ role: "user", content: [{ type: "text", text: "hello" }] }],});- instrumentation.recordStreamChunk({+ instrumentation.recordModelStreamPart({type: "tool-call",- payload: { toolCallId: "tool-1", toolName: "listObservations" },+ toolCallId: "tool-1",+ toolName: "listObservations",+ input: '{"limit":10}',+ });+ instrumentation.recordModelStreamPart({+ type: "finish",+ usage: {+ inputTokens: 1100,+ outputTokens: 50,+ totalTokens: 1150,+ cachedInputTokens: 800,+ cacheCreationInputTokens: 100,+ },+ finishReason: "tool-calls",});instrumentation.recordEvents([{@@ -1265,47 +1298,16 @@ describe("InAppAgentInstrumentation", () => {content: '{"ok":true}',},]);- instrumentation.recordStreamChunk({- type: "tool-result",- payload: { toolCallId: "tool-1", toolName: "listObservations" },- });- instrumentation.recordStepFinish({- usage: {- inputTokens: 1100,- outputTokens: 50,- totalTokens: 1150,- cachedInputTokens: 800,- cacheCreationInputTokens: 100,- },- finishReason: "tool-calls",- text: "",- toolCalls: [- {- toolCallId: "tool-1",- toolName: "listObservations",- args: { limit: 10 },- },++ instrumentation.recordModelCallStart({+ prompt: [+ { role: "user", content: [{ type: "text", text: "hello" }] },+ { role: "tool", content: [{ type: "tool-result" }] },],- response: {- messages: [- {- role: "assistant",- content: "",- tool_calls: [{ id: "tool-1" }],- },- {- role: "tool",- tool_call_id: "tool-1",- content: { ok: true },- },- ],- },});-- instrumentation.recordStreamChunk({ type: "step-start", payload: {} });- instrumentation.recordStreamChunk({+ instrumentation.recordModelStreamPart({type: "text-delta",- payload: { text: "Done" },+ delta: "Done",});instrumentation.recordEvents([{@@ -1313,7 +1315,8 @@ describe("InAppAgentInstrumentation", () => {delta: "Done",},]);- instrumentation.recordStepFinish({+ instrumentation.recordModelStreamPart({+ type: "finish",usage: {inputTokens: 1200,outputTokens: 30,@@ -1321,7 +1324,6 @@ describe("InAppAgentInstrumentation", () => {cachedInputTokens: 1000,},finishReason: "stop",- text: "Done",});instrumentation.end({});@@ -1349,7 +1351,6 @@ describe("InAppAgentInstrumentation", () => {);expect(generationCreates[0]?.[1].input.messages).toHaveLength(1);expect(generationCreates[0]?.[1].output).toEqual({- text: "",tool_calls: [{toolCallId: "tool-1",@@ -1394,11 +1395,9 @@ describe("InAppAgentInstrumentation", () => {expect(finalAgentCreate).not.toHaveProperty("model");});- it("emits tools after the generation and clamps start to generation end", () => {+ it("emits tools after the generation using actual execution timing", () => {const instrumentation = createInstrumentation();- const stepStart = new Date("2026-01-01T00:00:00.000Z");- const toolCallChunkA = new Date("2026-01-01T00:00:01.000Z");- const toolCallChunkB = new Date("2026-01-01T00:00:02.000Z");+ const modelCallStart = new Date("2026-01-01T00:00:00.000Z");const agUiToolAStart = new Date("2026-01-01T00:00:01.500Z");const agUiToolBStart = new Date("2026-01-01T00:00:02.500Z");const toolAStart = new Date("2026-01-01T00:00:02.900Z");@@ -1406,22 +1405,21 @@ describe("InAppAgentInstrumentation", () => {const toolAEnd = new Date("2026-01-01T00:00:03.000Z");const toolBEnd = new Date("2026-01-01T00:00:04.000Z");const providerFinish = new Date("2026-01-01T00:00:02.800Z");- const stepFinish = new Date("2026-01-01T00:00:04.200Z");- vi.setSystemTime(stepStart);- instrumentation.recordStreamChunk({ type: "step-start", payload: {} });+ vi.setSystemTime(modelCallStart);+ instrumentation.recordModelCallStart({ prompt: [] });- // tool-call chunks are request-time during the LLM stream — must not become- // tool observation startTimes (that sorted tools above the generation).- vi.setSystemTime(toolCallChunkA);- instrumentation.recordStreamChunk({+ instrumentation.recordModelStreamPart({type: "tool-call",- payload: { toolCallId: "tool-a", toolName: "listObservations" },+ toolCallId: "tool-a",+ toolName: "listObservations",+ input: "{}",});- vi.setSystemTime(toolCallChunkB);- instrumentation.recordStreamChunk({+ instrumentation.recordModelStreamPart({type: "tool-call",- payload: { toolCallId: "tool-b", toolName: "getTrace" },+ toolCallId: "tool-b",+ toolName: "getTrace",+ input: "{}",});vi.setSystemTime(agUiToolAStart);@@ -1459,10 +1457,9 @@ describe("InAppAgentInstrumentation", () => {},]);- // The model stream finishes before tool execution, but Mastra only invokes- // onStepFinish after the tools have resolved.vi.setSystemTime(providerFinish);- instrumentation.recordModelCallFinish({+ instrumentation.recordModelStreamPart({+ type: "finish",usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },finishReason: { unified: "tool-calls", raw: "tool_use" },});@@ -1471,10 +1468,6 @@ describe("InAppAgentInstrumentation", () => {instrumentation.recordToolExecutionStart("tool-a");vi.setSystemTime(toolAEnd);instrumentation.recordToolExecutionEnd("tool-a");- instrumentation.recordStreamChunk({- type: "tool-result",- payload: { toolCallId: "tool-a", toolName: "listObservations" },- });instrumentation.recordEvents([{type: EventType.TOOL_CALL_RESULT,@@ -1487,10 +1480,6 @@ describe("InAppAgentInstrumentation", () => {instrumentation.recordToolExecutionStart("tool-b");vi.setSystemTime(toolBEnd);instrumentation.recordToolExecutionEnd("tool-b");- instrumentation.recordStreamChunk({- type: "tool-result",- payload: { toolCallId: "tool-b", toolName: "getTrace" },- });instrumentation.recordEvents([{type: EventType.TOOL_CALL_RESULT,@@ -1499,19 +1488,6 @@ describe("InAppAgentInstrumentation", () => {},]);- // Tools completed during the open LLM step — deferred until step-finish.- expect(- mocks.handler.langfuse.enqueue.mock.calls.filter(- ([type]) => type === "tool-create",- ),- ).toHaveLength(0);-- vi.setSystemTime(stepFinish);- instrumentation.recordStepFinish({- usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },- finishReason: "tool-calls",- });-const enqueueOrder = mocks.handler.langfuse.enqueue.mock.calls.filter(([type]) => type === "generation-create" || type === "tool-create",@@ -1555,33 +1531,6 @@ describe("InAppAgentInstrumentation", () => {vi.useRealTimers();});- it("opens an LLM step on first model delta when step-start is missing", () => {- const instrumentation = createInstrumentation();- const firstDelta = new Date("2026-01-01T00:00:10.000Z");- const stepFinish = new Date("2026-01-01T00:00:12.000Z");-- vi.setSystemTime(firstDelta);- instrumentation.recordStreamChunk({- type: "reasoning-delta",- payload: { text: "thinking" },- });- vi.setSystemTime(stepFinish);- instrumentation.recordStepFinish({- usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },- finishReason: "stop",- });-- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(- "generation-create",- expect.objectContaining({- startTime: firstDelta,- endTime: stepFinish,- }),- );-- vi.useRealTimers();- });-it("does not force-close incomplete tools on a normal end (approval interrupt)", () => {const instrumentation = createInstrumentation();@@ -1609,10 +1558,10 @@ describe("InAppAgentInstrumentation", () => {);});- it("closes an open LLM step as ERROR when the run is aborted", () => {+ it("closes an open model call as ERROR when the run is aborted", () => {const instrumentation = createInstrumentation();- instrumentation.recordStreamChunk({ type: "step-start", payload: {} });+ instrumentation.recordModelCallStart({ prompt: [] });instrumentation.end({ aborted: true });expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(@@ -1633,79 +1582,22 @@ describe("InAppAgentInstrumentation", () => {);});- it("emits a generation when step finish arrives without a prior step-start", () => {- const instrumentation = createInstrumentation();-- instrumentation.recordStepFinish({- usage: { inputTokens: 100, outputTokens: 10, totalTokens: 110 },- finishReason: "stop",- });- instrumentation.end({});-- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(- "generation-create",- expect.objectContaining({- id: getInAppAgentLlmCallObservationId(runId, 1),- usageDetails: {- input: 100,- output: 10,- cache_read_input_tokens: 0,- cache_creation_input_tokens: 0,- total: 110,- },- model: "eu.anthropic.claude-sonnet-4-5-20250929-v1:0",- }),- );- });-- it("uses provider finish usage when approval ends before step finish", () => {- const instrumentation = createInstrumentation();-- instrumentation.recordStreamChunk({ type: "step-start", payload: {} });- instrumentation.recordModelCallFinish({- usage: {- inputTokens: {- total: 1_100,- noCache: 200,- cacheRead: 800,- cacheWrite: 100,- },- outputTokens: { total: 50, text: 50, reasoning: 0 },- },- finishReason: { unified: "tool-calls", raw: "tool_use" },- });- instrumentation.end({});-- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(- "generation-create",- expect.objectContaining({- id: getInAppAgentLlmCallObservationId(runId, 1),- model: "eu.anthropic.claude-sonnet-4-5-20250929-v1:0",- usageDetails: {- input: 200,- output: 50,- cache_read_input_tokens: 800,- cache_creation_input_tokens: 100,- total: 1_150,- },- metadata: expect.objectContaining({- finish_reason: "tool-calls",- }),- }),- );- });-it("keeps a provider-finished generation successful when the run later fails", () => {const instrumentation = createInstrumentation();- instrumentation.recordStreamChunk({ type: "step-start", payload: {} });+ instrumentation.recordModelCallStart({ prompt: [] });+ instrumentation.recordModelStreamPart({+ type: "text-delta",+ delta: "Calling the tool",+ });instrumentation.recordEvents([{type: EventType.TEXT_MESSAGE_CONTENT,delta: "Calling the tool",},]);- instrumentation.recordModelCallFinish({+ instrumentation.recordModelStreamPart({+ type: "finish",usage: {inputTokens: {total: 1_100,@@ -1731,9 +1623,7 @@ describe("InAppAgentInstrumentation", () => {output: 50,total: 1_150,}),- output: {- messages: [{ role: "assistant", content: "Calling the tool" }],- },+ output: { text: "Calling the tool" },}),);expect(generation).not.toHaveProperty("level");@@ -1747,20 +1637,6 @@ describe("InAppAgentInstrumentation", () => {}),);});-- it("ignores step finish events after instrumentation ended", () => {- const instrumentation = createInstrumentation();-- instrumentation.end({});- instrumentation.recordStepFinish({- usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },- });-- expect(mocks.handler.langfuse.enqueue).not.toHaveBeenCalledWith(- "generation-create",- expect.anything(),- );- });});function createInstrumentation(