Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(in-app-agent): trace model calls at stream boundary (#16231)

Lysander Kiesel63331b121598
diff --git a/packages/shared/src/in-app-agent/server/agent.ts b/packages/shared/src/in-app-agent/server/agent.ts
index e5485b6dca..b447c48f98 100644
--- a/packages/shared/src/in-app-agent/server/agent.ts
+++ b/packages/shared/src/in-app-agent/server/agent.ts
@@ -262,24 +262,17 @@ export async function createAgUiStream(params: {
recordInstrumentation("recordAvailableSkills", (instrumentation) =>
instrumentation.recordAvailableSkills?.(LANGFUSE_IN_APP_AGENT_SKILLS),
);
- const onStepFinish = instrumentation
- ? (event: unknown) => {
- recordInstrumentation("recordStepFinish", (instrumentation) =>
- instrumentation.recordStepFinish?.(event),
+ const onModelCallStart = instrumentation
+ ? (options: unknown) => {
+ recordInstrumentation("recordModelCallStart", (instrumentation) =>
+ instrumentation.recordModelCallStart?.(options),
);
}
: undefined;
- const onChunk = instrumentation
- ? (chunk: unknown) => {
- recordInstrumentation("recordStreamChunk", (instrumentation) =>
- instrumentation.recordStreamChunk?.(chunk),
- );
- }
- : undefined;
- const onModelCallFinish = instrumentation
- ? (event: unknown) => {
- recordInstrumentation("recordModelCallFinish", (instrumentation) =>
- instrumentation.recordModelCallFinish?.(event),
+ const onModelStreamPart = instrumentation
+ ? (part: unknown) => {
+ recordInstrumentation("recordModelStreamPart", (instrumentation) =>
+ instrumentation.recordModelStreamPart?.(part),
);
}
: undefined;
@@ -554,9 +547,8 @@ export async function createAgUiStream(params: {
recordInstrumentation("recordAvailableTools", (instrumentation) =>
instrumentation.recordAvailableTools?.(tools),
),
- onStepFinish,
- onChunk,
- onModelCallFinish,
+ onModelCallStart,
+ onModelStreamPart,
onToolExecutionStart,
onToolExecutionEnd,
})
@@ -626,9 +618,8 @@ export async function createAgUiStream(params: {
},
awsProfile,
instructions,
- onStepFinish,
- onChunk,
- onModelCallFinish,
+ onModelCallStart,
+ onModelStreamPart,
onToolExecutionStart,
onToolExecutionEnd,
});
@@ -857,11 +848,14 @@ type ExecutableInAppAgentTool = {
type BedrockLanguageModel = ReturnType<ReturnType<typeof createAmazonBedrock>>;
-function withModelCallFinish(
+function withModelTracing(
model: BedrockLanguageModel,
- onFinish?: (event: unknown) => void,
+ callbacks: {
+ onStart?: (options: unknown) => void;
+ onStreamPart?: (part: unknown) => void;
+ },
): BedrockLanguageModel {
- if (!onFinish) {
+ if (!callbacks.onStart && !callbacks.onStreamPart) {
return model;
}
@@ -872,6 +866,7 @@ function withModelCallFinish(
supportedUrls: model.supportedUrls,
doGenerate: (options) => model.doGenerate(options),
doStream: async (options) => {
+ callbacks.onStart?.(options);
const result = await model.doStream(options);
return {
@@ -879,9 +874,7 @@ function withModelCallFinish(
stream: result.stream.pipeThrough(
new TransformStream({
transform(part, controller) {
- if (part.type === "finish") {
- onFinish(part);
- }
+ callbacks.onStreamPart?.(part);
controller.enqueue(part);
},
}),
@@ -942,9 +935,8 @@ async function createMastraAdapter(params: {
awsProfile?: string;
instructions: string;
onToolsAvailable?: (tools: Record<string, unknown>) => void;
- onStepFinish?: (event: unknown) => void;
- onChunk?: (chunk: unknown) => void;
- onModelCallFinish?: (event: unknown) => void;
+ onModelCallStart?: (options: unknown) => void;
+ onModelStreamPart?: (part: unknown) => void;
onToolExecutionStart?: (toolCallId: string) => void;
onToolExecutionEnd?: (toolCallId: string) => void;
}) {
@@ -1078,11 +1070,14 @@ async function createMastraAdapter(params: {
// instruction channel so the model receives the same higher-priority
// guidance on resumed runs.
let developerGuidance: string | undefined;
- const model = withModelCallFinish(
+ const model = withModelTracing(
bedrock(
params.options.awsBedrock.modelId as Parameters<typeof bedrock>[0],
),
- params.onModelCallFinish,
+ {
+ onStart: params.onModelCallStart,
+ onStreamPart: params.onModelStreamPart,
+ },
);
const agent = new Agent({
id: "langfuse-in-app-assistant",
@@ -1099,11 +1094,6 @@ async function createMastraAdapter(params: {
...(params.options.sandboxWorkspaceWasReset
? { system: SANDBOX_WORKSPACE_RESET_INSTRUCTION }
: {}),
- // Fires once per LLM call with that call's token usage; the AG-UI
- // event stream itself never carries usage.
- ...(params.onStepFinish ? { onStepFinish: params.onStepFinish } : {}),
- // Tool execution windows (and optional step-start timing bookmarks).
- ...(params.onChunk ? { onChunk: params.onChunk } : {}),
...(reasoningProviderOptions
? { providerOptions: reasoningProviderOptions }
: {}),
diff --git a/packages/shared/src/in-app-agent/server/instrumentation.ts b/packages/shared/src/in-app-agent/server/instrumentation.ts
index 1b163f206a..0224f977d7 100644
--- a/packages/shared/src/in-app-agent/server/instrumentation.ts
+++ b/packages/shared/src/in-app-agent/server/instrumentation.ts
@@ -1,4 +1,5 @@
import { EventType } from "@ag-ui/core";
+import { Buffer } from "node:buffer";
import { getInternalTracingHandler, logger } from "../../server";
import {
@@ -116,16 +117,20 @@ type AgentRunToolSpan = {
failureMessage?: string;
parentMessageId?: string;
};
-// Lightweight bookmark for the in-flight Mastra step. Generations are emitted
-// from onStepFinish (which carries usage); this only supplies start timing and
-// the input-message boundary for context-growth visibility.
-type OpenLlmStep = {
- stepNumber: number;
+type ModelToolCall = {
+ toolCallId: string;
+ toolName: string;
+ args: unknown;
+};
+type OpenModelCall = {
+ callNumber: number;
startTime: Date;
- inputMessageCount: number;
+ input: unknown;
completionStartTime?: Date;
- providerFinish?: Record<string, unknown>;
- providerFinishTime?: Date;
+ textDeltas: string[];
+ toolCalls: ModelToolCall[];
+ modelId?: string;
+ streamErrorMessage?: string;
};
type ToolExecutionTimes = {
startTime?: Date;
@@ -198,17 +203,8 @@ export class InAppAgentInstrumentation {
private pendingThinking: AgentRunThinkingPart[] = [];
private ended = false;
private readonly model?: string;
- private nextStepNumber = 0;
- private openLlmStep?: OpenLlmStep;
- // Tools that finished while an LLM step was still open. Emit after the
- // generation so the tree sorts model → tools (tool-call chunks arrive during
- // the LLM stream, before onStepFinish).
- private readonly deferredToolCompletions = new Map<
- string,
- AgentRunToolSpan
- >();
- // Clamp tool execution windows to start at/after the last generation end.
- private lastLlmGenerationEndTime?: Date;
+ private nextModelCallNumber = 0;
+ private openModelCall?: OpenModelCall;
constructor(params: {
input: AgUiRunAgentInput;
@@ -348,120 +344,73 @@ export class InAppAgentInstrumentation {
this.toolExecutionTimes.set(toolCallId, times);
}
- // Bedrock's V3 stream emits exact usage before Mastra executes tools. Keep
- // it on the open step as a fallback because approval suspension can end the
- // run without Mastra ever invoking onStepFinish.
- recordModelCallFinish(event: unknown) {
- if (!isRecord(event)) {
- return;
- }
-
+ recordModelCallStart(options: unknown) {
if (this.ended) {
return;
}
const now = new Date();
- this.ensureOpenLlmStep(now);
- if (this.openLlmStep) {
- this.openLlmStep.providerFinish = event;
- this.openLlmStep.providerFinishTime = now;
- }
+ this.openModelCall = {
+ callNumber: this.openModelCall?.callNumber ?? ++this.nextModelCallNumber,
+ startTime: this.openModelCall?.startTime ?? now,
+ input: getModelCallInput(options),
+ textDeltas: [],
+ toolCalls: [],
+ };
}
- // Mastra stream chunks: step-start (or first model delta) bookmarks the LLM
- // window; tool-result ends tool execution. Do not treat tool-call chunks as
- // execution start — they fire while the model is still streaming.
- // Usage and output still come from recordStepFinish.
- recordStreamChunk(chunk: unknown) {
- if (this.ended || !isRecord(chunk)) {
+ recordModelStreamPart(part: unknown) {
+ if (this.ended || !this.openModelCall || !isRecord(part)) {
return;
}
- const type = chunk.type;
+ const type = part.type;
if (typeof type !== "string") {
return;
}
- const payload = isRecord(chunk.payload) ? chunk.payload : {};
const now = new Date();
- if (type === "step-start") {
- // A prior step should have been closed by onStepFinish. If not, emit it
- // as an error so we never silently drop a generation.
- if (this.openLlmStep) {
- const providerFinish = this.openLlmStep.providerFinish;
- this.emitLlmGeneration(
- this.openLlmStep,
- undefined,
- providerFinish
- ? undefined
- : {
- level: "ERROR",
- statusMessage: "LLM step closed by a subsequent step-start",
- },
- );
- this.flushDeferredToolCompletions();
+ if (type === "text-delta" || type === "reasoning-delta") {
+ this.openModelCall.completionStartTime ??= now;
+ if (type === "text-delta" && typeof part.delta === "string") {
+ this.openModelCall.textDeltas.push(part.delta);
}
-
- this.openLlmStep = {
- stepNumber: ++this.nextStepNumber,
- startTime: now,
- inputMessageCount: this.agentRunOutputMessages.length,
- };
return;
}
- // tool-call means "model requested this tool", not "tool started".
- // Ensure an LLM step is open so missing step-start still gets a real window.
if (type === "tool-call") {
- this.ensureOpenLlmStep(now);
+ const toolCallId = getStringValue(part.toolCallId);
+ const toolName = getStringValue(part.toolName);
+ if (toolCallId && toolName) {
+ this.openModelCall.toolCalls.push({
+ toolCallId,
+ toolName,
+ args:
+ typeof part.input === "string"
+ ? parseJsonOrString(part.input)
+ : part.input,
+ });
+ }
return;
}
- if (type === "tool-result" || type === "tool-error") {
- const toolCallId = getStringValue(payload.toolCallId);
- if (!toolCallId) {
- return;
- }
-
- const times = this.toolExecutionTimes.get(toolCallId) ?? {};
- times.endTime ??= now;
- this.toolExecutionTimes.set(toolCallId, times);
+ if (type === "error") {
+ this.openModelCall.streamErrorMessage = getErrorMessage(part.error);
return;
}
- if (
- type === "text-delta" ||
- type === "reasoning-delta" ||
- type === "reasoning"
- ) {
- this.ensureOpenLlmStep(now);
- if (this.openLlmStep) {
- this.openLlmStep.completionStartTime ??= now;
- }
+ if (type === "response-metadata") {
+ this.openModelCall.modelId = getStringValue(part.modelId);
+ return;
}
- }
- // Receives Mastra's per-LLM-call onStepFinish event and emits one generation
- // with that call's usage/model. The AG-UI event stream itself never carries usage.
- recordStepFinish(event: unknown) {
- if (this.ended) {
+ if (type !== "finish") {
return;
}
- const step =
- this.openLlmStep ??
- ({
- stepNumber: ++this.nextStepNumber,
- // Prefer a non-zero window when step-start never arrived: backdate to
- // the last generation end (or run start) so tools do not sort above us.
- startTime: this.lastLlmGenerationEndTime ?? this.agentRunStartTime,
- inputMessageCount: this.agentRunOutputMessages.length,
- } satisfies OpenLlmStep);
-
- this.emitLlmGeneration(step, event);
- this.openLlmStep = undefined;
- this.flushDeferredToolCompletions();
+ this.emitModelGeneration(this.openModelCall, part, now);
+ this.openModelCall = undefined;
}
recordAvailableSkills(skills: unknown[]) {
@@ -487,7 +436,7 @@ export class InAppAgentInstrumentation {
}
const message = error instanceof Error ? error.message : String(error);
- this.endOpenLlmStep({ error: message }, message);
+ this.endOpenModelCall(message);
this.endOpenToolSpans({ error: message }, message);
this.flushTrailingThinking();
this.emitRootAgentObservation({
@@ -512,9 +461,9 @@ export class InAppAgentInstrumentation {
}
if (params?.aborted) {
- this.endOpenLlmStep({ aborted: true }, "aborted");
+ this.endOpenModelCall("aborted");
} else {
- this.endOpenLlmStep();
+ this.endOpenModelCall("Model call ended before provider finish");
}
this.endOpenToolSpans(params?.aborted ? { aborted: true } : undefined);
this.flushTrailingThinking();
@@ -701,14 +650,6 @@ export class InAppAgentInstrumentation {
return;
}
- // Tool-call args/results often arrive while the LLM step is still open.
- // Hold the observation until step-finish so the tree sorts generation → tools.
- if (this.openLlmStep) {
- this.deferredToolCompletions.set(toolCallId, tool);
- this.toolSpans.delete(toolCallId);
- return;
- }
-
this.createToolObservation(toolCallId, tool);
this.toolSpans.delete(toolCallId);
}
@@ -735,22 +676,16 @@ export class InAppAgentInstrumentation {
toolCallApproval === "rejected"
? rawStartTime
: (executionTimes?.endTime ?? new Date());
- // Keep causal order: tool execution cannot start before the generation that
- // requested it finished (or the last known generation end).
const startTime =
- this.lastLlmGenerationEndTime &&
- rawStartTime.getTime() < this.lastLlmGenerationEndTime.getTime()
- ? this.lastLlmGenerationEndTime
- : rawStartTime;
+ rawStartTime.getTime() > endTime.getTime() ? endTime : rawStartTime;
const body: ObservationBody = {
id: toolCallId,
traceId: this.traceId,
parentObservationId: this.rootObservationId,
name: tool.name,
- startTime: startTime.getTime() > endTime.getTime() ? endTime : startTime,
+ startTime,
endTime,
- completionStartTime:
- startTime.getTime() > endTime.getTime() ? endTime : startTime,
+ completionStartTime: startTime,
input,
output,
...(failureMessage
@@ -799,40 +734,42 @@ export class InAppAgentInstrumentation {
});
}
- private emitLlmGeneration(
- step: OpenLlmStep,
- event: unknown,
+ private emitModelGeneration(
+ call: OpenModelCall,
+ finish: Record<string, unknown> | undefined,
+ endTime: Date,
options?: {
level?: "ERROR";
statusMessage?: string;
},
) {
- const eventRecord = isRecord(event) ? event : undefined;
- const providerFinish = step.providerFinish;
- const usageDetails =
- getStepUsageDetails(eventRecord?.usage) ??
- getStepUsageDetails(providerFinish?.usage);
- const finishReason =
- getStepFinishReason(eventRecord) ?? getStepFinishReason(providerFinish);
- const model =
- getModelIdFromStepEvent(eventRecord) ?? this.model ?? undefined;
- // Mastra can delay onStepFinish until after requested tools execute. The
- // raw provider finish is the actual model boundary and keeps the following
- // tool's execute() window intact instead of clamping it to zero duration.
- const endTime = step.providerFinishTime ?? new Date();
- this.lastLlmGenerationEndTime = endTime;
- const id = getInAppAgentLlmCallObservationId(this.runId, step.stepNumber);
+ const usageDetails = getModelUsageDetails(finish?.usage);
+ const finishReason = getModelFinishReason(finish);
+ const model = call.modelId ?? this.model;
+ const id = getInAppAgentLlmCallObservationId(this.runId, call.callNumber);
+ const text = call.textDeltas.join("");
+ const output =
+ text || call.toolCalls.length > 0
+ ? {
+ ...(text ? { text } : {}),
+ ...(call.toolCalls.length > 0
+ ? { tool_calls: call.toolCalls }
+ : {}),
+ }
+ : undefined;
this.enqueueObservation("generation-create", {
id,
traceId: this.traceId,
parentObservationId: this.rootObservationId,
name: IN_APP_AGENT_LLM_CALL_NAME,
- startTime: step.startTime,
+ startTime: call.startTime,
endTime,
- completionStartTime: step.completionStartTime ?? step.startTime,
- input: this.getLlmStepInput(step.inputMessageCount),
- output: this.getLlmStepOutput(step, eventRecord),
+ ...(call.completionStartTime
+ ? { completionStartTime: call.completionStartTime }
+ : {}),
+ input: call.input,
+ output,
// Model only travels with usage so Langfuse does not invent tokenizer-
// estimated costs for generations that never reported tokens.
...(usageDetails
@@ -852,19 +789,6 @@ export class InAppAgentInstrumentation {
});
}
- private getLlmStepInput(inputMessageCount: number) {
- const base = isRecord(this.agentRunInput) ? this.agentRunInput : {};
- const baseMessages = Array.isArray(base.messages) ? base.messages : [];
-
- return {
- ...base,
- messages: [
- ...baseMessages,
- ...this.agentRunOutputMessages.slice(0, inputMessageCount),
- ],
- };
- }
-
private getTraceInput() {
if (!this.approvalContinuation || !isRecord(this.agentRunInput)) {
return this.agentRunInput;
@@ -890,52 +814,17 @@ export class InAppAgentInstrumentation {
};
}
- private getLlmStepOutput(
- step: OpenLlmStep,
- event: Record<string, unknown> | undefined,
- ) {
- const text = typeof event?.text === "string" ? event.text : undefined;
- const toolCalls = Array.isArray(event?.toolCalls)
- ? event.toolCalls
- : undefined;
- if (text !== undefined || toolCalls?.length) {
- return {
- ...(text !== undefined ? { text } : {}),
- ...(toolCalls?.length ? { tool_calls: toolCalls } : {}),
- };
- }
-
- const response = isRecord(event?.response) ? event.response : undefined;
- if (Array.isArray(response?.messages) && response.messages.length > 0) {
- return { messages: response.messages };
- }
-
- const messages = this.agentRunOutputMessages.slice(step.inputMessageCount);
- return messages.length > 0 ? { messages } : undefined;
- }
-
- private endOpenLlmStep(
- metadata?: Record<string, unknown>,
- statusMessage?: string,
- ) {
- if (!this.openLlmStep) {
+ private endOpenModelCall(statusMessage: string) {
+ if (!this.openModelCall) {
return;
}
- this.emitLlmGeneration(this.openLlmStep, undefined, {
- ...(!this.openLlmStep.providerFinish && (statusMessage || metadata)
- ? {
- level: "ERROR" as const,
- statusMessage:
- statusMessage ??
- (typeof metadata?.error === "string"
- ? metadata.error
- : "LLM step closed before step finish"),
- }
- : {}),
+ const message = this.openModelCall.streamErrorMessage ?? statusMessage;
+ this.emitModelGeneration(this.openModelCall, undefined, new Date(), {
+ level: "ERROR",
+ statusMessage: message,
});
- this.openLlmStep = undefined;
- this.flushDeferredToolCompletions();
+ this.openModelCall = undefined;
}
private endOpenToolSpans(
@@ -963,34 +852,6 @@ export class InAppAgentInstrumentation {
});
this.toolSpans.delete(toolCallId);
}
-
- // endOpenLlmStep() has already drained completed tools before this method
- // runs; keep the unconditional flush to preserve that lifecycle invariant.
- this.flushDeferredToolCompletions();
- }
-
- private ensureOpenLlmStep(now: Date) {
- if (this.openLlmStep) {
- return;
- }
-
- this.openLlmStep = {
- stepNumber: ++this.nextStepNumber,
- startTime: now,
- inputMessageCount: this.agentRunOutputMessages.length,
- };
- }
-
- private flushDeferredToolCompletions() {
- if (this.deferredToolCompletions.size === 0) {
- return;
- }
-
- const deferred = [...this.deferredToolCompletions.entries()];
- this.deferredToolCompletions.clear();
- for (const [toolCallId, tool] of deferred) {
- this.createToolObservation(toolCallId, tool);
- }
}
private enqueueObservation(type: string, body: ObservationBody) {
@@ -1421,7 +1282,31 @@ function getFiniteNumber(value: unknown): number | undefined {
: undefined;
}
-function getStepUsageDetails(
+function getModelCallInput(options: unknown): unknown {
+ if (!isRecord(options)) {
+ return undefined;
+ }
+
+ const input = {
+ messages: options.prompt,
+ maxOutputTokens: options.maxOutputTokens,
+ temperature: options.temperature,
+ stopSequences: options.stopSequences,
+ topP: options.topP,
+ topK: options.topK,
+ presencePenalty: options.presencePenalty,
+ frequencyPenalty: options.frequencyPenalty,
+ responseFormat: options.responseFormat,
+ seed: options.seed,
+ tools: options.tools,
+ toolChoice: options.toolChoice,
+ providerOptions: options.providerOptions,
+ };
+
+ return toSerializableJson(input);
+}
+
+function getModelUsageDetails(
usage: unknown,
): Record<string, number> | undefined {
if (!isRecord(usage)) {
@@ -1454,7 +1339,7 @@ function getStepUsageDetails(
0;
const uncachedInputTokens = getFiniteNumber(nestedInputTokens?.noCache);
- // Mastra's inputTokens includes cache reads/writes, while Langfuse model
+ // Provider inputTokens includes cache reads/writes, while Langfuse model
// prices bill `input` as non-cached input tokens with separate cache keys.
return {
input:
@@ -1469,7 +1354,7 @@ function getStepUsageDetails(
};
}
-function getStepFinishReason(
+function getModelFinishReason(
event: Record<string, unknown> | undefined,
): string | undefined {
if (!event) {
@@ -1487,29 +1372,6 @@ function getStepFinishReason(
);
}
- return isRecord(event.stepResult)
- ? getStringValue(event.stepResult.reason)
- : undefined;
-}
-
-function getModelIdFromStepEvent(
- event: Record<string, unknown> | undefined,
-): string | undefined {
- if (!event) {
- return undefined;
- }
-
- if (isRecord(event.model)) {
- const modelId = getStringValue(event.model.modelId);
- if (modelId) {
- return modelId;
- }
- }
-
- if (isRecord(event.response)) {
- return getStringValue(event.response.modelId);
- }
-
return undefined;
}
@@ -1528,6 +1390,14 @@ function toSerializableJson(
seen = new WeakSet<object>(),
depth = 0,
): unknown {
+ if (value instanceof URL) {
+ return value.toString();
+ }
+
+ if (value instanceof Uint8Array) {
+ return Buffer.from(value).toString("base64");
+ }
+
if (value === null || typeof value === "string") {
return value;
}
@@ -1569,7 +1439,10 @@ function toSerializableJson(
seen.add(value);
const entries = Object.entries(value).flatMap(([key, item]) => {
- const serializableItem = toSerializableJson(item, seen, depth + 1);
+ const serializableItem =
+ key === "data" && item instanceof Uint8Array
+ ? `data:${getStringValue(value.mediaType) ?? "application/octet-stream"};base64,${Buffer.from(item).toString("base64")}`
+ : toSerializableJson(item, seen, depth + 1);
return serializableItem === undefined
? []
@@ -1592,3 +1465,11 @@ function parseJsonOrUndefined(value: string): unknown {
function getStringValue(value: unknown): string | undefined {
return typeof value === "string" && value.trim() ? value : undefined;
}
+
+function getErrorMessage(error: unknown): string {
+ if (error instanceof Error) {
+ return error.message;
+ }
+
+ return typeof error === "string" ? error : "Model stream failed";
+}
diff --git a/web/src/__tests__/server/in-app-agent-stream.servertest.ts b/web/src/__tests__/server/in-app-agent-stream.servertest.ts
index 0bf7b6714a..696a9547e4 100644
--- a/web/src/__tests__/server/in-app-agent-stream.servertest.ts
+++ b/web/src/__tests__/server/in-app-agent-stream.servertest.ts
@@ -98,9 +98,8 @@ const instrumentationMocks = vi.hoisted(() => {
recordToolCallApproval: vi.fn(),
recordToolExecutionStart: vi.fn(),
recordToolExecutionEnd: vi.fn(),
- recordModelCallFinish: vi.fn(),
- recordStepFinish: vi.fn(),
- recordStreamChunk: vi.fn(),
+ recordModelCallStart: vi.fn(),
+ recordModelStreamPart: vi.fn(),
end: vi.fn(),
endWithError: vi.fn(),
flush: vi.fn(),
@@ -555,11 +554,11 @@ describe("createAgUiStream", () => {
};
it("forwards model stream parts when finish tracing throws", async () => {
- instrumentationMocks.instrumentation.recordModelCallFinish.mockImplementationOnce(
- () => {
+ instrumentationMocks.instrumentation.recordModelStreamPart
+ .mockImplementationOnce(() => undefined)
+ .mockImplementationOnce(() => {
throw new Error("tracing failed");
- },
- );
+ });
await initializeBasicTracedAgent("run-provider-finish-tracing-error");
const textPart = { type: "text-delta", id: "text-1", delta: "hello" };
@@ -578,7 +577,8 @@ describe("createAgUiStream", () => {
}>;
};
- const modelResult = await model.doStream({});
+ const options = { prompt: [] };
+ const modelResult = await model.doStream(options);
const forwardedParts: unknown[] = [];
for await (const part of modelResult.stream) {
forwardedParts.push(part);
@@ -586,8 +586,14 @@ describe("createAgUiStream", () => {
expect(forwardedParts).toEqual([textPart, finishPart]);
expect(
- instrumentationMocks.instrumentation.recordModelCallFinish,
- ).toHaveBeenCalledWith(finishPart);
+ instrumentationMocks.instrumentation.recordModelCallStart,
+ ).toHaveBeenCalledWith(options);
+ expect(
+ instrumentationMocks.instrumentation.recordModelStreamPart,
+ ).toHaveBeenNthCalledWith(1, textPart);
+ expect(
+ instrumentationMocks.instrumentation.recordModelStreamPart,
+ ).toHaveBeenNthCalledWith(2, finishPart);
});
it("serializes valid events including adapter snapshots and reasoning messages", async () => {
@@ -935,26 +941,6 @@ describe("createAgUiStream", () => {
}),
model: "eu.anthropic.claude-opus-4-8",
});
- const onStepFinish = (
- agentConfig?.defaultOptions as
- | { onStepFinish?: (event: unknown) => void }
- | undefined
- )?.onStepFinish;
- expect(onStepFinish).toEqual(expect.any(Function));
- onStepFinish?.({ usage: { inputTokens: 10, outputTokens: 5 } });
- expect(
- instrumentationMocks.instrumentation.recordStepFinish,
- ).toHaveBeenCalledWith({ usage: { inputTokens: 10, outputTokens: 5 } });
- const onChunk = (
- agentConfig?.defaultOptions as
- | { onChunk?: (chunk: unknown) => void }
- | undefined
- )?.onChunk;
- expect(onChunk).toEqual(expect.any(Function));
- onChunk?.({ type: "step-start", payload: {} });
- expect(
- instrumentationMocks.instrumentation.recordStreamChunk,
- ).toHaveBeenCalledWith({ type: "step-start", payload: {} });
expect(
instrumentationMocks.instrumentation.recordEvents.mock.calls.flatMap(
([events]) => (events as AgUiEvent[]).map((event) => event.type),
diff --git a/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts b/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts
index e55b563398..3774dd33ca 100644
--- a/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts
+++ b/web/src/__tests__/server/unit/in-app-agent-instrumentation.servertest.ts
@@ -890,8 +890,29 @@ describe("InAppAgentInstrumentation", () => {
content: '{"name":"FOO","version":1}',
},
]);
- instrumentation.recordStreamChunk({ type: "step-start", payload: {} });
- instrumentation.recordStepFinish({
+ instrumentation.recordModelCallStart({
+ prompt: [
+ { role: "user", content: "hello" },
+ {
+ role: "assistant",
+ tool_calls: [
+ {
+ id: "tool-1",
+ name: "langfuse_createTextPrompt",
+ arguments: '{"name":"FOO"}',
+ type: "function",
+ },
+ ],
+ },
+ {
+ role: "tool",
+ tool_call_id: "tool-1",
+ content: { name: "FOO", version: 1 },
+ },
+ ],
+ });
+ instrumentation.recordModelStreamPart({
+ type: "finish",
usage: { inputTokens: 100, outputTokens: 10, totalTokens: 110 },
});
instrumentation.end({});
@@ -1222,7 +1243,7 @@ describe("InAppAgentInstrumentation", () => {
);
});
- it("emits one generation per LLM step with usage on children only", () => {
+ it("emits one generation per model call with usage on children only", () => {
const instrumentation = createInstrumentation();
const modelName = "eu.anthropic.claude-sonnet-4-5-20250929-v1:0";
@@ -1236,13 +1257,25 @@ describe("InAppAgentInstrumentation", () => {
}),
);
- instrumentation.recordStreamChunk({
- type: "step-start",
- payload: {},
+ instrumentation.recordModelCallStart({
+ prompt: [{ role: "user", content: [{ type: "text", text: "hello" }] }],
});
- instrumentation.recordStreamChunk({
+ instrumentation.recordModelStreamPart({
type: "tool-call",
- payload: { toolCallId: "tool-1", toolName: "listObservations" },
+ toolCallId: "tool-1",
+ toolName: "listObservations",
+ input: '{"limit":10}',
+ });
+ instrumentation.recordModelStreamPart({
+ type: "finish",
+ usage: {
+ inputTokens: 1100,
+ outputTokens: 50,
+ totalTokens: 1150,
+ cachedInputTokens: 800,
+ cacheCreationInputTokens: 100,
+ },
+ finishReason: "tool-calls",
});
instrumentation.recordEvents([
{
@@ -1265,47 +1298,16 @@ describe("InAppAgentInstrumentation", () => {
content: '{"ok":true}',
},
]);
- instrumentation.recordStreamChunk({
- type: "tool-result",
- payload: { toolCallId: "tool-1", toolName: "listObservations" },
- });
- instrumentation.recordStepFinish({
- usage: {
- inputTokens: 1100,
- outputTokens: 50,
- totalTokens: 1150,
- cachedInputTokens: 800,
- cacheCreationInputTokens: 100,
- },
- finishReason: "tool-calls",
- text: "",
- toolCalls: [
- {
- toolCallId: "tool-1",
- toolName: "listObservations",
- args: { limit: 10 },
- },
+
+ instrumentation.recordModelCallStart({
+ prompt: [
+ { role: "user", content: [{ type: "text", text: "hello" }] },
+ { role: "tool", content: [{ type: "tool-result" }] },
],
- response: {
- messages: [
- {
- role: "assistant",
- content: "",
- tool_calls: [{ id: "tool-1" }],
- },
- {
- role: "tool",
- tool_call_id: "tool-1",
- content: { ok: true },
- },
- ],
- },
});
-
- instrumentation.recordStreamChunk({ type: "step-start", payload: {} });
- instrumentation.recordStreamChunk({
+ instrumentation.recordModelStreamPart({
type: "text-delta",
- payload: { text: "Done" },
+ delta: "Done",
});
instrumentation.recordEvents([
{
@@ -1313,7 +1315,8 @@ describe("InAppAgentInstrumentation", () => {
delta: "Done",
},
]);
- instrumentation.recordStepFinish({
+ instrumentation.recordModelStreamPart({
+ type: "finish",
usage: {
inputTokens: 1200,
outputTokens: 30,
@@ -1321,7 +1324,6 @@ describe("InAppAgentInstrumentation", () => {
cachedInputTokens: 1000,
},
finishReason: "stop",
- text: "Done",
});
instrumentation.end({});
@@ -1349,7 +1351,6 @@ describe("InAppAgentInstrumentation", () => {
);
expect(generationCreates[0]?.[1].input.messages).toHaveLength(1);
expect(generationCreates[0]?.[1].output).toEqual({
- text: "",
tool_calls: [
{
toolCallId: "tool-1",
@@ -1394,11 +1395,9 @@ describe("InAppAgentInstrumentation", () => {
expect(finalAgentCreate).not.toHaveProperty("model");
});
- it("emits tools after the generation and clamps start to generation end", () => {
+ it("emits tools after the generation using actual execution timing", () => {
const instrumentation = createInstrumentation();
- const stepStart = new Date("2026-01-01T00:00:00.000Z");
- const toolCallChunkA = new Date("2026-01-01T00:00:01.000Z");
- const toolCallChunkB = new Date("2026-01-01T00:00:02.000Z");
+ const modelCallStart = new Date("2026-01-01T00:00:00.000Z");
const agUiToolAStart = new Date("2026-01-01T00:00:01.500Z");
const agUiToolBStart = new Date("2026-01-01T00:00:02.500Z");
const toolAStart = new Date("2026-01-01T00:00:02.900Z");
@@ -1406,22 +1405,21 @@ describe("InAppAgentInstrumentation", () => {
const toolAEnd = new Date("2026-01-01T00:00:03.000Z");
const toolBEnd = new Date("2026-01-01T00:00:04.000Z");
const providerFinish = new Date("2026-01-01T00:00:02.800Z");
- const stepFinish = new Date("2026-01-01T00:00:04.200Z");
- vi.setSystemTime(stepStart);
- instrumentation.recordStreamChunk({ type: "step-start", payload: {} });
+ vi.setSystemTime(modelCallStart);
+ instrumentation.recordModelCallStart({ prompt: [] });
- // tool-call chunks are request-time during the LLM stream — must not become
- // tool observation startTimes (that sorted tools above the generation).
- vi.setSystemTime(toolCallChunkA);
- instrumentation.recordStreamChunk({
+ instrumentation.recordModelStreamPart({
type: "tool-call",
- payload: { toolCallId: "tool-a", toolName: "listObservations" },
+ toolCallId: "tool-a",
+ toolName: "listObservations",
+ input: "{}",
});
- vi.setSystemTime(toolCallChunkB);
- instrumentation.recordStreamChunk({
+ instrumentation.recordModelStreamPart({
type: "tool-call",
- payload: { toolCallId: "tool-b", toolName: "getTrace" },
+ toolCallId: "tool-b",
+ toolName: "getTrace",
+ input: "{}",
});
vi.setSystemTime(agUiToolAStart);
@@ -1459,10 +1457,9 @@ describe("InAppAgentInstrumentation", () => {
},
]);
- // The model stream finishes before tool execution, but Mastra only invokes
- // onStepFinish after the tools have resolved.
vi.setSystemTime(providerFinish);
- instrumentation.recordModelCallFinish({
+ instrumentation.recordModelStreamPart({
+ type: "finish",
usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
finishReason: { unified: "tool-calls", raw: "tool_use" },
});
@@ -1471,10 +1468,6 @@ describe("InAppAgentInstrumentation", () => {
instrumentation.recordToolExecutionStart("tool-a");
vi.setSystemTime(toolAEnd);
instrumentation.recordToolExecutionEnd("tool-a");
- instrumentation.recordStreamChunk({
- type: "tool-result",
- payload: { toolCallId: "tool-a", toolName: "listObservations" },
- });
instrumentation.recordEvents([
{
type: EventType.TOOL_CALL_RESULT,
@@ -1487,10 +1480,6 @@ describe("InAppAgentInstrumentation", () => {
instrumentation.recordToolExecutionStart("tool-b");
vi.setSystemTime(toolBEnd);
instrumentation.recordToolExecutionEnd("tool-b");
- instrumentation.recordStreamChunk({
- type: "tool-result",
- payload: { toolCallId: "tool-b", toolName: "getTrace" },
- });
instrumentation.recordEvents([
{
type: EventType.TOOL_CALL_RESULT,
@@ -1499,19 +1488,6 @@ describe("InAppAgentInstrumentation", () => {
},
]);
- // Tools completed during the open LLM step — deferred until step-finish.
- expect(
- mocks.handler.langfuse.enqueue.mock.calls.filter(
- ([type]) => type === "tool-create",
- ),
- ).toHaveLength(0);
-
- vi.setSystemTime(stepFinish);
- instrumentation.recordStepFinish({
- usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
- finishReason: "tool-calls",
- });
-
const enqueueOrder = mocks.handler.langfuse.enqueue.mock.calls
.filter(
([type]) => type === "generation-create" || type === "tool-create",
@@ -1555,33 +1531,6 @@ describe("InAppAgentInstrumentation", () => {
vi.useRealTimers();
});
- it("opens an LLM step on first model delta when step-start is missing", () => {
- const instrumentation = createInstrumentation();
- const firstDelta = new Date("2026-01-01T00:00:10.000Z");
- const stepFinish = new Date("2026-01-01T00:00:12.000Z");
-
- vi.setSystemTime(firstDelta);
- instrumentation.recordStreamChunk({
- type: "reasoning-delta",
- payload: { text: "thinking" },
- });
- vi.setSystemTime(stepFinish);
- instrumentation.recordStepFinish({
- usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
- finishReason: "stop",
- });
-
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "generation-create",
- expect.objectContaining({
- startTime: firstDelta,
- endTime: stepFinish,
- }),
- );
-
- vi.useRealTimers();
- });
-
it("does not force-close incomplete tools on a normal end (approval interrupt)", () => {
const instrumentation = createInstrumentation();
@@ -1609,10 +1558,10 @@ describe("InAppAgentInstrumentation", () => {
);
});
- it("closes an open LLM step as ERROR when the run is aborted", () => {
+ it("closes an open model call as ERROR when the run is aborted", () => {
const instrumentation = createInstrumentation();
- instrumentation.recordStreamChunk({ type: "step-start", payload: {} });
+ instrumentation.recordModelCallStart({ prompt: [] });
instrumentation.end({ aborted: true });
expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
@@ -1633,79 +1582,22 @@ describe("InAppAgentInstrumentation", () => {
);
});
- it("emits a generation when step finish arrives without a prior step-start", () => {
- const instrumentation = createInstrumentation();
-
- instrumentation.recordStepFinish({
- usage: { inputTokens: 100, outputTokens: 10, totalTokens: 110 },
- finishReason: "stop",
- });
- instrumentation.end({});
-
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "generation-create",
- expect.objectContaining({
- id: getInAppAgentLlmCallObservationId(runId, 1),
- usageDetails: {
- input: 100,
- output: 10,
- cache_read_input_tokens: 0,
- cache_creation_input_tokens: 0,
- total: 110,
- },
- model: "eu.anthropic.claude-sonnet-4-5-20250929-v1:0",
- }),
- );
- });
-
- it("uses provider finish usage when approval ends before step finish", () => {
- const instrumentation = createInstrumentation();
-
- instrumentation.recordStreamChunk({ type: "step-start", payload: {} });
- instrumentation.recordModelCallFinish({
- usage: {
- inputTokens: {
- total: 1_100,
- noCache: 200,
- cacheRead: 800,
- cacheWrite: 100,
- },
- outputTokens: { total: 50, text: 50, reasoning: 0 },
- },
- finishReason: { unified: "tool-calls", raw: "tool_use" },
- });
- instrumentation.end({});
-
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "generation-create",
- expect.objectContaining({
- id: getInAppAgentLlmCallObservationId(runId, 1),
- model: "eu.anthropic.claude-sonnet-4-5-20250929-v1:0",
- usageDetails: {
- input: 200,
- output: 50,
- cache_read_input_tokens: 800,
- cache_creation_input_tokens: 100,
- total: 1_150,
- },
- metadata: expect.objectContaining({
- finish_reason: "tool-calls",
- }),
- }),
- );
- });
-
it("keeps a provider-finished generation successful when the run later fails", () => {
const instrumentation = createInstrumentation();
- instrumentation.recordStreamChunk({ type: "step-start", payload: {} });
+ instrumentation.recordModelCallStart({ prompt: [] });
+ instrumentation.recordModelStreamPart({
+ type: "text-delta",
+ delta: "Calling the tool",
+ });
instrumentation.recordEvents([
{
type: EventType.TEXT_MESSAGE_CONTENT,
delta: "Calling the tool",
},
]);
- instrumentation.recordModelCallFinish({
+ instrumentation.recordModelStreamPart({
+ type: "finish",
usage: {
inputTokens: {
total: 1_100,
@@ -1731,9 +1623,7 @@ describe("InAppAgentInstrumentation", () => {
output: 50,
total: 1_150,
}),
- output: {
- messages: [{ role: "assistant", content: "Calling the tool" }],
- },
+ output: { text: "Calling the tool" },
}),
);
expect(generation).not.toHaveProperty("level");
@@ -1747,20 +1637,6 @@ describe("InAppAgentInstrumentation", () => {
}),
);
});
-
- it("ignores step finish events after instrumentation ended", () => {
- const instrumentation = createInstrumentation();
-
- instrumentation.end({});
- instrumentation.recordStepFinish({
- usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
- });
-
- expect(mocks.handler.langfuse.enqueue).not.toHaveBeenCalledWith(
- "generation-create",
- expect.anything(),
- );
- });
});
function createInstrumentation(