Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(assistant): Improve instrumentation (#19152)

Ben Bachem866969b7df79
diff --git a/packages/shared/package.json b/packages/shared/package.json
index 3002a73833..43b58ae294 100644
--- a/packages/shared/package.json
+++ b/packages/shared/package.json
@@ -262,7 +262,6 @@
"ipaddr.js": "^2.2.0",
"jsonpath-plus": "10.3.0",
"kysely": "0.28.17",
- "langfuse": "3.38.4",
"lodash": "^4.18.1",
"lossless-json": "^4.1.1",
"lru-cache": "^11.2.7",
diff --git a/packages/shared/src/server/llm/getInternalTracingHandler.ts b/packages/shared/src/server/llm/getInternalTracingHandler.ts
index 1cc0e873d3..33145bfbd1 100644
--- a/packages/shared/src/server/llm/getInternalTracingHandler.ts
+++ b/packages/shared/src/server/llm/getInternalTracingHandler.ts
@@ -1,193 +1,127 @@
-import { Langfuse } from "langfuse";
-import { env } from "../../env";
-import { ProcessedTraceEvent, TraceSinkParams } from "./types";
-import { buildInternalTraceEventInputs } from "./internalTraceEvents";
-import { processEventBatch } from "../ingestion/processEventBatch";
-import { createUnknownSdkIngestionAttribution } from "../ingestion/ingestionAttribution";
+import { TraceSinkParams } from "./types";
+import {
+ INTERNAL_TRACE_EVENT_SOURCE,
+ type InternalTraceEventInput,
+} from "./internalTraceEvents";
+import type { ObservationType } from "../../domain/observations";
+import type { SkillsAvailableInput } from "../../features/skills/trace";
+import { stringifyValue } from "../../utils/stringChecks";
+import {
+ convertCallsToArrays,
+ convertDefinitionsToMap,
+ extractToolsFromObservation,
+} from "../ingestion/extractToolsBackend";
import { logger } from "../logger";
import { traceException } from "../instrumentation";
import { publishAiFeatureTraceViaOtelIngestion } from "../otel/internalAiFeatureOtelWriter";
-export function prepareInternalTraceEvents(params: {
- events: Array<{
- type: string;
- timestamp: string;
- body: Record<string, unknown>;
- }>;
- environment: string;
- prompt?: TraceSinkParams["prompt"];
-}): ProcessedTraceEvent[] {
- const { events, environment, prompt } = params;
-
- const blockedSpanIds = new Set();
- const blockedSpanNameSubstrings = ["RunnableLambda", "OutputParser"];
-
- for (const event of events) {
- const eventName = "name" in event.body ? event.body.name : "";
-
- if (typeof eventName !== "string" || eventName.length === 0) {
- continue;
- }
-
- if (
- blockedSpanNameSubstrings.some((blockedSubstring) =>
- eventName.includes(blockedSubstring),
- ) &&
- "id" in event.body &&
- event.type !== "trace-create"
- ) {
- blockedSpanIds.add(event.body.id);
- }
- }
-
- return events
- .filter((event) => {
- if ("id" in event.body) {
- return !blockedSpanIds.has(event.body.id);
- }
-
- return true;
- })
- .map((event) => {
- // Inject environment into all events
- return {
- ...event,
- body: {
- ...event.body,
- environment,
- },
- };
- })
- .map((event) => {
- if (event.type === "generation-create" && prompt) {
- return {
- ...event,
- body: {
- ...event.body,
- promptName: prompt.name,
- promptVersion: prompt.version,
- },
- };
- }
-
- return event;
+export type InternalObservation = {
+ id: string;
+ traceId: string;
+ parentObservationId?: string | null;
+ name: string;
+ startTime: Date;
+ endTime: Date;
+ completionStartTime?: Date;
+ input?: unknown;
+ output?: unknown;
+ model?: string;
+ usageDetails?: Record<string, number>;
+ level?: "DEFAULT" | "ERROR";
+ statusMessage?: string | null;
+ metadata?: Record<string, unknown>;
+ promptName?: string;
+ promptVersion?: number;
+ userId?: string;
+ sessionId?: string;
+ tags?: string[];
+ skillsAvailable?: SkillsAvailableInput;
+};
+
+export function getInternalTracingHandler(
+ params: Pick<
+ TraceSinkParams,
+ | "targetProjectId"
+ | "traceId"
+ | "traceName"
+ | "environment"
+ | "userId"
+ | "sessionId"
+ | "metadata"
+ | "prompt"
+ >,
+) {
+ const observations = new Map<string, InternalTraceEventInput>();
+
+ const recordObservation = (
+ type: ObservationType,
+ observation: InternalObservation,
+ ) => {
+ const { toolDefinitions, toolArguments } = extractToolsFromObservation(
+ observation.input,
+ observation.output,
+ );
+ const toolCalls = convertCallsToArrays(toolArguments);
+ observations.set(observation.id, {
+ projectId: params.targetProjectId,
+ traceId: params.traceId,
+ spanId: observation.id,
+ parentSpanId: observation.parentObservationId ?? undefined,
+ name: observation.name,
+ type,
+ environment: params.environment,
+ traceName: params.traceName,
+ startTimeISO: observation.startTime.toISOString(),
+ endTimeISO: observation.endTime.toISOString(),
+ completionStartTime: observation.completionStartTime?.toISOString(),
+ input:
+ observation.input === undefined
+ ? undefined
+ : stringifyValue(observation.input),
+ output:
+ observation.output === undefined
+ ? undefined
+ : stringifyValue(observation.output),
+ modelName: observation.model,
+ providedUsageDetails: observation.usageDetails,
+ level: observation.level ?? "DEFAULT",
+ statusMessage: observation.statusMessage ?? undefined,
+ promptName:
+ observation.promptName ??
+ (type === "GENERATION" ? params.prompt?.name : undefined),
+ promptVersion: (
+ observation.promptVersion ??
+ (type === "GENERATION" ? params.prompt?.version : undefined)
+ )?.toString(),
+ userId: observation.userId ?? params.userId,
+ sessionId: observation.sessionId ?? params.sessionId,
+ tags: observation.tags,
+ metadata: { ...params.metadata, ...observation.metadata },
+ toolDefinitions: convertDefinitionsToMap(toolDefinitions),
+ toolCalls: toolCalls.tool_calls,
+ toolCallNames: toolCalls.tool_call_names,
+ skillsAvailable: observation.skillsAvailable,
+ source: INTERNAL_TRACE_EVENT_SOURCE,
});
-}
-
-export function getInternalTracingHandler(traceSinkParams: TraceSinkParams): {
- handler: { langfuse: Langfuse };
- processTracedEvents: () => Promise<void>;
-} {
- const { prompt, targetProjectId, environment, eventsWriter } =
- traceSinkParams;
- const handler = {
- langfuse: new Langfuse({
- _projectId: targetProjectId,
- _isLocalEventExportEnabled: true,
- environment,
- persistence: "memory",
- sdkIntegration: "LANGCHAIN",
- }),
};
const processTracedEvents = async () => {
+ if (observations.size === 0) return;
+ const eventInputs = [...observations.values()];
+ observations.clear();
try {
- const events = await handler.langfuse._exportLocalEvents(
- traceSinkParams.targetProjectId,
- );
- const processedEvents = prepareInternalTraceEvents({
- events,
- environment,
- prompt,
+ await publishAiFeatureTraceViaOtelIngestion({
+ eventInputs,
+ projectId: params.targetProjectId,
});
-
- const useAiFeatureOtel =
- Boolean(traceSinkParams.aiFeatureOtelIngestion) &&
- env.LANGFUSE_MIGRATION_V4_WRITE_MODE !== "legacy";
-
- if (useAiFeatureOtel) {
- try {
- const { eventInputs } = buildInternalTraceEventInputs({
- processedEvents,
- traceId: traceSinkParams.traceId,
- projectId: targetProjectId,
- userId: traceSinkParams.userId,
- sessionId: traceSinkParams.sessionId,
- });
-
- if (eventInputs.length > 0) {
- await publishAiFeatureTraceViaOtelIngestion({
- eventInputs,
- projectId: targetProjectId,
- });
- }
- } catch (otelError) {
- traceException(otelError);
- logger.error(
- "Failed to process AI-feature traces via OTel ingestion",
- {
- error: otelError,
- },
- );
- }
-
- return;
- }
-
- // Legacy write to traces/observations tables
- try {
- const auth = {
- validKey: true as const,
- scope: {
- projectId: traceSinkParams.targetProjectId, // Important: this controls into what project traces are ingested.
- accessLevel: "project",
- } as any,
- };
-
- await processEventBatch(
- JSON.parse(JSON.stringify(processedEvents)), // stringify to emulate network event batch from network call
- auth,
- {
- isLangfuseInternal: true,
- forwardToEventsTable: eventsWriter ? false : undefined, // Do not dual write when we already direct event write
- attribution: createUnknownSdkIngestionAttribution({
- authCheck: auth,
- }),
- },
- );
- } catch (processingError) {
- traceException(processingError);
- logger.error("Failed to process traced events via legacy ingestion", {
- error: processingError,
- });
- }
-
- // Direct write to events table
- if (eventsWriter) {
- try {
- const { rootSpanId, eventInputs } = buildInternalTraceEventInputs({
- processedEvents,
- traceId: traceSinkParams.traceId,
- projectId: targetProjectId,
- userId: traceSinkParams.userId,
- sessionId: traceSinkParams.sessionId,
- experimentContext: eventsWriter.experimentContext,
- });
-
- if (eventInputs.length > 0) {
- await eventsWriter.write({ rootSpanId, eventInputs });
- }
- } catch (writeError) {
- traceException(writeError);
- logger.error("Failed to direct-write internal traced events", {
- error: writeError,
- });
- }
- }
- } catch (e) {
- logger.error("Failed to process traced events", { error: e });
+ } catch (error) {
+ traceException(error);
+ logger.error(
+ "Failed to process AI-feature observations via OTel ingestion",
+ { error },
+ );
}
};
- return { handler, processTracedEvents };
+ return { recordObservation, processTracedEvents };
}
diff --git a/packages/shared/src/server/otel/internalAiFeatureOtelWriter.ts b/packages/shared/src/server/otel/internalAiFeatureOtelWriter.ts
index 31d06623f6..84fb6101e2 100644
--- a/packages/shared/src/server/otel/internalAiFeatureOtelWriter.ts
+++ b/packages/shared/src/server/otel/internalAiFeatureOtelWriter.ts
@@ -43,12 +43,6 @@ function attributesToOtelList(
.filter((attribute): attribute is OtelAttribute => Boolean(attribute));
}
-/**
- * Keep the SDK `trace-create` span (id === trace id). Legacy ingestion stored
- * that as a TRACE row; event propagation then materializes it as the wrapping
- * `t-{traceId}` SPAN with conversation input/output. Emitting it here gives
- * OTel-native assistant traces the same root span above the AGENT turn.
- */
function aiFeatureTraceInputsToResourceSpans(
eventInputs: InternalTraceEventInput[],
): ResourceSpan[] {
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 6552286393..1c7b544770 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -448,9 +448,6 @@ importers:
kysely:
specifier: 0.28.17
version: 0.28.17
- langfuse:
- specifier: 3.38.4
- version: 3.38.4
lodash:
specifier: ^4.18.1
version: 4.18.1
diff --git a/worker/src/__tests__/getInternalTracingHandler.test.ts b/worker/src/__tests__/getInternalTracingHandler.test.ts
deleted file mode 100644
index e894658b02..0000000000
--- a/worker/src/__tests__/getInternalTracingHandler.test.ts
+++ /dev/null
@@ -1,159 +0,0 @@
-import { describe, expect, it } from "vitest";
-
-import {
- LangfuseInternalTraceEnvironment,
- prepareInternalTraceEvents,
-} from "@langfuse/shared/src/server";
-
-describe("prepareInternalTraceEvents", () => {
- it("sets the configured environment on every forwarded event", () => {
- const preparedEvents = prepareInternalTraceEvents({
- events: [
- {
- type: "trace-create",
- timestamp: new Date().toISOString(),
- body: {
- id: "trace-123",
- name: "internal-trace",
- environment: "wrong-environment",
- },
- },
- {
- type: "span-create",
- timestamp: new Date().toISOString(),
- body: {
- id: "blocked-span",
- traceId: "trace-123",
- name: "RunnableLambda",
- environment: "wrong-environment",
- },
- },
- {
- type: "span-update",
- timestamp: new Date().toISOString(),
- body: {
- id: "blocked-span",
- traceId: "trace-123",
- },
- },
- {
- type: "generation-create",
- timestamp: new Date().toISOString(),
- body: {
- id: "generation-123",
- traceId: "trace-123",
- name: "ChatOpenAI",
- },
- },
- {
- type: "generation-update",
- timestamp: new Date().toISOString(),
- body: {
- id: "generation-123",
- traceId: "trace-123",
- output: {
- role: "assistant",
- content: "Berlin",
- },
- },
- },
- {
- type: "span-create",
- timestamp: new Date().toISOString(),
- body: {
- id: "allowed-span",
- traceId: "trace-123",
- name: "Retriever",
- environment: "another-environment",
- },
- },
- {
- type: "span-update",
- timestamp: new Date().toISOString(),
- body: {
- id: "allowed-span",
- traceId: "trace-123",
- },
- },
- ],
- environment: LangfuseInternalTraceEnvironment.PromptExperiments,
- });
-
- expect(preparedEvents.map((event) => event.body.id)).not.toContain(
- "blocked-span",
- );
- expect(preparedEvents).toHaveLength(5);
- expect(
- preparedEvents.every(
- (event) =>
- event.body.environment ===
- LangfuseInternalTraceEnvironment.PromptExperiments,
- ),
- ).toBe(true);
- });
-
- it("adds prompt metadata only to generation-create events", () => {
- const preparedEvents = prepareInternalTraceEvents({
- events: [
- {
- type: "trace-create",
- timestamp: new Date().toISOString(),
- body: {
- id: "trace-123",
- name: "internal-trace",
- },
- },
- {
- type: "generation-create",
- timestamp: new Date().toISOString(),
- body: {
- id: "generation-123",
- traceId: "trace-123",
- name: "ChatOpenAI",
- },
- },
- {
- type: "generation-update",
- timestamp: new Date().toISOString(),
- body: {
- id: "generation-123",
- traceId: "trace-123",
- },
- },
- ],
- environment: LangfuseInternalTraceEnvironment.PromptExperiments,
- prompt: {
- name: "internal-prompt",
- version: 3,
- },
- });
-
- expect(preparedEvents).toEqual(
- expect.arrayContaining([
- expect.objectContaining({
- type: "generation-create",
- body: expect.objectContaining({
- id: "generation-123",
- environment: LangfuseInternalTraceEnvironment.PromptExperiments,
- promptName: "internal-prompt",
- promptVersion: 3,
- }),
- }),
- expect.objectContaining({
- type: "generation-update",
- body: expect.not.objectContaining({
- promptName: "internal-prompt",
- promptVersion: 3,
- }),
- }),
- expect.objectContaining({
- type: "trace-create",
- body: expect.not.objectContaining({
- promptName: "internal-prompt",
- promptVersion: 3,
- }),
- }),
- ]),
- );
- });
-});
diff --git a/worker/src/features/experiments/__tests__/scheduleExperimentEvals.test.ts b/worker/src/features/experiments/__tests__/scheduleExperimentEvals.test.ts
index dbc33a1a58..7b8521f902 100644
--- a/worker/src/features/experiments/__tests__/scheduleExperimentEvals.test.ts
+++ b/worker/src/features/experiments/__tests__/scheduleExperimentEvals.test.ts
@@ -6,7 +6,6 @@ import {
import {
buildInternalTraceEventInputs,
materializeInternalTrace,
- prepareInternalTraceEvents,
} from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
import type { ObservationForEval } from "../../evaluation/observationEval";
@@ -356,11 +355,18 @@ const realExperimentEvents = [
];
function getProcessedExperimentEvents() {
- return prepareInternalTraceEvents({
- events: realExperimentEvents as any,
- environment: "langfuse-prompt-experiment",
- prompt,
- });
+ return realExperimentEvents
+ .filter((event) => event.body.id !== blockedParserSpanId)
+ .map((event) => ({
+ ...event,
+ body: {
+ ...event.body,
+ environment: LangfuseInternalTraceEnvironment.PromptExperiments,
+ ...(event.type === "generation-create"
+ ? { promptName: prompt.name, promptVersion: prompt.version }
+ : {}),
+ },
+ }));
}
function getExperimentContext() {
@@ -385,16 +391,13 @@ describe("prompt experiment direct-write materialization", () => {
vi.clearAllMocks();
});
- it("should filter parser spans and use the trace root as the experiment root", () => {
+ it("should use the trace root as the experiment root", () => {
const processedEvents = getProcessedExperimentEvents();
const { rootSpanId, snapshots } = materializeInternalTrace({
processedEvents,
traceId,
});
- expect(processedEvents.map((event) => event.body.id)).not.toContain(
- blockedParserSpanId,
- );
expect(rootSpanId).toBe(traceId);
expect(snapshots).toHaveLength(2);
diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.ts
index 6d82a03053..9ba99b842d 100644
--- a/worker/src/features/in-app-agent/executeInAppAgentRun.ts
+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.ts
@@ -631,6 +631,7 @@ export async function executeInAppAgentRun(params: {
projectId,
conversationId: conversation.id,
runId,
+ runStartedAt: run.claimedAt ?? run.createdAt,
user: {
id: run.triggeredByUserId,
email: access.email,
@@ -805,6 +806,7 @@ function buildTracingConfig(params: {
projectId: string;
conversationId: string;
runId: string;
+ runStartedAt: Date;
user: {
id: string;
email: string | null;
@@ -838,6 +840,7 @@ function buildTracingConfig(params: {
targetProjectId: traceSinkParams.targetProjectId,
environment: traceSinkParams.environment,
runId: params.runId,
+ runStartedAt: params.runStartedAt,
user: {
id: params.user.id,
email: params.user.email,
diff --git a/worker/src/features/in-app-agent/runtime/instrumentation.test.ts b/worker/src/features/in-app-agent/runtime/instrumentation.test.ts
index 79fe17c38b..911daa3c44 100644
--- a/worker/src/features/in-app-agent/runtime/instrumentation.test.ts
+++ b/worker/src/features/in-app-agent/runtime/instrumentation.test.ts
@@ -1,37 +1,33 @@
import { EventType } from "@ag-ui/core";
import { beforeEach, describe, expect, it, vi } from "vitest";
+import { assembleTranscript } from "../../../../../packages/shared/src/server/transcript/transcript";
+import { orderObservations } from "../../../../../packages/shared/src/server/transcript/ordering";
+import type { TranscriptObservation } from "../../../../../packages/shared/src/server/transcript/types";
import {
getInAppAgentInstrumentationObservationId,
getInAppAgentInstrumentationTraceId,
getInAppAgentLlmCallObservationId,
IN_APP_AGENT_TOOL_APPROVAL_EVENT_NAME,
+ IN_APP_AGENT_SILENT_MCP_OUTPUT_TYPE,
} from "@langfuse/shared/in-app-agent";
import type { AgUiRunAgentInput } from "./types";
-import { InAppAgentInstrumentation } from "./instrumentation";
+import {
+ createInAppAgentInstrumentation,
+ InAppAgentInstrumentation,
+} from "./instrumentation";
const runId = "run-1";
const traceId = getInAppAgentInstrumentationTraceId(runId);
const agentRunObservationId = getInAppAgentInstrumentationObservationId(runId);
const mocks = vi.hoisted(() => {
- const trace = {
- generation: vi.fn(),
- update: vi.fn(),
- };
- const handler = {
- langfuse: {
- trace: vi.fn(() => trace),
- enqueue: vi.fn(),
- },
- };
-
+ const recordObservation = vi.fn();
return {
- trace,
- handler,
+ recordObservation,
processTracedEvents: vi.fn(async () => undefined),
getInternalTracingHandler: vi.fn(() => ({
- handler,
+ recordObservation,
processTracedEvents: vi.fn(async () => undefined),
})),
};
@@ -64,7 +60,6 @@ const input: AgUiRunAgentInput = {
const expectedAgentRunInput = {
messages: [{ role: "user", content: "hello" }],
};
-const expectedWrappingRootInput = "hello";
describe("InAppAgentInstrumentation", () => {
beforeEach(() => {
@@ -143,17 +138,8 @@ describe("InAppAgentInstrumentation", () => {
sessionId: "conversation-1",
}),
);
- expect(mocks.handler.langfuse.trace).toHaveBeenCalledWith(
- expect.objectContaining({ id: traceId, name: "agent-turn" }),
- );
- expect(mocks.handler.langfuse.trace).toHaveBeenCalledWith(
- expect.objectContaining({
- input: expectedWrappingRootInput,
- }),
- );
- expect(mocks.trace.generation).not.toHaveBeenCalled();
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "tool-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "TOOL",
expect.objectContaining({
id: "tool-1",
traceId,
@@ -162,7 +148,12 @@ describe("InAppAgentInstrumentation", () => {
userId: "user-1",
sessionId: "conversation-1",
input: { limit: 10 },
- output: toolOutput,
+ output: {
+ role: "tool",
+ name: "listObservations",
+ tool_call_id: "tool-1",
+ content: toolOutput,
+ },
startTime: expect.any(Date),
endTime: expect.any(Date),
completionStartTime: expect.any(Date),
@@ -173,12 +164,11 @@ describe("InAppAgentInstrumentation", () => {
}),
);
expect(
- mocks.handler.langfuse.enqueue.mock.calls.find(
- ([type]) => type === "tool-create",
- )?.[1].metadata,
+ mocks.recordObservation.mock.calls.find(([type]) => type === "TOOL")?.[1]
+ .metadata,
).not.toHaveProperty("toolCallApproval");
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "agent-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "AGENT",
expect.objectContaining({
id: agentRunObservationId,
traceId,
@@ -195,7 +185,7 @@ describe("InAppAgentInstrumentation", () => {
{
role: "tool",
tool_call_id: "tool-1",
- content: toolOutput,
+ content: "Output recorded in TOOL observation tool-1.",
},
],
text: "hi there",
@@ -203,21 +193,463 @@ describe("InAppAgentInstrumentation", () => {
},
}),
);
- const agentCreateBody = mocks.handler.langfuse.enqueue.mock.calls.find(
- ([type]) => type === "agent-create",
+ const agentCreateBody = mocks.recordObservation.mock.calls.find(
+ ([type]) => type === "AGENT",
)?.[1];
expect(agentCreateBody).not.toHaveProperty("usageDetails");
expect(agentCreateBody).not.toHaveProperty("model");
- expect(mocks.handler.langfuse.enqueue).not.toHaveBeenCalledWith(
- "span-create",
+ expect(mocks.recordObservation).not.toHaveBeenCalledWith(
+ "SPAN",
expect.anything(),
);
- expect(mocks.trace.update).toHaveBeenCalledWith(
- expect.objectContaining({
- input: expectedWrappingRootInput,
- output: "hi there",
- }),
+ });
+
+ it.each([false, true])(
+ "retains full TOOL output while bounding root duplicates (silent: %s)",
+ (silent) => {
+ const instrumentation = createInstrumentation();
+ const result = { data: '😀"\\'.repeat(100_000) };
+ const output = silent
+ ? {
+ type: IN_APP_AGENT_SILENT_MCP_OUTPUT_TYPE,
+ output: result,
+ toolCallId: "tool-1",
+ toolName: "listObservations",
+ }
+ : result;
+ instrumentation.recordEvents([
+ {
+ type: EventType.TOOL_CALL_START,
+ toolCallId: "tool-1",
+ toolCallName: "listObservations",
+ },
+ { type: EventType.TOOL_CALL_ARGS, toolCallId: "tool-1", delta: "{}" },
+ { type: EventType.TOOL_CALL_END, toolCallId: "tool-1" },
+ {
+ type: EventType.TOOL_CALL_RESULT,
+ toolCallId: "tool-1",
+ content: JSON.stringify(output),
+ },
+ { type: EventType.TEXT_MESSAGE_CONTENT, delta: "Done" },
+ ]);
+ instrumentation.end({});
+
+ const tool = mocks.recordObservation.mock.calls.find(
+ ([type]) => type === "TOOL",
+ )?.[1];
+ const root = mocks.recordObservation.mock.calls
+ .filter(([type]) => type === "AGENT")
+ .at(-1)?.[1];
+ expect(tool.output).toEqual({
+ role: "tool",
+ name: "listObservations",
+ tool_call_id: tool.id,
+ content: output,
+ });
+ expect(root.output.messages).toEqual([
+ {
+ role: "assistant",
+ content: "",
+ tool_calls: [
+ {
+ id: "tool-1",
+ name: "listObservations",
+ arguments: "{}",
+ type: "function",
+ },
+ ],
+ },
+ {
+ role: "tool",
+ tool_call_id: "tool-1",
+ content: "Output recorded in TOOL observation tool-1.",
+ },
+ { role: "assistant", content: "Done" },
+ ]);
+ expect(Buffer.byteLength(JSON.stringify(root.output))).toBeLessThan(1024);
+ },
+ );
+
+ it("preserves supported media in structured tool responses", () => {
+ const instrumentation = createInstrumentation();
+ const content = [
+ { type: "text", text: "Preview" },
+ {
+ type: "image_url",
+ image_url: { url: "https://example.com/preview.png" },
+ },
+ ];
+ instrumentation.recordModelCallStart({
+ prompt: [{ role: "user", content: "Preview" }],
+ });
+ instrumentation.recordModelStreamPart({
+ type: "tool-call",
+ toolCallId: "media-tool",
+ toolName: "preview",
+ input: "{}",
+ });
+ instrumentation.recordModelStreamPart({
+ type: "finish",
+ finishReason: "tool-calls",
+ });
+ instrumentation.recordEvents([
+ {
+ type: EventType.TOOL_CALL_START,
+ toolCallId: "media-tool",
+ toolCallName: "preview",
+ },
+ { type: EventType.TOOL_CALL_END, toolCallId: "media-tool" },
+ {
+ type: EventType.TOOL_CALL_RESULT,
+ toolCallId: "media-tool",
+ content: JSON.stringify(content),
+ },
+ ]);
+ instrumentation.end({});
+ const observations: TranscriptObservation[] =
+ mocks.recordObservation.mock.calls
+ .filter(([type]) => type === "GENERATION" || type === "TOOL")
+ .map(([type, body]) => ({ ...body, type, metadata: {} }));
+ expect(observations.find(({ type }) => type === "TOOL")?.output).toEqual({
+ role: "tool",
+ name: "preview",
+ tool_call_id: "media-tool",
+ content,
+ });
+ const transcript = assembleTranscript(orderObservations(observations));
+ expect(JSON.stringify(transcript)).toContain(
+ "https://example.com/preview.png",
+ );
+ expect(JSON.stringify(transcript)).toContain('"mediaType":"image/*"');
+ });
+
+ it("references every root tool result, including small results", () => {
+ const instrumentation = createInstrumentation();
+ const output = { data: "x".repeat(6000) };
+ for (let index = 0; index < 40; index++) {
+ const toolCallId = `tool-${index}`;
+ instrumentation.recordEvents([
+ {
+ type: EventType.TOOL_CALL_START,
+ toolCallId,
+ toolCallName: "listObservations",
+ },
+ { type: EventType.TOOL_CALL_ARGS, toolCallId, delta: "{}" },
+ { type: EventType.TOOL_CALL_END, toolCallId },
+ {
+ type: EventType.TOOL_CALL_RESULT,
+ toolCallId,
+ content: JSON.stringify(output),
+ },
+ ]);
+ }
+ instrumentation.end({});
+
+ const tools = mocks.recordObservation.mock.calls.filter(
+ ([type]) => type === "TOOL",
+ );
+ expect(tools).toHaveLength(40);
+ for (const [, tool] of tools) {
+ expect(tool.output).toEqual({
+ role: "tool",
+ name: "listObservations",
+ tool_call_id: tool.id,
+ content: output,
+ });
+ }
+ const root = mocks.recordObservation.mock.calls
+ .filter(([type]) => type === "AGENT")
+ .at(-1)?.[1];
+ expect(root.output.messages).toHaveLength(80);
+ expect(root.output.tool_calls).toHaveLength(40);
+ expect(
+ root.output.messages.filter(
+ (message: { role: string }) => message.role === "tool",
+ ),
+ ).toEqual(
+ Array.from({ length: 40 }, (_, index) => ({
+ role: "tool",
+ tool_call_id: `tool-${index}`,
+ content: `Output recorded in TOOL observation tool-${index}.`,
+ })),
+ );
+ expect(JSON.stringify(root.output)).not.toContain(output.data);
+ });
+
+ it("keeps transcript continuity with full TOOL outputs and referenced generation results", () => {
+ const instrumentation = createInstrumentation();
+ const user = { role: "user", content: [{ type: "text", text: "hello" }] };
+ const call = {
+ role: "assistant",
+ content: [
+ {
+ type: "tool-call",
+ toolCallId: "tool-1",
+ toolName: "listObservations",
+ input: {},
+ },
+ ],
+ };
+ const toolOutput = { type: "json", value: { data: "x".repeat(200_000) } };
+ const result = {
+ role: "tool",
+ content: [
+ {
+ type: "tool-result",
+ toolCallId: "tool-1",
+ toolName: "listObservations",
+ output: toolOutput,
+ providerOptions: { mastra: { modelOutput: toolOutput } },
+ },
+ ],
+ };
+ instrumentation.recordModelCallStart({ prompt: [user] });
+ instrumentation.recordModelStreamPart({
+ type: "tool-call",
+ toolCallId: "tool-1",
+ toolName: "listObservations",
+ input: "{}",
+ });
+ instrumentation.recordModelStreamPart({
+ type: "finish",
+ finishReason: "tool-calls",
+ });
+ instrumentation.recordEvents([
+ {
+ type: EventType.TOOL_CALL_START,
+ toolCallId: "tool-1",
+ toolCallName: "listObservations",
+ },
+ { type: EventType.TOOL_CALL_ARGS, toolCallId: "tool-1", delta: "{}" },
+ { type: EventType.TOOL_CALL_END, toolCallId: "tool-1" },
+ {
+ type: EventType.TOOL_CALL_RESULT,
+ toolCallId: "tool-1",
+ content: JSON.stringify({ data: "x".repeat(200_000) }),
+ },
+ ]);
+ const providerPrompt = [user, call, result];
+ const originalProviderPrompt = JSON.stringify(providerPrompt);
+ instrumentation.recordModelCallStart({ prompt: providerPrompt });
+ expect(JSON.stringify(providerPrompt)).toBe(originalProviderPrompt);
+ instrumentation.recordModelStreamPart({
+ type: "text-delta",
+ delta: "Done",
+ });
+ instrumentation.recordModelStreamPart({
+ type: "finish",
+ finishReason: "stop",
+ });
+ instrumentation.end({});
+
+ const observations: TranscriptObservation[] =
+ mocks.recordObservation.mock.calls
+ .filter(([type]) => type === "GENERATION" || type === "TOOL")
+ .map(([type, body]) => ({ ...body, type }));
+ const generations = observations.filter(
+ ({ type }) => type === "GENERATION",
+ );
+ const referencedResult = {
+ role: "tool",
+ content: [
+ {
+ type: "tool-result",
+ toolCallId: "tool-1",
+ toolName: "listObservations",
+ output: {
+ type: "text",
+ value: "Output recorded in TOOL observation tool-1.",
+ },
+ },
+ ],
+ };
+ expect(generations[1]?.input).toEqual({
+ messages: [user, call, referencedResult],
+ });
+ const nextTurn: TranscriptObservation = {
+ ...generations[1]!,
+ id: "next-generation",
+ traceId: "next-trace",
+ parentObservationId: null,
+ startTime: new Date(Date.now() + 1000),
+ input: {
+ messages: [
+ user,
+ call,
+ referencedResult,
+ { role: "assistant", content: "Done" },
+ { role: "user", content: "Thanks" },
+ ],
+ },
+ output: { text: "Welcome" },
+ };
+ const transcript = assembleTranscript(
+ orderObservations([...observations, nextTurn]),
);
+ expect(transcript?.threads).toHaveLength(1);
+ expect(
+ transcript?.threads[0]?.currentTurn.messages.map(({ role }) => role),
+ ).toEqual(["user", "assistant"]);
+ expect(
+ transcript?.threads[0]?.conversationHistory.filter(
+ ({ role }) => role === "tool",
+ ),
+ ).toHaveLength(1);
+ expect(
+ JSON.stringify(transcript?.threads[0]?.conversationHistory),
+ ).toContain(toolOutput.value.data);
+ });
+
+ it("pairs parallel filter results by ID without metadata or duplicate history results", () => {
+ const instrumentation = createInstrumentation();
+ const name = "langfuse_getObservationFilterValues";
+ const calls = ["userId", "sessionId"].map((column, index) => ({
+ toolCallId: `tooluse_sanitized_${index}`,
+ toolName: name,
+ input: JSON.stringify({ column }),
+ result: { type: "VALUES", column, values: [], meta: {} },
+ }));
+ const user = {
+ role: "user",
+ content: "List available user and session IDs",
+ };
+ instrumentation.recordModelCallStart({ prompt: [user] });
+ for (const call of calls) {
+ instrumentation.recordModelStreamPart({ type: "tool-call", ...call });
+ instrumentation.recordEvents([
+ {
+ type: EventType.TOOL_CALL_START,
+ toolCallId: call.toolCallId,
+ toolCallName: name,
+ },
+ {
+ type: EventType.TOOL_CALL_ARGS,
+ toolCallId: call.toolCallId,
+ delta: call.input,
+ },
+ { type: EventType.TOOL_CALL_END, toolCallId: call.toolCallId },
+ ]);
+ }
+ instrumentation.recordModelStreamPart({
+ type: "finish",
+ finishReason: "tool-calls",
+ });
+ // Name-based FIFO matching would swap these responses.
+ for (const call of [...calls].reverse()) {
+ instrumentation.recordEvents([
+ {
+ type: EventType.TOOL_CALL_RESULT,
+ toolCallId: call.toolCallId,
+ content: JSON.stringify(call.result),
+ },
+ ]);
+ }
+ instrumentation.recordModelCallStart({
+ prompt: [
+ user,
+ {
+ role: "assistant",
+ content: calls.map((call) => ({
+ type: "tool-call",
+ toolCallId: call.toolCallId,
+ toolName: name,
+ input: JSON.parse(call.input),
+ })),
+ },
+ {
+ role: "tool",
+ content: calls.map((call) => ({
+ type: "tool-result",
+ toolCallId: call.toolCallId,
+ toolName: name,
+ output: { type: "json", value: call.result },
+ })),
+ },
+ ],
+ });
+ instrumentation.recordModelStreamPart({
+ type: "text-delta",
+ delta: "No values found",
+ });
+ instrumentation.recordModelStreamPart({
+ type: "finish",
+ finishReason: "stop",
+ });
+ instrumentation.end({});
+
+ const observations: TranscriptObservation[] =
+ mocks.recordObservation.mock.calls
+ .filter(([type]) => type === "GENERATION" || type === "TOOL")
+ .map(([type, body]) => ({ ...body, type, metadata: {} }));
+ for (const call of calls) {
+ expect(
+ observations.find(({ id }) => id === call.toolCallId)?.output,
+ ).toEqual({
+ role: "tool",
+ name,
+ tool_call_id: call.toolCallId,
+ content: call.result,
+ });
+ }
+ const transcript = assembleTranscript(orderObservations(observations));
+ expect(transcript?.threads).toHaveLength(1);
+ const messages = transcript!.threads[0]!.currentTurn.messages;
+ const results = messages
+ .flatMap(({ parts }) => parts)
+ .filter((part) => part.type === "tool-result");
+ expect(results).toHaveLength(2);
+ for (const call of calls) {
+ expect(
+ results.find((part) => part.toolCallId === call.toolCallId),
+ ).toMatchObject({
+ toolName: name,
+ output: { type: "json", value: call.result },
+ });
+ }
+ });
+
+ it("keeps batched tool-result identities but omits their payloads from generation telemetry", () => {
+ const instrumentation = createInstrumentation();
+ const results = ["tool-1", "tool-2"].map((toolCallId) => ({
+ type: "tool-result",
+ toolCallId,
+ toolName: "listObservations",
+ output: { type: "json", value: { privatePayload: toolCallId } },
+ providerOptions: {
+ mastra: { modelOutput: { privatePayload: toolCallId } },
+ },
+ }));
+ const prompt = [
+ { role: "user", content: [{ type: "text", text: "hello" }] },
+ { role: "tool", content: results },
+ ];
+ const originalPrompt = JSON.stringify(prompt);
+ instrumentation.recordModelCallStart({ prompt });
+ instrumentation.recordModelStreamPart({
+ type: "finish",
+ finishReason: "stop",
+ });
+
+ const generation = mocks.recordObservation.mock.calls.find(
+ ([type]) => type === "GENERATION",
+ )?.[1];
+ expect(generation.input.messages).toEqual([
+ prompt[0],
+ {
+ role: "tool",
+ content: ["tool-1", "tool-2"].map((toolCallId) => ({
+ type: "tool-result",
+ toolCallId,
+ toolName: "listObservations",
+ output: {
+ type: "text",
+ value: `Output recorded in TOOL observation ${toolCallId}.`,
+ },
+ })),
+ },
+ ]);
+ expect(JSON.stringify(generation.input)).not.toContain("privatePayload");
+ expect(JSON.stringify(prompt)).toBe(originalPrompt);
});
it("records approved manual tool approval metadata", () => {
@@ -250,8 +682,8 @@ describe("InAppAgentInstrumentation", () => {
},
]);
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "tool-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "TOOL",
expect.objectContaining({
metadata: expect.objectContaining({
toolCallId: "tool-1",
@@ -297,8 +729,8 @@ describe("InAppAgentInstrumentation", () => {
},
]);
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "tool-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "TOOL",
expect.objectContaining({
metadata: expect.objectContaining({
toolCallId: "tool-1",
@@ -345,8 +777,8 @@ describe("InAppAgentInstrumentation", () => {
},
]);
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "tool-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "TOOL",
expect.objectContaining({
startTime: rejectedAt,
endTime: rejectedAt,
@@ -385,8 +817,8 @@ describe("InAppAgentInstrumentation", () => {
]);
instrumentation.endWithError(new Error("agent failed"));
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "tool-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "TOOL",
expect.objectContaining({
input: '{"traceId":',
level: "ERROR",
@@ -397,8 +829,8 @@ describe("InAppAgentInstrumentation", () => {
}),
}),
);
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "agent-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "AGENT",
expect.objectContaining({
id: agentRunObservationId,
name: "agent-turn",
@@ -453,8 +885,8 @@ describe("InAppAgentInstrumentation", () => {
]);
instrumentation.end();
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "agent-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "AGENT",
expect.objectContaining({
name: "agent-turn",
input: {
@@ -529,16 +961,18 @@ describe("InAppAgentInstrumentation", () => {
},
]);
- const toolCreateBody = mocks.handler.langfuse.enqueue.mock.calls.find(
- ([type]) => type === "tool-create",
+ const toolCreateBody = mocks.recordObservation.mock.calls.find(
+ ([type]) => type === "TOOL",
)?.[1];
expect(toolCreateBody).toEqual(
expect.objectContaining({
input: { type: "TRACE" },
output: {
- error: true,
- message: "Tool input validation failed",
+ role: "tool",
+ name: "listObservations",
+ tool_call_id: "tool-1",
+ content: { error: true, message: "Tool input validation failed" },
},
level: "ERROR",
statusMessage: "Tool input validation failed",
@@ -546,6 +980,63 @@ describe("InAppAgentInstrumentation", () => {
);
});
+ it("preserves the root replacement key across approval continuations", () => {
+ const runStartedAt = new Date("2026-08-14T10:00:00.123Z");
+ const tracing = {
+ runId,
+ runStartedAt,
+ targetProjectId: "project-1",
+ environment: "production",
+ metadata: {},
+ user: { id: "user-1", isAdmin: false },
+ };
+ const initial = createInAppAgentInstrumentation({ input, tracing });
+ expect(initial).toBeDefined();
+ initial!.end();
+
+ const continuationRunId = "continuation-run-1";
+ const continuation = createInAppAgentInstrumentation({
+ input: {
+ ...input,
+ runId: continuationRunId,
+ forwardedProps: {
+ command: {
+ resume: {
+ approved: true,
+ rootRunId: runId,
+ traceStartedAt: runStartedAt.toISOString(),
+ approvalRequest: {
+ type: "tool_approval_request",
+ toolCallId: "tool-1",
+ toolName: "langfuse_createTextPrompt",
+ runId,
+ },
+ },
+ },
+ },
+ },
+ tracing: {
+ ...tracing,
+ runId: continuationRunId,
+ runStartedAt: new Date("2026-08-14T10:02:00.456Z"),
+ },
+ });
+ expect(continuation).toBeDefined();
+ continuation!.end();
+
+ const roots = mocks.recordObservation.mock.calls.filter(
+ ([type]) => type === "AGENT",
+ );
+ expect(roots).toHaveLength(4);
+ for (const [, root] of roots) {
+ expect(root).toMatchObject({
+ id: agentRunObservationId,
+ traceId,
+ startTime: runStartedAt,
+ });
+ }
+ });
+
it.each([
{
name: "top-level event errors",
@@ -622,10 +1113,15 @@ describe("InAppAgentInstrumentation", () => {
},
]);
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "tool-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "TOOL",
expect.objectContaining({
- output: expectedOutput,
+ output: {
+ role: "tool",
+ name: "createScoreConfig",
+ tool_call_id: "tool-1",
+ content: expectedOutput,
+ },
level: "ERROR",
statusMessage: expectedMessage,
}),
@@ -684,12 +1180,17 @@ describe("InAppAgentInstrumentation", () => {
},
]);
- const toolCreateBody = mocks.handler.langfuse.enqueue.mock.calls.find(
- ([type]) => type === "tool-create",
+ const toolCreateBody = mocks.recordObservation.mock.calls.find(
+ ([type]) => type === "TOOL",
)?.[1];
expect(toolCreateBody).toEqual(
expect.objectContaining({
- output: expectedOutput,
+ output: {
+ role: "tool",
+ name: "search",
+ tool_call_id: "tool-1",
+ content: expectedOutput,
+ },
}),
);
expect(toolCreateBody).not.toHaveProperty("level");
@@ -719,24 +1220,11 @@ describe("InAppAgentInstrumentation", () => {
},
}),
);
- expect(mocks.handler.langfuse.trace).toHaveBeenCalledWith(
- expect.objectContaining({
- metadata: {
- langfuse_project_id: "project-1",
- langfuse_user_email: "[email protected]",
- langfuse_user_project_role: "ADMIN",
- langfuse_user_is_admin: true,
- prompt_name: "in-app-agent-system-prompt",
- prompt_version: 3,
- },
- }),
- );
- expect(mocks.trace.generation).not.toHaveBeenCalled();
instrumentation.end();
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "agent-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "AGENT",
expect.objectContaining({
id: agentRunObservationId,
name: "agent-turn",
@@ -760,8 +1248,8 @@ describe("InAppAgentInstrumentation", () => {
},
]);
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "agent-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "AGENT",
expect.objectContaining({
name: "agent-turn",
input: expectedAgentRunInput,
@@ -771,12 +1259,6 @@ describe("InAppAgentInstrumentation", () => {
},
}),
);
- expect(mocks.trace.update).toHaveBeenCalledWith(
- expect.objectContaining({
- input: expectedWrappingRootInput,
- output: "second turn output",
- }),
- );
});
it("records AG-UI context in the agent turn input", () => {
@@ -798,14 +1280,8 @@ describe("InAppAgentInstrumentation", () => {
],
});
- expect(mocks.trace.generation).not.toHaveBeenCalled();
- expect(mocks.handler.langfuse.trace).toHaveBeenCalledWith(
- expect.objectContaining({
- input: expectedWrappingRootInput,
- }),
- );
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "agent-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "AGENT",
expect.objectContaining({
input: {
messages: [{ role: "user", content: "hello" }],
@@ -863,14 +1339,8 @@ describe("InAppAgentInstrumentation", () => {
],
});
- expect(mocks.handler.langfuse.trace).toHaveBeenCalledWith(
- expect.objectContaining({
- name: "agent-turn",
- input: expectedWrappingRootInput,
- }),
- );
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "agent-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "AGENT",
expect.objectContaining({
input: {
messages: [{ role: "user", content: "hello" }],
@@ -952,7 +1422,7 @@ describe("InAppAgentInstrumentation", () => {
{
role: "tool",
tool_call_id: "tool-1",
- content: { name: "FOO", version: 1 },
+ content: "Output recorded in the corresponding TOOL observation.",
},
],
});
@@ -962,7 +1432,7 @@ describe("InAppAgentInstrumentation", () => {
});
instrumentation.end({});
- for (const [, observation] of mocks.handler.langfuse.enqueue.mock.calls) {
+ for (const [, observation] of mocks.recordObservation.mock.calls) {
expect(observation).toMatchObject({
userId: "user-1",
sessionId: "conversation-1",
@@ -990,21 +1460,8 @@ describe("InAppAgentInstrumentation", () => {
approval_decided_at: approvalDecidedAt,
};
- expect(mocks.handler.langfuse.trace).toHaveBeenCalledWith(
- expect.objectContaining({
- id: parentTraceId,
- name: "agent-turn",
- sessionId: "conversation-1",
- timestamp: new Date(traceStartedAt),
- input: expectedWrappingRootInput,
- tags: ["in-app-agent"],
- metadata: expect.objectContaining({
- approval_continuation_count: 2,
- }),
- }),
- );
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "agent-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "AGENT",
expect.objectContaining({
id: parentObservationId,
traceId: parentTraceId,
@@ -1016,8 +1473,8 @@ describe("InAppAgentInstrumentation", () => {
}),
}),
);
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "span-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "SPAN",
expect.objectContaining({
id: `${runId}-approval-wait`,
traceId: parentTraceId,
@@ -1029,8 +1486,8 @@ describe("InAppAgentInstrumentation", () => {
metadata: expect.objectContaining(approvalMetadata),
}),
);
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "generation-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "GENERATION",
expect.objectContaining({
traceId: parentTraceId,
parentObservationId,
@@ -1049,20 +1506,13 @@ describe("InAppAgentInstrumentation", () => {
{
role: "tool",
tool_call_id: "tool-1",
- content: { name: "FOO", version: 1 },
+ content:
+ "Output recorded in the corresponding TOOL observation.",
},
],
}),
}),
);
- expect(mocks.trace.update).toHaveBeenCalledWith(
- expect.objectContaining({
- input: expectedWrappingRootInput,
- metadata: expect.objectContaining({
- approval_continuation_count: 2,
- }),
- }),
- );
},
);
@@ -1103,8 +1553,8 @@ describe("InAppAgentInstrumentation", () => {
]);
instrumentation.end({});
- const finalRootObservation = mocks.handler.langfuse.enqueue.mock.calls
- .filter(([type]) => type === "agent-create")
+ const finalRootObservation = mocks.recordObservation.mock.calls
+ .filter(([type]) => type === "AGENT")
.at(-1)?.[1];
expect(finalRootObservation).toEqual(
expect.objectContaining({
@@ -1117,12 +1567,6 @@ describe("InAppAgentInstrumentation", () => {
}),
}),
);
- expect(mocks.trace.update).toHaveBeenCalledWith(
- expect.objectContaining({
- input: expectedWrappingRootInput,
- output: "I need your approval.\n\nThe tool completed.",
- }),
- );
});
it("compacts text message chunks before recording output", () => {
@@ -1145,8 +1589,8 @@ describe("InAppAgentInstrumentation", () => {
},
]);
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "agent-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "AGENT",
expect.objectContaining({
name: "agent-turn",
input: expectedAgentRunInput,
@@ -1228,8 +1672,8 @@ describe("InAppAgentInstrumentation", () => {
},
]);
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "agent-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "AGENT",
expect.objectContaining({
name: "agent-turn",
input: expectedAgentRunInput,
@@ -1241,7 +1685,11 @@ describe("InAppAgentInstrumentation", () => {
thinking: [{ type: "thinking", content: "Checking filters" }],
tool_calls: [toolCall],
},
- { role: "tool", tool_call_id: "tool-1", content: toolOutput },
+ {
+ role: "tool",
+ tool_call_id: "tool-1",
+ content: "Output recorded in TOOL observation tool-1.",
+ },
{
role: "assistant",
content: "Done",
@@ -1253,8 +1701,8 @@ describe("InAppAgentInstrumentation", () => {
},
}),
);
- const agentCreateBody = mocks.handler.langfuse.enqueue.mock.calls.find(
- ([type]) => type === "agent-create",
+ const agentCreateBody = mocks.recordObservation.mock.calls.find(
+ ([type]) => type === "AGENT",
)?.[1] as { metadata?: Record<string, unknown> };
expect(agentCreateBody.metadata).not.toHaveProperty("reasoning");
});
@@ -1273,8 +1721,8 @@ describe("InAppAgentInstrumentation", () => {
},
]);
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "agent-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "AGENT",
expect.objectContaining({
name: "agent-turn",
output: {
@@ -1302,8 +1750,8 @@ describe("InAppAgentInstrumentation", () => {
},
]);
- expect(mocks.handler.langfuse.enqueue).not.toHaveBeenCalledWith(
- "tool-create",
+ expect(mocks.recordObservation).not.toHaveBeenCalledWith(
+ "TOOL",
expect.anything(),
);
});
@@ -1314,8 +1762,8 @@ describe("InAppAgentInstrumentation", () => {
// Root agent observation must exist before children so tools/generations
// do not orphan onto the trace.
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "agent-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "AGENT",
expect.objectContaining({
id: agentRunObservationId,
name: "agent-turn",
@@ -1392,8 +1840,8 @@ describe("InAppAgentInstrumentation", () => {
});
instrumentation.end({});
- const generationCreates = mocks.handler.langfuse.enqueue.mock.calls.filter(
- ([type]) => type === "generation-create",
+ const generationCreates = mocks.recordObservation.mock.calls.filter(
+ ([type]) => type === "GENERATION",
);
expect(generationCreates).toHaveLength(2);
@@ -1446,8 +1894,8 @@ describe("InAppAgentInstrumentation", () => {
generationCreates[0]?.[1].input.messages.length,
);
- const agentCreates = mocks.handler.langfuse.enqueue.mock.calls.filter(
- ([type]) => type === "agent-create",
+ const agentCreates = mocks.recordObservation.mock.calls.filter(
+ ([type]) => type === "AGENT",
);
const finalAgentCreate = agentCreates.at(-1)?.[1];
expect(finalAgentCreate).toEqual(
@@ -1475,8 +1923,8 @@ describe("InAppAgentInstrumentation", () => {
finishReason: "stop",
});
- const generation = mocks.handler.langfuse.enqueue.mock.calls.find(
- ([type]) => type === "generation-create",
+ const generation = mocks.recordObservation.mock.calls.find(
+ ([type]) => type === "GENERATION",
)?.[1];
expect(generation?.input).toEqual({
messages: [{ role: "user", content: "hello" }],
@@ -1576,26 +2024,24 @@ describe("InAppAgentInstrumentation", () => {
},
]);
- const enqueueOrder = mocks.handler.langfuse.enqueue.mock.calls
- .filter(
- ([type]) => type === "generation-create" || type === "tool-create",
- )
+ const enqueueOrder = mocks.recordObservation.mock.calls
+ .filter(([type]) => type === "GENERATION" || type === "TOOL")
.map(([type, body]) => ({ type, id: body.id }));
expect(enqueueOrder.map((entry) => entry.type)).toEqual([
- "generation-create",
- "tool-create",
- "tool-create",
+ "GENERATION",
+ "TOOL",
+ "TOOL",
]);
- const generationCreate = mocks.handler.langfuse.enqueue.mock.calls.find(
- ([type]) => type === "generation-create",
+ const generationCreate = mocks.recordObservation.mock.calls.find(
+ ([type]) => type === "GENERATION",
)?.[1];
expect(generationCreate).toEqual(
expect.objectContaining({ endTime: providerFinish }),
);
- const toolCreates = mocks.handler.langfuse.enqueue.mock.calls.filter(
- ([type]) => type === "tool-create",
+ const toolCreates = mocks.recordObservation.mock.calls.filter(
+ ([type]) => type === "TOOL",
);
expect(toolCreates).toHaveLength(2);
// Actual execute() timing wins over the delayed AG-UI event timestamps.
@@ -1640,8 +2086,8 @@ describe("InAppAgentInstrumentation", () => {
]);
instrumentation.end({});
- expect(mocks.handler.langfuse.enqueue).not.toHaveBeenCalledWith(
- "tool-create",
+ expect(mocks.recordObservation).not.toHaveBeenCalledWith(
+ "TOOL",
expect.objectContaining({ id: "tool-awaiting-approval" }),
);
});
@@ -1652,8 +2098,8 @@ describe("InAppAgentInstrumentation", () => {
instrumentation.recordModelCallStart({ prompt: [] });
instrumentation.end({ aborted: true });
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "generation-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "GENERATION",
expect.objectContaining({
id: getInAppAgentLlmCallObservationId(runId, 1),
parentObservationId: agentRunObservationId,
@@ -1661,8 +2107,8 @@ describe("InAppAgentInstrumentation", () => {
statusMessage: "aborted",
}),
);
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "agent-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "AGENT",
expect.objectContaining({
id: agentRunObservationId,
metadata: expect.objectContaining({ aborted: true }),
@@ -1699,8 +2145,8 @@ describe("InAppAgentInstrumentation", () => {
});
instrumentation.endWithError(new Error("tool execution failed"));
- const generation = mocks.handler.langfuse.enqueue.mock.calls.find(
- ([type]) => type === "generation-create",
+ const generation = mocks.recordObservation.mock.calls.find(
+ ([type]) => type === "GENERATION",
)?.[1];
expect(generation).toEqual(
expect.objectContaining({
@@ -1716,8 +2162,8 @@ describe("InAppAgentInstrumentation", () => {
);
expect(generation).not.toHaveProperty("level");
expect(generation).not.toHaveProperty("statusMessage");
- expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
- "agent-create",
+ expect(mocks.recordObservation).toHaveBeenCalledWith(
+ "AGENT",
expect.objectContaining({
id: agentRunObservationId,
level: "ERROR",
@@ -1739,6 +2185,7 @@ function createInstrumentation(
userProjectRole: "ADMIN",
userIsAdmin: true,
runId: input.runId,
+ runStartedAt: new Date(),
targetProjectId: "project-1",
environment: "prod",
prompt,
diff --git a/worker/src/features/in-app-agent/runtime/instrumentation.ts b/worker/src/features/in-app-agent/runtime/instrumentation.ts
index bbf8b1232e..bb18c00da9 100644
--- a/worker/src/features/in-app-agent/runtime/instrumentation.ts
+++ b/worker/src/features/in-app-agent/runtime/instrumentation.ts
@@ -36,6 +36,7 @@ export type InAppAgentTracingConfig = {
isAdmin: boolean;
};
runId: string;
+ runStartedAt: Date;
targetProjectId: string;
prompt?: InAppAgentPromptMetadata;
};
@@ -54,10 +55,6 @@ export type InAppAgentPromptMetadata = {
const IN_APP_AGENT_TURN_NAME = "agent-turn";
const IN_APP_AGENT_LLM_CALL_NAME = "invoke-model";
type InternalTracingHandler = ReturnType<typeof getInternalTracingHandler>;
-type InAppAgentTrace = ReturnType<
- InternalTracingHandler["handler"]["langfuse"]["trace"]
->;
-type InAppAgentLangfuse = InternalTracingHandler["handler"]["langfuse"];
type AgentRunToolCall = {
id: string;
name: string;
@@ -91,27 +88,9 @@ type AgentRunChatMessage = {
tool_calls?: AgentRunToolCall[];
tool_call_id?: string;
};
-type ObservationBody = {
- id: string;
- traceId: string;
- parentObservationId?: string | null;
- name: string;
- startTime: Date;
- endTime: Date;
- completionStartTime?: Date;
- input?: unknown;
- output?: unknown;
- model?: string;
- usageDetails?: Record<string, number>;
- level?: "DEFAULT" | "ERROR";
- statusMessage?: string | null;
- metadata?: Record<string, unknown>;
- promptName?: string;
- promptVersion?: number;
- userId?: string;
- sessionId?: string;
- skillsAvailable?: SkillsAvailableInput;
-};
+type ObservationBody = Parameters<
+ InternalTracingHandler["recordObservation"]
+>[1];
type ToolCallApprovalStatus = "approved" | "rejected";
type AgentRunToolSpan = {
name: string;
@@ -171,6 +150,7 @@ export function createInAppAgentInstrumentation({
userProjectRole: tracing.user.projectRole,
userIsAdmin: tracing.user.isAdmin,
runId: tracing.runId,
+ runStartedAt: tracing.runStartedAt,
targetProjectId: tracing.targetProjectId,
environment: tracing.environment,
prompt: tracing.prompt,
@@ -185,8 +165,7 @@ export function createInAppAgentInstrumentation({
export class InAppAgentInstrumentation {
private readonly processTracedEvents: () => Promise<void>;
private flushPromise?: Promise<void>;
- private readonly langfuse: InAppAgentLangfuse;
- private readonly trace: InAppAgentTrace;
+ private readonly recordObservation: InternalTracingHandler["recordObservation"];
private readonly runId: string;
private readonly traceId: string;
private readonly rootObservationId: string;
@@ -195,7 +174,6 @@ export class InAppAgentInstrumentation {
private readonly traceStartTime: Date;
private readonly agentRunStartTime: Date;
private agentRunInput: unknown;
- private readonly wrappingRootInput: string | undefined;
private readonly approvalContinuation?: ApprovalContinuation;
private readonly prompt?: InAppAgentPromptMetadata;
private readonly toolSpans = new Map<string, AgentRunToolSpan>();
@@ -232,6 +210,7 @@ export class InAppAgentInstrumentation {
userProjectRole?: InAppAgentUserAccess["projectRole"];
userIsAdmin: boolean;
runId: string;
+ runStartedAt: Date;
targetProjectId: string;
environment: string;
prompt?: InAppAgentPromptMetadata;
@@ -259,7 +238,6 @@ export class InAppAgentInstrumentation {
: {}),
};
this.agentRunInput = getAgentRunInput(params.input);
- this.wrappingRootInput = getWrappingRootUserText(params.input);
this.prompt = params.prompt;
this.targetProjectId = params.targetProjectId;
this.model = params.model;
@@ -271,8 +249,10 @@ export class InAppAgentInstrumentation {
this.rootObservationId =
getInAppAgentInstrumentationObservationId(rootRunId);
this.agentRunStartTime = new Date();
+ // V4 replacement keys include exact start_time; continuations must reuse
+ // the persisted root timestamp rather than instrumentation initialization.
this.traceStartTime =
- this.approvalContinuation?.traceStartedAt ?? this.agentRunStartTime;
+ this.approvalContinuation?.traceStartedAt ?? params.runStartedAt;
const traceSinkParams = {
targetProjectId: params.targetProjectId,
@@ -283,23 +263,11 @@ export class InAppAgentInstrumentation {
sessionId: params.input.threadId,
metadata: this.metadata,
prompt: params.prompt,
- aiFeatureOtelIngestion: true,
};
- const { handler, processTracedEvents } =
+ const { recordObservation, processTracedEvents } =
getInternalTracingHandler(traceSinkParams);
this.processTracedEvents = processTracedEvents;
- this.langfuse = handler.langfuse;
-
- this.trace = this.langfuse.trace({
- id: this.traceId,
- name: IN_APP_AGENT_TURN_NAME,
- userId: params.userId,
- sessionId: params.input.threadId,
- timestamp: this.traceStartTime,
- input: this.wrappingRootInput,
- metadata: this.metadata,
- tags: ["in-app-agent"],
- });
+ this.recordObservation = recordObservation;
// Create the root observation immediately so tool/generation children
// emitted during the run have a parent that already exists (avoids
@@ -500,11 +468,6 @@ export class InAppAgentInstrumentation {
error: message,
},
});
- this.trace.update({
- input: this.wrappingRootInput,
- output: this.getWrappingRootOutput(),
- metadata: { ...this.metadata, error: message },
- });
this.ended = true;
}
@@ -526,11 +489,6 @@ export class InAppAgentInstrumentation {
...(params?.result ? { result: params.result } : {}),
};
this.emitRootAgentObservation({ metadata });
- this.trace.update({
- input: this.wrappingRootInput,
- output: this.getWrappingRootOutput(),
- metadata,
- });
this.ended = true;
}
@@ -754,7 +712,15 @@ export class InAppAgentInstrumentation {
endTime,
completionStartTime: startTime,
input,
- output,
+ output:
+ output === undefined
+ ? undefined
+ : {
+ role: "tool",
+ name: tool.name,
+ tool_call_id: toolCallId,
+ content: output,
+ },
...(failureMessage
? { level: "ERROR", statusMessage: failureMessage }
: {}),
@@ -775,7 +741,7 @@ export class InAppAgentInstrumentation {
this.toolCallApprovalSources.delete(toolCallId);
this.toolExecutionTimes.delete(toolCallId);
- this.enqueueObservation("tool-create", body);
+ this.enqueueObservation("TOOL", body);
}
private emitRootAgentObservation(params: {
@@ -783,10 +749,11 @@ export class InAppAgentInstrumentation {
level?: "ERROR";
statusMessage?: string;
}) {
- this.enqueueObservation("agent-create", {
+ this.enqueueObservation("AGENT", {
id: this.rootObservationId,
traceId: this.traceId,
name: IN_APP_AGENT_TURN_NAME,
+ tags: ["in-app-agent"],
startTime: this.traceStartTime,
endTime: new Date(),
input: this.getTraceInput(),
@@ -832,7 +799,7 @@ export class InAppAgentInstrumentation {
}
: undefined;
- this.enqueueObservation("generation-create", {
+ this.enqueueObservation("GENERATION", {
id,
traceId: this.traceId,
parentObservationId: this.rootObservationId,
@@ -937,14 +904,13 @@ export class InAppAgentInstrumentation {
);
}
- private enqueueObservation(type: string, body: ObservationBody) {
- (
- this.langfuse as unknown as {
- enqueue: (type: string, body: ObservationBody) => void;
- }
- ).enqueue(type, {
+ private enqueueObservation(
+ type: Parameters<InternalTracingHandler["recordObservation"]>[0],
+ body: ObservationBody,
+ ) {
+ this.recordObservation(type, {
...body,
- ...((type === "generation-create" || type === "tool-create") &&
+ ...((type === "GENERATION" || type === "TOOL") &&
this.skillsAvailable.length > 0
? { skillsAvailable: this.skillsAvailable }
: {}),
@@ -1048,23 +1014,6 @@ export class InAppAgentInstrumentation {
}
}
- private getWrappingRootOutput(): string | undefined {
- const texts = this.getAgentRunOutputMessages().flatMap((message) => {
- if (message.role !== "assistant") {
- return [];
- }
-
- const text = getRenderedMessageText(message.content);
- return text.trim() ? [text] : [];
- });
-
- if (texts.length === 0) {
- return undefined;
- }
-
- return texts.join("\n\n");
- }
-
private getAgentRunOutput() {
const messages = this.getAgentRunOutputMessages();
@@ -1094,7 +1043,17 @@ export class InAppAgentInstrumentation {
? getPriorAgentRunOutputMessages(this.agentRunInput.messages)
: [];
- return [...priorMessages, ...this.agentRunOutputMessages];
+ const messages = [...priorMessages, ...this.agentRunOutputMessages];
+ return messages.map((message) => {
+ if (message.role !== "tool") {
+ return message;
+ }
+
+ return {
+ ...message,
+ content: `Output recorded in TOOL observation ${message.tool_call_id}.`,
+ };
+ });
}
private emitApprovalWaitObservation() {
@@ -1111,7 +1070,7 @@ export class InAppAgentInstrumentation {
const endTime =
decidedAt.getTime() < requestedAt.getTime() ? requestedAt : decidedAt;
- this.enqueueObservation("span-create", {
+ this.enqueueObservation("SPAN", {
id: `${this.runId}-approval-wait`,
traceId: this.traceId,
parentObservationId: this.rootObservationId,
@@ -1192,37 +1151,6 @@ function getPriorAgentRunOutputMessages(
);
}
-function getWrappingRootUserText(input: AgUiRunAgentInput): string | undefined {
- const lastUserMessage = input.messages.findLast(
- (message) => message.role === "user",
- );
-
- if (!lastUserMessage) {
- return undefined;
- }
-
- const text = getRenderedMessageText(lastUserMessage.content);
- return text.length > 0 ? text : undefined;
-}
-
-function getRenderedMessageText(content: unknown): string {
- if (typeof content === "string") {
- return content;
- }
-
- if (Array.isArray(content)) {
- return content
- .flatMap((part) =>
- isRecord(part) && part.type === "text" && typeof part.text === "string"
- ? [part.text]
- : [],
- )
- .join("");
- }
-
- return "";
-}
-
function getAgentRunInput(input: AgUiRunAgentInput): unknown {
const messages = getAgentRunMessages(getCurrentTurnMessages(input.messages));
const context = getAgentRunContext(input);
@@ -1451,10 +1379,45 @@ function getModelCallInput(options: unknown): unknown {
providerOptions: options.providerOptions,
});
- // The prompt is the exact provider-bound payload. Preserve it losslessly in
- // tracing; this copy does not affect the options passed to the model.
+ // Only change telemetry, never the provider-bound prompt. Stable tool-result
+ // IDs let session transcripts attach the full TOOL observation output.
+ const messages = Array.isArray(options.prompt)
+ ? options.prompt.map((message: unknown) => {
+ if (!isRecord(message) || message.role !== "tool") {
+ return message;
+ }
+
+ if (!Array.isArray(message.content)) {
+ return {
+ ...message,
+ content: "Output recorded in the corresponding TOOL observation.",
+ };
+ }
+
+ return {
+ ...message,
+ content: message.content.map((part: unknown) => {
+ if (!isRecord(part) || part.type !== "tool-result") {
+ return part;
+ }
+
+ // Allowlist fields: Mastra also duplicates the payload in
+ // providerOptions.mastra.modelOutput.
+ return {
+ type: part.type,
+ toolCallId: part.toolCallId,
+ toolName: part.toolName,
+ output: {
+ type: "text",
+ value: `Output recorded in TOOL observation ${part.toolCallId}.`,
+ },
+ };
+ }),
+ };
+ })
+ : options.prompt;
return {
- ...(options.prompt === undefined ? {} : { messages: options.prompt }),
+ ...(messages === undefined ? {} : { messages }),
...(isRecord(serializedOptions) ? serializedOptions : {}),
};
}
diff --git a/worker/src/features/internal-tracing/__tests__/getInternalTracingHandler.test.ts b/worker/src/features/internal-tracing/__tests__/getInternalTracingHandler.test.ts
index 64a9c41549..98ad06ee98 100644
--- a/worker/src/features/internal-tracing/__tests__/getInternalTracingHandler.test.ts
+++ b/worker/src/features/internal-tracing/__tests__/getInternalTracingHandler.test.ts
@@ -1,195 +1,138 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
-const {
- mockExportLocalEvents,
- mockProcessEventBatch,
- mockWrite,
- mockPublishAiFeatureTrace,
-} = vi.hoisted(() => ({
- mockExportLocalEvents: vi.fn(),
- mockProcessEventBatch: vi.fn(),
- mockWrite: vi.fn(),
- mockPublishAiFeatureTrace: vi.fn(),
+const { publish, traceException, logError } = vi.hoisted(() => ({
+ publish: vi.fn(),
+ traceException: vi.fn(),
+ logError: vi.fn(),
}));
-vi.mock("langfuse", () => {
- return {
- Langfuse: class MockLangfuse {
- public _exportLocalEvents = mockExportLocalEvents;
- },
- };
-});
-
-vi.mock(
- "../../../../../packages/shared/src/server/ingestion/processEventBatch",
- () => ({
- processEventBatch: mockProcessEventBatch,
- }),
-);
-
vi.mock(
"../../../../../packages/shared/src/server/otel/internalAiFeatureOtelWriter",
() => ({
- publishAiFeatureTraceViaOtelIngestion: mockPublishAiFeatureTrace,
+ publishAiFeatureTraceViaOtelIngestion: publish,
}),
);
+vi.mock("../../../../../packages/shared/src/server/instrumentation", () => ({
+ traceException,
+}));
+vi.mock("../../../../../packages/shared/src/server/logger", () => ({
+ logger: { error: logError },
+}));
import { getInternalTracingHandler } from "../../../../../packages/shared/src/server/llm/getInternalTracingHandler";
-import { LangfuseInternalTraceEnvironment } from "@langfuse/shared";
-import { env } from "../../../../../packages/shared/src/env";
-const traceId = "trace-123";
-const processedEventsFixture = [
- {
- type: "trace-create",
- timestamp: "2026-01-31T06:58:02.221Z",
- body: {
- id: traceId,
- traceId,
- startTime: "2026-01-31T06:58:02.218Z",
- environment: LangfuseInternalTraceEnvironment.PromptExperiments,
- name: "dataset-run-item-008e4",
- input: [{ role: "user", content: "Hello" }],
- },
- },
- {
- type: "generation-create",
- timestamp: "2026-01-31T06:58:02.222Z",
- body: {
- id: "generation-123",
- traceId,
- parentObservationId: traceId,
- startTime: "2026-01-31T06:58:02.220Z",
- environment: LangfuseInternalTraceEnvironment.PromptExperiments,
- name: "ChatOpenAI",
- input: [{ role: "user", content: "Hello" }],
- model: "gpt-4.1",
- },
- },
- {
- type: "generation-update",
- timestamp: "2026-01-31T06:58:03.708Z",
- body: {
- id: "generation-123",
- traceId,
- output: { role: "assistant", content: "Hi" },
- endTime: "2026-01-31T06:58:03.707Z",
- usageDetails: {
- input: 10,
- output: 5,
- total: 15,
- },
- },
- },
-];
+const params = {
+ targetProjectId: "project-123",
+ traceId: "trace-123",
+ traceName: "agent-turn",
+ environment: "production",
+ userId: "user-123",
+ sessionId: "session-123",
+ metadata: { feature: "in-app-agent" },
+};
+const agent = {
+ id: "agent-123",
+ traceId: params.traceId,
+ name: "agent-turn",
+ startTime: new Date("2026-01-31T06:58:02.218Z"),
+ endTime: new Date("2026-01-31T06:58:03.707Z"),
+ input: { messages: [{ role: "user", content: "Hello" }] },
+ tags: ["in-app-agent"],
+};
describe("getInternalTracingHandler", () => {
beforeEach(() => {
vi.clearAllMocks();
- mockExportLocalEvents.mockResolvedValue(processedEventsFixture);
- mockProcessEventBatch.mockResolvedValue({
- successes: [],
- errors: [],
- });
- mockWrite.mockResolvedValue(undefined);
- mockPublishAiFeatureTrace.mockResolvedValue(undefined);
+ publish.mockResolvedValue(undefined);
});
- it("disables staging propagation when direct event write is enabled", async () => {
- const { processTracedEvents } = getInternalTracingHandler({
- targetProjectId: "project-123",
- traceId,
- traceName: "internal-trace",
- environment: LangfuseInternalTraceEnvironment.PromptExperiments,
- eventsWriter: {
- experimentContext: {
- id: "run-123",
- name: "Prompt run",
- datasetId: "dataset-123",
- itemId: "item-123",
- itemVersion: "2026-01-31T06:57:38.646Z",
- },
- write: mockWrite,
- },
+ it("publishes one root AGENT with its latest output and child generations", async () => {
+ const handler = getInternalTracingHandler(params);
+ handler.recordObservation("AGENT", agent);
+ handler.recordObservation("GENERATION", {
+ ...agent,
+ id: "generation-123",
+ parentObservationId: agent.id,
+ name: "invoke-model",
+ model: "gpt-4.1",
+ completionStartTime: agent.endTime,
+ usageDetails: { input: 10, output: 5, total: 15 },
+ promptName: "system-prompt",
+ promptVersion: 3,
});
-
- await processTracedEvents();
-
- expect(mockProcessEventBatch).toHaveBeenCalledWith(
- expect.any(Array),
- expect.any(Object),
- expect.objectContaining({
- isLangfuseInternal: true,
- forwardToEventsTable: false,
- }),
- );
- });
-
- it("keeps legacy forwarding behavior when direct event write is disabled", async () => {
- const { processTracedEvents } = getInternalTracingHandler({
- targetProjectId: "project-123",
- traceId,
- traceName: "internal-trace",
- environment: LangfuseInternalTraceEnvironment.LLMJudge,
+ handler.recordObservation("AGENT", { ...agent, output: { text: "Hi" } });
+ await handler.processTracedEvents();
+
+ expect(publish).toHaveBeenCalledExactlyOnceWith({
+ projectId: params.targetProjectId,
+ eventInputs: [
+ expect.objectContaining({
+ spanId: agent.id,
+ traceId: params.traceId,
+ parentSpanId: undefined,
+ type: "AGENT",
+ input: JSON.stringify(agent.input),
+ output: JSON.stringify({ text: "Hi" }),
+ environment: "production",
+ userId: params.userId,
+ sessionId: params.sessionId,
+ metadata: params.metadata,
+ tags: ["in-app-agent"],
+ }),
+ expect.objectContaining({
+ spanId: "generation-123",
+ parentSpanId: agent.id,
+ type: "GENERATION",
+ modelName: "gpt-4.1",
+ providedUsageDetails: { input: 10, output: 5, total: 15 },
+ promptName: "system-prompt",
+ promptVersion: "3",
+ completionStartTime: agent.endTime.toISOString(),
+ }),
+ ],
});
-
- await processTracedEvents();
-
- expect(mockProcessEventBatch).toHaveBeenCalledWith(
- expect.any(Array),
- expect.any(Object),
- expect.objectContaining({
- isLangfuseInternal: true,
- forwardToEventsTable: undefined,
- }),
- );
- expect(mockWrite).not.toHaveBeenCalled();
+ await handler.processTracedEvents();
+ expect(publish).toHaveBeenCalledTimes(1);
});
- it("publishes AI-feature traces via OTel on v4 write modes", async () => {
- const previousWriteMode = env.LANGFUSE_MIGRATION_V4_WRITE_MODE;
- env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "events_only";
-
- try {
- const { processTracedEvents } = getInternalTracingHandler({
- targetProjectId: "project-123",
- traceId,
- traceName: "agent-turn",
- environment: "production",
- aiFeatureOtelIngestion: true,
+ it.each(["GENERATION", "TOOL"] as const)(
+ "preserves available skills in published %s event inputs",
+ async (type) => {
+ const handler = getInternalTracingHandler(params);
+ const skillsAvailable = [{ name: "trace-analysis", version: 2 }];
+ handler.recordObservation(type, { ...agent, skillsAvailable });
+
+ await handler.processTracedEvents();
+
+ expect(publish).toHaveBeenCalledExactlyOnceWith({
+ projectId: params.targetProjectId,
+ eventInputs: [
+ expect.objectContaining({
+ spanId: agent.id,
+ type,
+ skillsAvailable,
+ }),
+ ],
});
+ },
+ );
- await processTracedEvents();
-
- expect(mockProcessEventBatch).not.toHaveBeenCalled();
- expect(mockPublishAiFeatureTrace).toHaveBeenCalledWith({
- projectId: "project-123",
- eventInputs: expect.any(Array),
- });
- } finally {
- env.LANGFUSE_MIGRATION_V4_WRITE_MODE = previousWriteMode;
- }
+ it("does not publish an empty batch", async () => {
+ await getInternalTracingHandler(params).processTracedEvents();
+ expect(publish).not.toHaveBeenCalled();
});
- it("keeps processEventBatch for AI-feature traces in legacy write mode", async () => {
- const previousWriteMode = env.LANGFUSE_MIGRATION_V4_WRITE_MODE;
- env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "legacy";
-
- try {
- const { processTracedEvents } = getInternalTracingHandler({
- targetProjectId: "project-123",
- traceId,
- traceName: "agent-turn",
- environment: "production",
- aiFeatureOtelIngestion: true,
- });
-
- await processTracedEvents();
-
- expect(mockProcessEventBatch).toHaveBeenCalled();
- expect(mockPublishAiFeatureTrace).not.toHaveBeenCalled();
- } finally {
- env.LANGFUSE_MIGRATION_V4_WRITE_MODE = previousWriteMode;
- }
+ it("records errors without failing the agent", async () => {
+ const error = new Error("queue unavailable");
+ publish.mockRejectedValueOnce(error);
+ const handler = getInternalTracingHandler(params);
+ handler.recordObservation("AGENT", {
+ ...agent,
+ level: "ERROR",
+ statusMessage: "agent failed",
+ });
+ await expect(handler.processTracedEvents()).resolves.toBeUndefined();
+ expect(traceException).toHaveBeenCalledWith(error);
+ expect(logError).toHaveBeenCalledWith(expect.any(String), { error });
});
});