Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(in-app-agent): ingest AI feature traces (starting with in-app agent, other features will follow) via v4 OTel as production (#16697)

Lysander Kiesela19768430aeb
diff --git a/packages/shared/src/in-app-agent/constants.ts b/packages/shared/src/in-app-agent/constants.ts
index 7fbaa1b81d..268bc80338 100644
--- a/packages/shared/src/in-app-agent/constants.ts
+++ b/packages/shared/src/in-app-agent/constants.ts
@@ -63,3 +63,7 @@ export const getInAppAgentLlmCallObservationId = (
runId: string,
stepNumber: number,
) => `${runId}-llm-${stepNumber}`;
+
+// Product traces and feedback scores for the in-app agent. Must match so
+// scores attach to the same traces evaluators in the AI-features project see.
+export const IN_APP_AGENT_PRODUCT_ENVIRONMENT = "production";
diff --git a/packages/shared/src/server/ingestion/ingestionAttribution.ts b/packages/shared/src/server/ingestion/ingestionAttribution.ts
index 681fd2fd72..78ed508fa8 100644
--- a/packages/shared/src/server/ingestion/ingestionAttribution.ts
+++ b/packages/shared/src/server/ingestion/ingestionAttribution.ts
@@ -15,6 +15,7 @@ export const UNKNOWN_INGESTION_SDK_VALUE = "unknown";
export const INTERNAL_INGESTION_SDK_NAMES = [
"langfuse-internal-ai-sdk",
"langfuse-internal-otel-writer",
+ "langfuse-internal-ai-features",
] as const;
export const LANGFUSE_SDK_LATEST_MAJOR = {
diff --git a/packages/shared/src/server/llm/getInternalTracingHandler.ts b/packages/shared/src/server/llm/getInternalTracingHandler.ts
index 55b1ff0b59..93bacaa015 100644
--- a/packages/shared/src/server/llm/getInternalTracingHandler.ts
+++ b/packages/shared/src/server/llm/getInternalTracingHandler.ts
@@ -1,10 +1,12 @@
import { Langfuse } from "langfuse";
+import { env } from "../../env";
import { ProcessedTraceEvent, TraceSinkParams } from "./types";
import { buildInternalTraceEventInputs } from "./internalTraceEvents";
import { processEventBatch } from "../ingestion/processEventBatch";
import { createUnknownSdkIngestionAttribution } from "../ingestion/ingestionAttribution";
import { logger } from "../logger";
import { traceException } from "../instrumentation";
+import { publishAiFeatureTraceViaOtelIngestion } from "../otel/aiFeatureOtelWriter";
export function prepareInternalTraceEvents(params: {
events: Array<{
@@ -99,6 +101,39 @@ export function getInternalTracingHandler(traceSinkParams: TraceSinkParams): {
prompt,
});
+ const useAiFeatureOtel =
+ Boolean(traceSinkParams.aiFeatureOtelIngestion) &&
+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE !== "legacy";
+
+ if (useAiFeatureOtel) {
+ try {
+ const { eventInputs } = buildInternalTraceEventInputs({
+ processedEvents,
+ traceId: traceSinkParams.traceId,
+ projectId: targetProjectId,
+ userId: traceSinkParams.userId,
+ sessionId: traceSinkParams.sessionId,
+ });
+
+ if (eventInputs.length > 0) {
+ await publishAiFeatureTraceViaOtelIngestion({
+ eventInputs,
+ projectId: targetProjectId,
+ });
+ }
+ } catch (otelError) {
+ traceException(otelError);
+ logger.error(
+ "Failed to process AI-feature traces via OTel ingestion",
+ {
+ error: otelError,
+ },
+ );
+ }
+
+ return;
+ }
+
// Legacy write to traces/observations tables
try {
const auth = {
@@ -134,6 +169,8 @@ export function getInternalTracingHandler(traceSinkParams: TraceSinkParams): {
processedEvents,
traceId: traceSinkParams.traceId,
projectId: targetProjectId,
+ userId: traceSinkParams.userId,
+ sessionId: traceSinkParams.sessionId,
experimentContext: eventsWriter.experimentContext,
});
diff --git a/packages/shared/src/server/llm/internalTraceEvents.ts b/packages/shared/src/server/llm/internalTraceEvents.ts
index df7895bba9..5b4496c711 100644
--- a/packages/shared/src/server/llm/internalTraceEvents.ts
+++ b/packages/shared/src/server/llm/internalTraceEvents.ts
@@ -6,6 +6,10 @@ import {
asStringArray,
} from "../../utils/objects";
import { stringifyValue } from "../../utils/stringChecks";
+import {
+ ObservationTypeDomain,
+ type ObservationType,
+} from "../../domain/observations";
import {
convertCallsToArrays,
convertDefinitionsToMap,
@@ -108,7 +112,7 @@ type InternalTraceSnapshot = {
traceId: string;
parentSpanId?: string;
name?: string;
- type: "SPAN" | "GENERATION";
+ type: ObservationType;
environment?: string;
version?: string;
release?: string;
@@ -142,8 +146,18 @@ function isCreateEvent(type: string): boolean {
return type.endsWith("-create");
}
-function getSnapshotType(eventType: string): "SPAN" | "GENERATION" {
- return eventType.startsWith("generation-") ? "GENERATION" : "SPAN";
+function getSnapshotType(eventType: string): ObservationType {
+ const prefix = eventType.replace(/-(create|update)$/i, "").toUpperCase();
+ const parsed = ObservationTypeDomain.safeParse(prefix);
+ return parsed.success ? parsed.data : "SPAN";
+}
+
+function asIsoTimestamp(value: unknown): string | undefined {
+ if (value instanceof Date && !Number.isNaN(value.getTime())) {
+ return value.toISOString();
+ }
+
+ return asString(value);
}
function getEventTime(
@@ -153,6 +167,10 @@ function getEventTime(
const candidates = [body.startTime, body.timestamp, event.timestamp];
for (const candidate of candidates) {
+ if (candidate instanceof Date && !Number.isNaN(candidate.getTime())) {
+ return candidate.getTime();
+ }
+
if (typeof candidate === "string") {
const parsed = new Date(candidate).getTime();
if (!Number.isNaN(parsed)) {
@@ -205,8 +223,9 @@ function mergeSnapshotEvent(
event: ProcessedTraceEvent,
): InternalTraceSnapshot {
const { body } = event;
- const startTime = asString(body.startTime);
- const timestamp = asString(body.timestamp) ?? asString(event.timestamp);
+ const startTime = asIsoTimestamp(body.startTime);
+ const timestamp =
+ asIsoTimestamp(body.timestamp) ?? asIsoTimestamp(event.timestamp);
const metadata = asRecord(body.metadata);
return {
@@ -214,18 +233,16 @@ function mergeSnapshotEvent(
traceId: asString(body.traceId) ?? snapshot.traceId,
parentSpanId: asString(body.parentObservationId) ?? snapshot.parentSpanId,
type:
- snapshot.type === "GENERATION"
- ? snapshot.type
- : getSnapshotType(event.type),
+ snapshot.type !== "SPAN" ? snapshot.type : getSnapshotType(event.type),
name: asString(body.name) ?? snapshot.name,
environment: asString(body.environment) ?? snapshot.environment,
version: asString(body.version) ?? snapshot.version,
release: asString(body.release) ?? snapshot.release,
startTimeISO:
startTime ?? snapshot.startTimeISO ?? timestamp ?? snapshot.startTimeISO,
- endTimeISO: asString(body.endTime) ?? snapshot.endTimeISO,
+ endTimeISO: asIsoTimestamp(body.endTime) ?? snapshot.endTimeISO,
completionStartTime:
- asString(body.completionStartTime) ?? snapshot.completionStartTime,
+ asIsoTimestamp(body.completionStartTime) ?? snapshot.completionStartTime,
level: asString(body.level) ?? snapshot.level,
statusMessage: asString(body.statusMessage) ?? snapshot.statusMessage,
promptName: asString(body.promptName) ?? snapshot.promptName,
@@ -311,6 +328,8 @@ export function buildInternalTraceEventInputs(params: {
processedEvents: ProcessedTraceEvent[];
traceId: string;
projectId: string;
+ userId?: string;
+ sessionId?: string;
experimentContext?: InternalTraceExperimentContext;
}): {
rootSpanId: string;
@@ -375,8 +394,9 @@ export function buildInternalTraceEventInputs(params: {
tags: rootSnapshot.tags ?? [],
bookmarked: rootSnapshot.bookmarked,
public: rootSnapshot.public,
- userId: rootSnapshot.userId,
- sessionId: rootSnapshot.sessionId,
+ userId: snapshot.userId ?? rootSnapshot.userId ?? params.userId,
+ sessionId:
+ snapshot.sessionId ?? rootSnapshot.sessionId ?? params.sessionId,
level: snapshot.level ?? "DEFAULT",
statusMessage: snapshot.statusMessage,
promptName: snapshot.promptName,
diff --git a/packages/shared/src/server/llm/types.ts b/packages/shared/src/server/llm/types.ts
index 8d0ade6257..182bf3323e 100644
--- a/packages/shared/src/server/llm/types.ts
+++ b/packages/shared/src/server/llm/types.ts
@@ -541,6 +541,7 @@ export type TraceSinkParams = {
// NOTE: These strings must be whitelisted in the TS SDK to allow ingestion of traces by Langfuse. Please mirror edits to this string in https://github.com/langfuse/langfuse-js/blob/main/langfuse-core/src/index.ts.
environment: string;
userId?: string;
+ sessionId?: string;
metadata?: Record<string, unknown>;
prompt?: {
name: string;
@@ -553,4 +554,12 @@ export type TraceSinkParams = {
* events are still written to the legacy traces/observations tables.
*/
eventsWriter?: InternalEventsWriter;
+ /**
+ * When true, v4 write modes (`dual` / `events_only`) publish through
+ * `publishAiFeatureTraceViaOtelIngestion` instead of `processEventBatch`.
+ * Used by in-app agent product traces. Those traces use environment
+ * `production` (not `langfuse-*`) so they stay eligible as observation-eval
+ * targets. Legacy write mode keeps `processEventBatch`.
+ */
+ aiFeatureOtelIngestion?: boolean;
};
diff --git a/packages/shared/src/server/otel/aiFeatureOtelWriter.test.ts b/packages/shared/src/server/otel/aiFeatureOtelWriter.test.ts
new file mode 100644
index 0000000000..f43b0f6786
--- /dev/null
+++ b/packages/shared/src/server/otel/aiFeatureOtelWriter.test.ts
@@ -0,0 +1,199 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+import { buildInternalTraceEventInputs } from "../llm/internalTraceEvents";
+import {
+ AI_FEATURE_OTEL_SDK_NAME,
+ publishAiFeatureTraceViaOtelIngestion,
+} from "./aiFeatureOtelWriter";
+
+const publishToOtelIngestionQueue = vi.fn().mockResolvedValue(undefined);
+const processorConstructor = vi.fn();
+
+vi.mock("./OtelIngestionProcessor", async (importOriginal) => {
+ const actual =
+ await importOriginal<typeof import("./OtelIngestionProcessor")>();
+ return {
+ ...actual,
+ OtelIngestionProcessor: class {
+ constructor(config: unknown) {
+ processorConstructor(config);
+ }
+ publishToOtelIngestionQueue = publishToOtelIngestionQueue;
+ },
+ };
+});
+
+const START_ISO = "2026-08-27T12:00:00.000Z";
+const END_ISO = "2026-08-27T12:00:01.000Z";
+const RUN_ID = "arun_01234567-89ab-cdef-0123-456789abcdef";
+const TRACE_ID = `${RUN_ID}-trace`;
+
+const processedEvents = [
+ {
+ type: "trace-create",
+ timestamp: START_ISO,
+ body: {
+ id: TRACE_ID,
+ traceId: TRACE_ID,
+ name: "agent-turn",
+ environment: "production",
+ tags: ["in-app-agent"],
+ userId: "user-1",
+ sessionId: "conversation-1",
+ startTime: START_ISO,
+ endTime: END_ISO,
+ input: "hello",
+ output: "hi",
+ },
+ },
+ {
+ type: "agent-create",
+ timestamp: START_ISO,
+ body: {
+ id: RUN_ID,
+ traceId: TRACE_ID,
+ name: "agent-turn",
+ environment: "production",
+ startTime: START_ISO,
+ endTime: END_ISO,
+ input: { messages: [{ role: "user", content: "hello" }] },
+ output: { text: "hi" },
+ metadata: { langfuse_ai_feature: "in-app-agent" },
+ },
+ },
+ {
+ type: "tool-create",
+ timestamp: START_ISO,
+ body: {
+ id: "tool-call-1",
+ traceId: TRACE_ID,
+ parentObservationId: RUN_ID,
+ name: "langfuse_listPrompts",
+ environment: "production",
+ startTime: START_ISO,
+ endTime: END_ISO,
+ },
+ },
+ {
+ type: "generation-create",
+ timestamp: START_ISO,
+ body: {
+ id: `${RUN_ID}-llm-0`,
+ traceId: TRACE_ID,
+ parentObservationId: RUN_ID,
+ name: "invoke-model",
+ environment: "production",
+ startTime: new Date(START_ISO),
+ endTime: new Date(END_ISO),
+ model: "claude-opus",
+ usageDetails: { input: 10, output: 5, total: 15 },
+ },
+ },
+];
+
+const processPublishedSpans = async () => {
+ expect(publishToOtelIngestionQueue).toHaveBeenCalledTimes(1);
+ const resourceSpans = publishToOtelIngestionQueue.mock.calls[0][0];
+
+ const { OtelIngestionProcessor } = await vi.importActual<
+ typeof import("./OtelIngestionProcessor")
+ >("./OtelIngestionProcessor");
+ return new OtelIngestionProcessor({
+ projectId: "project-1",
+ publicKey: "",
+ sdkName: AI_FEATURE_OTEL_SDK_NAME,
+ sdkVersion: "unknown",
+ ingestionVersion: "4",
+ isLangfuseInternal: false,
+ }).processToEvent(resourceSpans);
+};
+
+beforeEach(() => {
+ vi.clearAllMocks();
+});
+
+describe("publishAiFeatureTraceViaOtelIngestion", () => {
+ it("preserves agent and tool observation types from SDK event types", async () => {
+ const { eventInputs } = buildInternalTraceEventInputs({
+ processedEvents,
+ traceId: TRACE_ID,
+ projectId: "project-1",
+ });
+
+ await publishAiFeatureTraceViaOtelIngestion({
+ projectId: "project-1",
+ eventInputs,
+ });
+
+ expect(processorConstructor).toHaveBeenCalledWith(
+ expect.objectContaining({
+ ingestionVersion: "4",
+ isLangfuseInternal: false,
+ sdkName: AI_FEATURE_OTEL_SDK_NAME,
+ }),
+ );
+
+ const events = await processPublishedSpans();
+ expect(events).toHaveLength(4);
+
+ const wrappingRoot = events.find((event) => event.spanId === TRACE_ID);
+ const agent = events.find((event) => event.spanId === RUN_ID);
+ const tool = events.find((event) => event.spanId === "tool-call-1");
+ const generation = events.find(
+ (event) => event.spanId === `${RUN_ID}-llm-0`,
+ );
+
+ expect(wrappingRoot).toMatchObject({
+ traceId: TRACE_ID,
+ parentSpanId: null,
+ name: "agent-turn",
+ type: "SPAN",
+ environment: "production",
+ userId: "user-1",
+ sessionId: "conversation-1",
+ });
+ expect(wrappingRoot?.input).toBe("hello");
+ expect(wrappingRoot?.output).toBe("hi");
+ expect(wrappingRoot?.tags).toEqual(["in-app-agent"]);
+ expect(agent).toMatchObject({
+ traceId: TRACE_ID,
+ parentSpanId: TRACE_ID,
+ name: "agent-turn",
+ type: "AGENT",
+ environment: "production",
+ });
+ expect(generation).toMatchObject({
+ traceId: TRACE_ID,
+ parentSpanId: RUN_ID,
+ name: "invoke-model",
+ type: "GENERATION",
+ environment: "production",
+ modelName: "claude-opus",
+ userId: "user-1",
+ sessionId: "conversation-1",
+ });
+ expect(tool).toMatchObject({
+ traceId: TRACE_ID,
+ parentSpanId: RUN_ID,
+ name: "langfuse_listPrompts",
+ type: "TOOL",
+ environment: "production",
+ userId: "user-1",
+ sessionId: "conversation-1",
+ });
+ expect(generation?.providedUsageDetails).toMatchObject({
+ input: 10,
+ output: 5,
+ total: 15,
+ });
+ });
+
+ it("skips an empty input", async () => {
+ await publishAiFeatureTraceViaOtelIngestion({
+ projectId: "project-1",
+ eventInputs: [],
+ });
+
+ expect(publishToOtelIngestionQueue).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/shared/src/server/otel/aiFeatureOtelWriter.ts b/packages/shared/src/server/otel/aiFeatureOtelWriter.ts
new file mode 100644
index 0000000000..6e6ca81c80
--- /dev/null
+++ b/packages/shared/src/server/otel/aiFeatureOtelWriter.ts
@@ -0,0 +1,121 @@
+import type { InternalTraceEventInput } from "../llm/internalTraceEvents";
+import { type ResourceSpan } from "./OtelIngestionProcessor";
+import {
+ internalTraceEventToOtelAttributes,
+ publishOtelResourceSpans,
+} from "./internalTraceOtelWriter";
+
+export const AI_FEATURE_OTEL_SDK_NAME = "langfuse-internal-ai-features";
+const AI_FEATURE_OTEL_SCOPE = "langfuse-ai-features-writer";
+const AI_FEATURE_SERVICE_NAME = "langfuse-ai-features";
+
+type OtelAttribute = {
+ key: string;
+ value: { stringValue: string };
+};
+
+function timestampToUnixNano(timestamp: string): string {
+ const milliseconds = Date.parse(timestamp);
+ if (Number.isNaN(milliseconds)) {
+ throw new Error(`Invalid AI-feature trace timestamp: ${timestamp}`);
+ }
+
+ return `${BigInt(milliseconds) * 1_000_000n}`;
+}
+
+function attributesToOtelList(
+ attributes: ReturnType<typeof internalTraceEventToOtelAttributes>,
+): OtelAttribute[] {
+ return Object.entries(attributes)
+ .map(([key, value]) => {
+ if (value === undefined || value === null) return undefined;
+ return {
+ key,
+ value: {
+ stringValue:
+ typeof value === "string" ? value : JSON.stringify(value),
+ },
+ };
+ })
+ .filter((attribute): attribute is OtelAttribute => Boolean(attribute));
+}
+
+/**
+ * Keep the SDK `trace-create` span (id === trace id). Legacy ingestion stored
+ * that as a TRACE row; event propagation then materializes it as the wrapping
+ * `t-{traceId}` SPAN with conversation input/output. Emitting it here gives
+ * OTel-native assistant traces the same root span above the AGENT turn.
+ */
+function aiFeatureTraceInputsToResourceSpans(
+ eventInputs: InternalTraceEventInput[],
+): ResourceSpan[] {
+ const spans = eventInputs.map((eventInput) => {
+ const isRoot = !eventInput.parentSpanId;
+
+ return {
+ traceId: eventInput.traceId,
+ spanId: eventInput.spanId,
+ ...(eventInput.parentSpanId
+ ? { parentSpanId: eventInput.parentSpanId }
+ : {}),
+ name: eventInput.name ?? "span",
+ kind: 1,
+ startTimeUnixNano: timestampToUnixNano(eventInput.startTimeISO),
+ endTimeUnixNano: timestampToUnixNano(eventInput.endTimeISO),
+ attributes: attributesToOtelList(
+ internalTraceEventToOtelAttributes(eventInput, { isRoot }),
+ ),
+ status: {
+ ...(eventInput.level === "ERROR" ? { code: 2 } : {}),
+ ...(eventInput.statusMessage
+ ? { message: eventInput.statusMessage }
+ : {}),
+ },
+ };
+ });
+
+ return [
+ {
+ resource: {
+ attributes: [
+ {
+ key: "service.name",
+ value: { stringValue: AI_FEATURE_SERVICE_NAME },
+ },
+ ],
+ },
+ scopeSpans: [
+ {
+ scope: {
+ name: AI_FEATURE_OTEL_SCOPE,
+ version: "unknown",
+ },
+ spans,
+ },
+ ],
+ },
+ ] as ResourceSpan[];
+}
+
+/**
+ * Publishes AI-features product traces through the same OTel ingestion queue
+ * as `writeInternalTraceViaOtelIngestion`. Span construction stays here so
+ * non-W3C ids (`arun_*`, `${runId}-llm-N`) and `production` survive; attribute
+ * names and observation types come from `internalTraceEventToOtelAttributes`.
+ */
+export async function publishAiFeatureTraceViaOtelIngestion(params: {
+ eventInputs: InternalTraceEventInput[];
+ projectId: string;
+}): Promise<void> {
+ if (params.eventInputs.length === 0) return;
+
+ await publishOtelResourceSpans({
+ resourceSpans: aiFeatureTraceInputsToResourceSpans(params.eventInputs),
+ projectId: params.projectId,
+ sdkName: AI_FEATURE_OTEL_SDK_NAME,
+ ingestionVersion: "4",
+ // Product traces must look like customer data: the public schema keeps
+ // `production`, and observation evals are allowed to match them.
+ isLangfuseInternal: false,
+ });
+}
diff --git a/packages/shared/src/server/otel/index.ts b/packages/shared/src/server/otel/index.ts
index 92dae9b006..5ec05d7406 100644
--- a/packages/shared/src/server/otel/index.ts
+++ b/packages/shared/src/server/otel/index.ts
@@ -1,3 +1,4 @@
export * from "./OtelIngestionProcessor";
export * from "./OtelMediaProcessor";
export * from "./internalTraceOtelWriter";
+export * from "./aiFeatureOtelWriter";
diff --git a/packages/shared/src/server/otel/internalTraceOtelWriter.ts b/packages/shared/src/server/otel/internalTraceOtelWriter.ts
index 14ae5d81a1..1ade651eb0 100644
--- a/packages/shared/src/server/otel/internalTraceOtelWriter.ts
+++ b/packages/shared/src/server/otel/internalTraceOtelWriter.ts
@@ -15,15 +15,195 @@ import {
type ReadableSpan,
} from "@opentelemetry/sdk-trace-base";
+import { ObservationTypeDomain } from "../../domain/observations";
import type { InternalTraceEventInput } from "../llm/internalTraceEvents";
import { logger } from "../logger";
import { LangfuseOtelSpanAttributes } from "./attributes";
-import { OtelIngestionProcessor } from "./OtelIngestionProcessor";
+import {
+ OtelIngestionProcessor,
+ type ResourceSpan,
+} from "./OtelIngestionProcessor";
const INTERNAL_TRACE_WRITER_SDK_NAME = "langfuse-internal-otel-writer";
const INTERNAL_TRACE_WRITER_SCOPE = "langfuse-internal-trace-writer";
const W3C_TRACE_ID_PATTERN = /^[0-9a-f]{32}$/;
+function asOtelAttributeString(value: unknown): string | undefined {
+ if (value === undefined || value === null) return undefined;
+ return typeof value === "string" ? value : JSON.stringify(value);
+}
+
+function observationTypeToOtelAttribute(
+ type: string | undefined,
+): string | undefined {
+ const parsed = ObservationTypeDomain.safeParse(type?.toUpperCase());
+ return parsed.success ? parsed.data.toLowerCase() : undefined;
+}
+
+/**
+ * Maps an internal trace event onto the Langfuse OTel attribute names the
+ * ingestion processor already understands (`langfuse.observation.type`,
+ * generation fields, trace I/O on the root). Shared so type matching is not
+ * re-implemented per caller.
+ */
+export function internalTraceEventToOtelAttributes(
+ eventInput: Pick<
+ InternalTraceEventInput,
+ | "environment"
+ | "level"
+ | "statusMessage"
+ | "input"
+ | "output"
+ | "metadata"
+ | "type"
+ | "modelName"
+ | "modelParameters"
+ | "providedUsageDetails"
+ | "providedCostDetails"
+ | "promptName"
+ | "promptVersion"
+ | "completionStartTime"
+ | "userId"
+ | "sessionId"
+ | "tags"
+ | "public"
+ | "release"
+ | "traceName"
+ >,
+ options: { isRoot: boolean },
+): Attributes {
+ const observationType = observationTypeToOtelAttribute(eventInput.type);
+ const metadataJson = asOtelAttributeString(eventInput.metadata);
+ const { isRoot } = options;
+
+ return {
+ ...(eventInput.environment !== undefined
+ ? { [LangfuseOtelSpanAttributes.ENVIRONMENT]: eventInput.environment }
+ : {}),
+ ...(observationType
+ ? { [LangfuseOtelSpanAttributes.OBSERVATION_TYPE]: observationType }
+ : {}),
+ ...(eventInput.level !== undefined
+ ? { [LangfuseOtelSpanAttributes.OBSERVATION_LEVEL]: eventInput.level }
+ : {}),
+ ...(eventInput.statusMessage !== undefined
+ ? {
+ [LangfuseOtelSpanAttributes.OBSERVATION_STATUS_MESSAGE]:
+ eventInput.statusMessage,
+ }
+ : {}),
+ ...(eventInput.input !== undefined
+ ? { [LangfuseOtelSpanAttributes.OBSERVATION_INPUT]: eventInput.input }
+ : {}),
+ ...(eventInput.output !== undefined
+ ? { [LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT]: eventInput.output }
+ : {}),
+ ...(metadataJson !== undefined
+ ? { [LangfuseOtelSpanAttributes.OBSERVATION_METADATA]: metadataJson }
+ : {}),
+ ...(eventInput.modelName !== undefined
+ ? { [LangfuseOtelSpanAttributes.OBSERVATION_MODEL]: eventInput.modelName }
+ : {}),
+ ...(eventInput.modelParameters !== undefined
+ ? {
+ [LangfuseOtelSpanAttributes.OBSERVATION_MODEL_PARAMETERS]:
+ JSON.stringify(eventInput.modelParameters),
+ }
+ : {}),
+ ...(eventInput.providedUsageDetails !== undefined
+ ? {
+ [LangfuseOtelSpanAttributes.OBSERVATION_USAGE_DETAILS]:
+ JSON.stringify(eventInput.providedUsageDetails),
+ }
+ : {}),
+ ...(eventInput.providedCostDetails !== undefined
+ ? {
+ [LangfuseOtelSpanAttributes.OBSERVATION_COST_DETAILS]: JSON.stringify(
+ eventInput.providedCostDetails,
+ ),
+ }
+ : {}),
+ ...(eventInput.promptName !== undefined
+ ? {
+ [LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_NAME]:
+ eventInput.promptName,
+ }
+ : {}),
+ ...(eventInput.promptVersion !== undefined
+ ? {
+ [LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_VERSION]:
+ eventInput.promptVersion,
+ }
+ : {}),
+ ...(eventInput.completionStartTime !== undefined
+ ? {
+ [LangfuseOtelSpanAttributes.OBSERVATION_COMPLETION_START_TIME]:
+ eventInput.completionStartTime,
+ }
+ : {}),
+ ...(eventInput.userId !== undefined
+ ? { [LangfuseOtelSpanAttributes.TRACE_USER_ID]: eventInput.userId }
+ : {}),
+ ...(eventInput.sessionId !== undefined
+ ? { [LangfuseOtelSpanAttributes.TRACE_SESSION_ID]: eventInput.sessionId }
+ : {}),
+ ...(eventInput.tags !== undefined
+ ? {
+ [LangfuseOtelSpanAttributes.TRACE_TAGS]: JSON.stringify(
+ eventInput.tags,
+ ),
+ }
+ : {}),
+ ...(eventInput.public !== undefined
+ ? {
+ [LangfuseOtelSpanAttributes.TRACE_PUBLIC]: String(eventInput.public),
+ }
+ : {}),
+ ...(eventInput.release !== undefined
+ ? { [LangfuseOtelSpanAttributes.RELEASE]: eventInput.release }
+ : {}),
+ ...(isRoot ? { [LangfuseOtelSpanAttributes.IS_APP_ROOT]: "true" } : {}),
+ ...(isRoot && eventInput.traceName !== undefined
+ ? { [LangfuseOtelSpanAttributes.TRACE_NAME]: eventInput.traceName }
+ : {}),
+ ...(isRoot && eventInput.input !== undefined
+ ? { [LangfuseOtelSpanAttributes.TRACE_INPUT]: eventInput.input }
+ : {}),
+ ...(isRoot && eventInput.output !== undefined
+ ? { [LangfuseOtelSpanAttributes.TRACE_OUTPUT]: eventInput.output }
+ : {}),
+ ...(isRoot && metadataJson !== undefined
+ ? { [LangfuseOtelSpanAttributes.TRACE_METADATA]: metadataJson }
+ : {}),
+ };
+}
+
+/**
+ * Enqueues already-built OTLP resource spans on the regular OTel ingestion
+ * path. Callers choose `isLangfuseInternal` and `sdkName`; judge/code-eval
+ * stay internal, AI-feature product traces do not.
+ */
+export async function publishOtelResourceSpans(params: {
+ resourceSpans: ResourceSpan[];
+ projectId: string;
+ sdkName: string;
+ ingestionVersion?: string;
+ isLangfuseInternal: boolean;
+}): Promise<void> {
+ if (params.resourceSpans.length === 0) return;
+
+ const processor = new OtelIngestionProcessor({
+ projectId: params.projectId,
+ publicKey: "", // internal ingestion has no API key; mirrors internal event writes
+ sdkName: params.sdkName,
+ sdkVersion: "unknown",
+ ingestionVersion: params.ingestionVersion,
+ isLangfuseInternal: params.isLangfuseInternal,
+ });
+
+ await processor.publishToOtelIngestionQueue(params.resourceSpans);
+}
+
/**
* Publishes internally captured OTel spans through the regular OTel ingestion
* pipeline (same S3 + queue path as the public /api/public/otel/v1/traces
@@ -44,11 +224,10 @@ export async function publishInternalOtelSpans(params: {
if (!resourceSpans || resourceSpans.length === 0) return;
- const processor = new OtelIngestionProcessor({
+ await publishOtelResourceSpans({
+ resourceSpans,
projectId: params.projectId,
- publicKey: "", // internal ingestion has no API key; mirrors internal event writes
sdkName: params.sdkName,
- sdkVersion: "unknown",
// Opt into the v4-native direct events write like a modern SDK batch:
// only that path runs processToEvent -> createEventRecord, which is
// the sole extractor of langfuse.experiment.* into experiment_*
@@ -63,8 +242,6 @@ export async function publishInternalOtelSpans(params: {
// trace-upsert eval-loop guard.
isLangfuseInternal: true,
});
-
- await processor.publishToOtelIngestionQueue(resourceSpans);
}
/**
@@ -153,38 +330,9 @@ export async function writeInternalTraceViaOtelIngestion(trace: {
for (const eventInput of eventInputs) {
currentTraceId = eventInput.traceId.toLowerCase();
const isRoot = !eventInput.parentSpanId;
- const metadataJson = JSON.stringify(eventInput.metadata);
-
- const attributes: Attributes = {
- ...(eventInput.environment !== undefined
- ? { [LangfuseOtelSpanAttributes.ENVIRONMENT]: eventInput.environment }
- : {}),
- ...(eventInput.level !== undefined
- ? { [LangfuseOtelSpanAttributes.OBSERVATION_LEVEL]: eventInput.level }
- : {}),
- ...(eventInput.statusMessage !== undefined
- ? {
- [LangfuseOtelSpanAttributes.OBSERVATION_STATUS_MESSAGE]:
- eventInput.statusMessage,
- }
- : {}),
- ...(eventInput.input !== undefined
- ? { [LangfuseOtelSpanAttributes.OBSERVATION_INPUT]: eventInput.input }
- : {}),
- ...(eventInput.output !== undefined
- ? {
- [LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT]:
- eventInput.output,
- }
- : {}),
- [LangfuseOtelSpanAttributes.OBSERVATION_METADATA]: metadataJson,
- ...(isRoot && eventInput.traceName !== undefined
- ? { [LangfuseOtelSpanAttributes.TRACE_NAME]: eventInput.traceName }
- : {}),
- ...(isRoot
- ? { [LangfuseOtelSpanAttributes.TRACE_METADATA]: metadataJson }
- : {}),
- };
+ const attributes = internalTraceEventToOtelAttributes(eventInput, {
+ isRoot,
+ });
let context = ROOT_CONTEXT;
if (eventInput.parentSpanId) {
diff --git a/web/src/features/in-app-agent/server/router.ts b/web/src/features/in-app-agent/server/router.ts
index f1ccc84c79..77e477c6fe 100644
--- a/web/src/features/in-app-agent/server/router.ts
+++ b/web/src/features/in-app-agent/server/router.ts
@@ -16,6 +16,7 @@ import {
AgUiContextSchema,
getInAppAgentInstrumentationObservationId,
getInAppAgentInstrumentationTraceId,
+ IN_APP_AGENT_PRODUCT_ENVIRONMENT,
} from "@langfuse/shared/in-app-agent";
import { InAppAgentMessageFeedbackValueSchema } from "../schema";
import {
@@ -95,7 +96,7 @@ const SubmitFeedbackInput = ConversationIdInput.extend({
});
const IN_APP_AGENT_FEEDBACK_SCORE_NAME = "in_app_agent_feedback";
-const IN_APP_AGENT_FEEDBACK_ENVIRONMENT = "langfuse-in-app-agent";
+const IN_APP_AGENT_FEEDBACK_ENVIRONMENT = IN_APP_AGENT_PRODUCT_ENVIRONMENT;
export const inAppAgentRouter = createTRPCRouter({
// Each row carries its newest run summary for the activity poll and badges.
diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.ts
index e9d2b11eb9..71d6cff860 100644
--- a/worker/src/features/in-app-agent/executeInAppAgentRun.ts
+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.ts
@@ -16,6 +16,7 @@ import {
InAppAgentRunRequestSchema,
InAppAgentRunStatus,
getInAppAgentInstrumentationTraceId,
+ IN_APP_AGENT_PRODUCT_ENVIRONMENT,
parseInAppAgentInterruptEvent,
type AgUiEvent,
type InAppAgentRunRequest,
@@ -158,7 +159,7 @@ export async function executeInAppAgentRun(params: {
// The uncovered durability window: an approved mutation may have started
// but its result never persisted. Never generically retried. Hoisted to
// function scope because both the loop callbacks and the outer catch can
- // record the terminal state (failStream races the drain rejection).
+ // record the terminal state.
const failureCode = () =>
isApprovedContinuation && !approvedToolResultPersisted
? {
@@ -168,9 +169,9 @@ export async function executeInAppAgentRun(params: {
}
: undefined;
- // Single-flight (same shape as web's withInAppAgentMcpApiKeyCleanup): the
- // loop's onFinish and the outer catch can invoke this concurrently because
- // failStream errors the stream without awaiting its onError/onFinish chain.
+ // Single-flight (same shape as web's withInAppAgentMcpApiKeyCleanup):
+ // onFinish cleans up before the stream errors, then the outer catch
+ // runs after drainStream rejects and may call this again.
const cleanupMcpApiKey = (): Promise<void> => {
if (!mcpApiKey) return Promise.resolve();
const keyId = mcpApiKey.id;
@@ -629,12 +630,12 @@ export async function executeInAppAgentRun(params: {
await drainStream(stream);
} catch (error) {
// Two classes land here: pre-loop failures (revalidation, input build,
- // key minting) and loop failures surfaced through the errored stream —
- // failStream fires the loop's onError without awaiting it and errors the
- // stream synchronously, so this catch can RACE that onError and win the
- // terminal CAS. Both writers therefore classify identically: the
- // durability check first, then agent_error for loop-phase failures and
- // init_failed only before the loop existed.
+ // key minting) and loop failures surfaced through the errored stream
+ // after tracing flush, onError, and onFinish. finishClaimedRun is
+ // idempotent, so a second write is a no-op if onError already recorded
+ // FAILED. Both writers classify identically: the durability check first,
+ // then agent_error for loop-phase failures and init_failed only before
+ // the loop existed.
await finishClaimedRun({
prisma,
projectId,
@@ -768,7 +769,7 @@ function buildTracingConfig(params: {
}
const traceSinkParams = getLangfuseAITraceSinkParams({
- environment: "langfuse-in-app-agent",
+ environment: IN_APP_AGENT_PRODUCT_ENVIRONMENT,
feature: "in-app-agent",
projectId: params.projectId,
traceId: getInAppAgentInstrumentationTraceId(params.runId),
diff --git a/worker/src/features/in-app-agent/runtime/agent.test.ts b/worker/src/features/in-app-agent/runtime/agent.test.ts
index b3e0bb7503..f99ddd1a6f 100644
--- a/worker/src/features/in-app-agent/runtime/agent.test.ts
+++ b/worker/src/features/in-app-agent/runtime/agent.test.ts
@@ -1309,6 +1309,71 @@ describe("createAgUiStream", () => {
expect(instrumentationMocks.instrumentation.flush).toHaveBeenCalled();
});
+ it("waits for tracing flush before finishing an aborted agent stream", async () => {
+ const { createAgUiStream } = await import("./agent");
+ let resolveFlush: (() => void) | undefined;
+ instrumentationMocks.instrumentation.flush.mockReturnValue(
+ new Promise<void>((resolve) => {
+ resolveFlush = resolve;
+ }),
+ );
+ const input = {
+ threadId: "conversation-1",
+ runId: "run-flush-abort",
+ messages: [
+ {
+ id: "user-message-1",
+ role: "user" as const,
+ content: "hello",
+ },
+ ],
+ tools: [],
+ context: [],
+ state: {
+ type: "existingConversation" as const,
+ projectId: "project-1",
+ conversationId: "conversation-1",
+ },
+ forwardedProps: {},
+ };
+ adapterEvents.items = [];
+ const onAbort = vi.fn();
+ const abortController = new AbortController();
+ abortController.abort("worker_shutdown");
+
+ const stream = await createAgUiStream({
+ input,
+ signal: abortController.signal,
+ options: {
+ onAbort,
+ model: testBedrockModel("test-model"),
+ langfuseMcp: {
+ url: "https://example.com/api/public/mcp",
+ publicKey: "pk",
+ secretKey: "sk",
+ toolPolicy: defaultInAppAgentToolPolicy,
+ },
+ redirectAction: { projectId: "project-1", isV4Enabled: false },
+ langfuseClient: {
+ getPrompt: promptMocks.getPrompt,
+ } as unknown as Langfuse,
+ useLocalPrompt: false,
+ langfuseTracing: createTestTracingConfig(),
+ },
+ });
+
+ const streamDone = readStream(stream);
+ await vi.waitFor(() => {
+ expect(instrumentationMocks.instrumentation.flush).toHaveBeenCalled();
+ });
+ expect(onAbort).not.toHaveBeenCalled();
+
+ resolveFlush?.();
+ await streamDone;
+
+ expect(onAbort).toHaveBeenCalledOnce();
+ });
+
it("does not enable Bedrock reasoning for non-Claude models", async () => {
const { createAgUiStream } = await import("./agent");
const input = {
diff --git a/worker/src/features/in-app-agent/runtime/agent.ts b/worker/src/features/in-app-agent/runtime/agent.ts
index 755384169d..01b244d50a 100644
--- a/worker/src/features/in-app-agent/runtime/agent.ts
+++ b/worker/src/features/in-app-agent/runtime/agent.ts
@@ -340,14 +340,21 @@ export async function createAgUiStream(params: {
operation: string,
callback: (
activeInstrumentation: NonNullable<typeof instrumentation>,
- ) => void,
- ) => {
+ ) => void | Promise<void>,
+ ): Promise<void> => {
if (!instrumentation) {
- return;
+ return Promise.resolve();
}
try {
- callback(instrumentation);
+ return Promise.resolve(callback(instrumentation)).catch((error) => {
+ logger.warn("Failed to record in-app agent Langfuse tracing", {
+ error,
+ operation,
+ runId: params.input.runId,
+ threadId: params.input.threadId,
+ });
+ });
} catch (error) {
logger.warn("Failed to record in-app agent Langfuse tracing", {
error,
@@ -355,6 +362,7 @@ export async function createAgUiStream(params: {
runId: params.input.runId,
threadId: params.input.threadId,
});
+ return Promise.resolve();
}
};
recordInstrumentation("recordAvailableSkills", (instrumentation) =>
@@ -473,7 +481,7 @@ export async function createAgUiStream(params: {
recordInstrumentation("endWithError", (instrumentation) =>
instrumentation.endWithError(error),
);
- recordInstrumentation("flush", (instrumentation) =>
+ const flushPromise = recordInstrumentation("flush", (instrumentation) =>
instrumentation.flush(),
);
ending = true;
@@ -491,18 +499,25 @@ export async function createAgUiStream(params: {
});
runTerminalCallback(
- () => params.options.onError?.(error),
- "Error while marking agent stream as failed",
+ () => flushPromise,
+ "Error while flushing agent stream tracing after failure",
)
+ .then(() =>
+ runTerminalCallback(
+ () => params.options.onError?.(error),
+ "Error while marking agent stream as failed",
+ ),
+ )
.then(() =>
runTerminalCallback(
runOnFinish,
"Error while running agent stream finish callback after failure",
),
)
+ .then(() => {
+ controller.error(error);
+ })
.finally(finish);
-
- controller.error(error);
};
const enqueueEvent = (
@@ -582,7 +597,7 @@ export async function createAgUiStream(params: {
recordInstrumentation("end", (instrumentation) =>
instrumentation.end({ aborted: true }),
);
- recordInstrumentation("flush", (instrumentation) =>
+ const flushPromise = recordInstrumentation("flush", (instrumentation) =>
instrumentation.flush(),
);
ending = true;
@@ -591,6 +606,12 @@ export async function createAgUiStream(params: {
interruptAdapter?.();
subscription?.unsubscribe();
eventQueue
+ .then(() =>
+ runTerminalCallback(
+ () => flushPromise,
+ "Error while flushing agent stream tracing after abort",
+ ),
+ )
.then(() =>
runTerminalCallback(
() => params.options.onAbort?.(),
@@ -826,8 +847,8 @@ export async function createAgUiStream(params: {
}
if (streamedRunError !== null) {
- closeController(() => {
- recordInstrumentation("flush", (instrumentation) =>
+ closeController(async () => {
+ await recordInstrumentation("flush", (instrumentation) =>
instrumentation.flush(),
);
return handleStreamedRunError();
@@ -866,7 +887,7 @@ export async function createAgUiStream(params: {
closeController(
streamedRunError === null
- ? () => {
+ ? async () => {
const truncatedByStepLimit =
isTruncatedByStepLimit(stepLimitState);
recordInstrumentation("end", (instrumentation) =>
@@ -881,7 +902,7 @@ export async function createAgUiStream(params: {
: {},
),
);
- recordInstrumentation("flush", (instrumentation) =>
+ await recordInstrumentation("flush", (instrumentation) =>
instrumentation.flush(),
);
return params.options.onComplete?.({
@@ -889,8 +910,8 @@ export async function createAgUiStream(params: {
truncatedByStepLimit,
});
}
- : () => {
- recordInstrumentation("flush", (instrumentation) =>
+ : async () => {
+ await recordInstrumentation("flush", (instrumentation) =>
instrumentation.flush(),
);
return handleStreamedRunError();
@@ -936,7 +957,7 @@ export async function createAgUiStream(params: {
recordInstrumentation("end", (instrumentation) =>
instrumentation.end({ aborted: true }),
);
- recordInstrumentation("flush", (instrumentation) =>
+ const flushPromise = recordInstrumentation("flush", (instrumentation) =>
instrumentation.flush(),
);
shouldEnqueue = false;
@@ -944,6 +965,12 @@ export async function createAgUiStream(params: {
interruptAdapter?.();
subscription?.unsubscribe();
eventQueue
+ .then(() =>
+ runTerminalCallback(
+ () => flushPromise,
+ "Error while flushing agent stream tracing after cancel",
+ ),
+ )
.then(() =>
runTerminalCallback(
() => params.options.onAbort?.(),
diff --git a/worker/src/features/in-app-agent/runtime/instrumentation.test.ts b/worker/src/features/in-app-agent/runtime/instrumentation.test.ts
index b2731e6756..fd6cdf3c9f 100644
--- a/worker/src/features/in-app-agent/runtime/instrumentation.test.ts
+++ b/worker/src/features/in-app-agent/runtime/instrumentation.test.ts
@@ -64,6 +64,7 @@ const input: AgUiRunAgentInput = {
const expectedAgentRunInput = {
messages: [{ role: "user", content: "hello" }],
};
+const expectedWrappingRootInput = "hello";
describe("InAppAgentInstrumentation", () => {
beforeEach(() => {
@@ -139,6 +140,7 @@ describe("InAppAgentInstrumentation", () => {
traceId,
traceName: "agent-turn",
userId: "user-1",
+ sessionId: "conversation-1",
}),
);
expect(mocks.handler.langfuse.trace).toHaveBeenCalledWith(
@@ -146,7 +148,7 @@ describe("InAppAgentInstrumentation", () => {
);
expect(mocks.handler.langfuse.trace).toHaveBeenCalledWith(
expect.objectContaining({
- input: expectedAgentRunInput,
+ input: expectedWrappingRootInput,
}),
);
expect(mocks.trace.generation).not.toHaveBeenCalled();
@@ -157,6 +159,8 @@ describe("InAppAgentInstrumentation", () => {
traceId,
parentObservationId: agentRunObservationId,
name: "listObservations",
+ userId: "user-1",
+ sessionId: "conversation-1",
input: { limit: 10 },
output: toolOutput,
startTime: expect.any(Date),
@@ -210,24 +214,8 @@ describe("InAppAgentInstrumentation", () => {
);
expect(mocks.trace.update).toHaveBeenCalledWith(
expect.objectContaining({
- input: expectedAgentRunInput,
- output: {
- messages: [
- { role: "assistant", content: "hi there" },
- {
- role: "assistant",
- content: "",
- tool_calls: [toolCall],
- },
- {
- role: "tool",
- tool_call_id: "tool-1",
- content: toolOutput,
- },
- ],
- text: "hi there",
- tool_calls: [toolCall],
- },
+ input: expectedWrappingRootInput,
+ output: "hi there",
}),
);
});
@@ -785,11 +773,8 @@ describe("InAppAgentInstrumentation", () => {
);
expect(mocks.trace.update).toHaveBeenCalledWith(
expect.objectContaining({
- input: expectedAgentRunInput,
- output: {
- messages: [{ role: "assistant", content: "second turn output" }],
- text: "second turn output",
- },
+ input: expectedWrappingRootInput,
+ output: "second turn output",
}),
);
});
@@ -815,6 +800,12 @@ describe("InAppAgentInstrumentation", () => {
expect(mocks.trace.generation).not.toHaveBeenCalled();
expect(mocks.handler.langfuse.trace).toHaveBeenCalledWith(
+ expect.objectContaining({
+ input: expectedWrappingRootInput,
+ }),
+ );
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "agent-create",
expect.objectContaining({
input: {
messages: [{ role: "user", content: "hello" }],
@@ -875,6 +866,12 @@ describe("InAppAgentInstrumentation", () => {
expect(mocks.handler.langfuse.trace).toHaveBeenCalledWith(
expect.objectContaining({
name: "agent-turn",
+ input: expectedWrappingRootInput,
+ }),
+ );
+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(
+ "agent-create",
+ expect.objectContaining({
input: {
messages: [{ role: "user", content: "hello" }],
},
@@ -985,7 +982,7 @@ describe("InAppAgentInstrumentation", () => {
name: "agent-turn",
sessionId: "conversation-1",
timestamp: new Date(traceStartedAt),
- input: expectedAgentRunInput,
+ input: expectedWrappingRootInput,
tags: ["in-app-agent"],
metadata: expect.objectContaining({
approval_continuation_count: 2,
@@ -1046,7 +1043,7 @@ describe("InAppAgentInstrumentation", () => {
);
expect(mocks.trace.update).toHaveBeenCalledWith(
expect.objectContaining({
- input: expectedAgentRunInput,
+ input: expectedWrappingRootInput,
metadata: expect.objectContaining({
approval_continuation_count: 2,
}),
@@ -1106,6 +1103,12 @@ describe("InAppAgentInstrumentation", () => {
}),
}),
);
+ expect(mocks.trace.update).toHaveBeenCalledWith(
+ expect.objectContaining({
+ input: expectedWrappingRootInput,
+ output: "I need your approval.\n\nThe tool completed.",
+ }),
+ );
});
it("compacts text message chunks before recording output", () => {
@@ -1585,7 +1588,7 @@ describe("InAppAgentInstrumentation", () => {
expect(toolCreates[0]?.[1]).toEqual(
expect.objectContaining({
id: "tool-a",
- parentObservationId: agentRunObservationId,
+ parentObservationId: getInAppAgentLlmCallObservationId(runId, 1),
startTime: toolAStart,
endTime: toolAEnd,
}),
@@ -1593,7 +1596,7 @@ describe("InAppAgentInstrumentation", () => {
expect(toolCreates[1]?.[1]).toEqual(
expect.objectContaining({
id: "tool-b",
- parentObservationId: agentRunObservationId,
+ parentObservationId: getInAppAgentLlmCallObservationId(runId, 1),
startTime: toolBStart,
endTime: toolBEnd,
}),
diff --git a/worker/src/features/in-app-agent/runtime/instrumentation.ts b/worker/src/features/in-app-agent/runtime/instrumentation.ts
index 4f708e93e6..da4ed7d4a2 100644
--- a/worker/src/features/in-app-agent/runtime/instrumentation.ts
+++ b/worker/src/features/in-app-agent/runtime/instrumentation.ts
@@ -104,6 +104,8 @@ type ObservationBody = {
metadata?: Record<string, unknown>;
promptName?: string;
promptVersion?: number;
+ userId?: string;
+ sessionId?: string;
};
type ToolCallApprovalStatus = "approved" | "rejected";
type AgentRunToolSpan = {
@@ -177,14 +179,18 @@ export function createInAppAgentInstrumentation({
export class InAppAgentInstrumentation {
private readonly processTracedEvents: () => Promise<void>;
+ private flushPromise?: Promise<void>;
private readonly langfuse: InAppAgentLangfuse;
private readonly trace: InAppAgentTrace;
private readonly runId: string;
private readonly traceId: string;
private readonly rootObservationId: string;
+ private readonly userId: string;
+ private readonly sessionId: string;
private readonly traceStartTime: Date;
private readonly agentRunStartTime: Date;
private agentRunInput: unknown;
+ private readonly wrappingRootInput: string | undefined;
private readonly approvalContinuation?: ApprovalContinuation;
private readonly prompt?: InAppAgentPromptMetadata;
private readonly toolSpans = new Map<string, AgentRunToolSpan>();
@@ -197,6 +203,9 @@ export class InAppAgentInstrumentation {
InAppAgentToolApprovalSource
>();
private readonly toolExecutionTimes = new Map<string, ToolExecutionTimes>();
+ private readonly toolCallToModelObservationId = new Map<string, string>();
+ private openModelObservationId?: string;
+ private lastModelObservationId?: string;
private readonly metadata: Record<string, unknown>;
private readonly agentRunOutputMessages: AgentRunChatMessage[] = [];
private readonly agentRunToolCalls: AgentRunToolCall[] = [];
@@ -243,9 +252,12 @@ export class InAppAgentInstrumentation {
: {}),
};
this.agentRunInput = getAgentRunInput(params.input);
+ this.wrappingRootInput = getWrappingRootUserText(params.input);
this.prompt = params.prompt;
this.model = params.model;
this.runId = params.runId;
+ this.userId = params.userId;
+ this.sessionId = params.input.threadId;
const rootRunId = this.approvalContinuation?.rootRunId ?? params.runId;
this.traceId = getInAppAgentInstrumentationTraceId(rootRunId);
this.rootObservationId =
@@ -260,8 +272,10 @@ export class InAppAgentInstrumentation {
traceName: IN_APP_AGENT_TURN_NAME,
environment: params.environment,
userId: params.userId,
+ sessionId: params.input.threadId,
metadata: this.metadata,
prompt: params.prompt,
+ aiFeatureOtelIngestion: true,
};
const { handler, processTracedEvents } =
getInternalTracingHandler(traceSinkParams);
@@ -274,7 +288,7 @@ export class InAppAgentInstrumentation {
userId: params.userId,
sessionId: params.input.threadId,
timestamp: this.traceStartTime,
- input: this.getTraceInput(),
+ input: this.wrappingRootInput,
metadata: this.metadata,
tags: ["in-app-agent"],
});
@@ -352,8 +366,14 @@ export class InAppAgentInstrumentation {
}
const now = new Date();
+ const callNumber =
+ this.openModelCall?.callNumber ?? ++this.nextModelCallNumber;
+ this.openModelObservationId = getInAppAgentLlmCallObservationId(
+ this.runId,
+ callNumber,
+ );
this.openModelCall = {
- callNumber: this.openModelCall?.callNumber ?? ++this.nextModelCallNumber,
+ callNumber,
startTime: this.openModelCall?.startTime ?? now,
input: getModelCallInput(options),
textDeltas: [],
@@ -450,8 +470,8 @@ export class InAppAgentInstrumentation {
},
});
this.trace.update({
- input: this.getTraceInput(),
- output: this.getAgentRunOutput(),
+ input: this.wrappingRootInput,
+ output: this.getWrappingRootOutput(),
metadata: { ...this.metadata, error: message },
});
this.ended = true;
@@ -476,17 +496,19 @@ export class InAppAgentInstrumentation {
};
this.emitRootAgentObservation({ metadata });
this.trace.update({
- input: this.getTraceInput(),
- output: this.getAgentRunOutput(),
+ input: this.wrappingRootInput,
+ output: this.getWrappingRootOutput(),
metadata,
});
this.ended = true;
}
- flush() {
- this.processTracedEvents().catch((error) => {
+ flush(): Promise<void> {
+ this.flushPromise ??= this.processTracedEvents().catch((error) => {
logger.warn("Failed to flush in-app agent Langfuse tracing", error);
});
+
+ return this.flushPromise;
}
private recordEvent(event: AgUiEvent) {
@@ -693,7 +715,7 @@ export class InAppAgentInstrumentation {
const body: ObservationBody = {
id: toolCallId,
traceId: this.traceId,
- parentObservationId: this.rootObservationId,
+ parentObservationId: this.getToolParentObservationId(toolCallId),
name: tool.name,
startTime,
endTime,
@@ -761,6 +783,11 @@ export class InAppAgentInstrumentation {
const finishReason = getModelFinishReason(finish);
const model = call.modelId ?? this.model;
const id = getInAppAgentLlmCallObservationId(this.runId, call.callNumber);
+ this.lastModelObservationId = id;
+ this.openModelObservationId = undefined;
+ for (const toolCall of call.toolCalls) {
+ this.toolCallToModelObservationId.set(toolCall.toolCallId, id);
+ }
const text = call.textDeltas.join("");
const output =
text || call.toolCalls.length > 0
@@ -868,12 +895,25 @@ export class InAppAgentInstrumentation {
}
}
+ private getToolParentObservationId(toolCallId: string): string {
+ return (
+ this.toolCallToModelObservationId.get(toolCallId) ??
+ this.openModelObservationId ??
+ this.lastModelObservationId ??
+ this.rootObservationId
+ );
+ }
+
private enqueueObservation(type: string, body: ObservationBody) {
(
this.langfuse as unknown as {
enqueue: (type: string, body: ObservationBody) => void;
}
- ).enqueue(type, body);
+ ).enqueue(type, {
+ ...body,
+ userId: this.userId,
+ sessionId: this.sessionId,
+ });
}
// Reasoning that no assistant message followed (e.g. aborted or errored
@@ -970,17 +1010,34 @@ export class InAppAgentInstrumentation {
}
}
+ private getWrappingRootOutput(): string | undefined {
+ const texts = this.getAgentRunOutputMessages().flatMap((message) => {
+ if (message.role !== "assistant") {
+ return [];
+ }
+
+ const text = getRenderedMessageText(message.content);
+ return text.trim() ? [text] : [];
+ });
+
+ if (texts.length === 0) {
+ return undefined;
+ }
+
+ return texts.join("\n\n");
+ }
+
private getAgentRunOutput() {
- const priorMessages =
- this.approvalContinuation && isRecord(this.agentRunInput)
- ? getPriorAgentRunOutputMessages(this.agentRunInput.messages)
- : [];
- const messages = [...priorMessages, ...this.agentRunOutputMessages];
+ const messages = this.getAgentRunOutputMessages();
if (messages.length === 0) {
return undefined;
}
+ const priorMessages = messages.slice(
+ 0,
+ messages.length - this.agentRunOutputMessages.length,
+ );
const priorToolCalls = priorMessages.flatMap(
(message) => message.tool_calls ?? [],
);
@@ -993,6 +1050,15 @@ export class InAppAgentInstrumentation {
};
}
+ private getAgentRunOutputMessages(): AgentRunChatMessage[] {
+ const priorMessages =
+ this.approvalContinuation && isRecord(this.agentRunInput)
+ ? getPriorAgentRunOutputMessages(this.agentRunInput.messages)
+ : [];
+
+ return [...priorMessages, ...this.agentRunOutputMessages];
+ }
+
private emitApprovalWaitObservation() {
const continuation = this.approvalContinuation;
if (!continuation) {
@@ -1088,6 +1154,37 @@ function getPriorAgentRunOutputMessages(
);
}
+function getWrappingRootUserText(input: AgUiRunAgentInput): string | undefined {
+ const lastUserMessage = input.messages.findLast(
+ (message) => message.role === "user",
+ );
+
+ if (!lastUserMessage) {
+ return undefined;
+ }
+
+ const text = getRenderedMessageText(lastUserMessage.content);
+ return text.length > 0 ? text : undefined;
+}
+
+function getRenderedMessageText(content: unknown): string {
+ if (typeof content === "string") {
+ return content;
+ }
+
+ if (Array.isArray(content)) {
+ return content
+ .flatMap((part) =>
+ isRecord(part) && part.type === "text" && typeof part.text === "string"
+ ? [part.text]
+ : [],
+ )
+ .join("");
+ }
+
+ return "";
+}
+
function getAgentRunInput(input: AgUiRunAgentInput): unknown {
const messages = getAgentRunMessages(getCurrentTurnMessages(input.messages));
const context = getAgentRunContext(input);
diff --git a/worker/src/features/internal-tracing/__tests__/getInternalTracingHandler.test.ts b/worker/src/features/internal-tracing/__tests__/getInternalTracingHandler.test.ts
index bc03e62fec..f5c9629a34 100644
--- a/worker/src/features/internal-tracing/__tests__/getInternalTracingHandler.test.ts
+++ b/worker/src/features/internal-tracing/__tests__/getInternalTracingHandler.test.ts
@@ -1,12 +1,16 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
-const { mockExportLocalEvents, mockProcessEventBatch, mockWrite } = vi.hoisted(
- () => ({
- mockExportLocalEvents: vi.fn(),
- mockProcessEventBatch: vi.fn(),
- mockWrite: vi.fn(),
- }),
-);
+const {
+ mockExportLocalEvents,
+ mockProcessEventBatch,
+ mockWrite,
+ mockPublishAiFeatureTrace,
+} = vi.hoisted(() => ({
+ mockExportLocalEvents: vi.fn(),
+ mockProcessEventBatch: vi.fn(),
+ mockWrite: vi.fn(),
+ mockPublishAiFeatureTrace: vi.fn(),
+}));
vi.mock("langfuse", () => {
return {
@@ -23,8 +27,16 @@ vi.mock(
}),
);
+vi.mock(
+ "../../../../../packages/shared/src/server/otel/aiFeatureOtelWriter",
+ () => ({
+ publishAiFeatureTraceViaOtelIngestion: mockPublishAiFeatureTrace,
+ }),
+);
+
import { getInternalTracingHandler } from "../../../../../packages/shared/src/server/llm/getInternalTracingHandler";
import { LangfuseInternalTraceEnvironment } from "@langfuse/shared";
+import { env } from "../../../../../packages/shared/src/env";
const traceId = "trace-123";
const processedEventsFixture = [
@@ -80,6 +92,7 @@ describe("getInternalTracingHandler", () => {
errors: [],
});
mockWrite.mockResolvedValue(undefined);
+ mockPublishAiFeatureTrace.mockResolvedValue(undefined);
});
it("disables staging propagation when direct event write is enabled", async () => {
@@ -132,4 +145,51 @@ describe("getInternalTracingHandler", () => {
);
expect(mockWrite).not.toHaveBeenCalled();
});
+
+ it("publishes AI-feature traces via OTel on v4 write modes", async () => {
+ const previousWriteMode = env.LANGFUSE_MIGRATION_V4_WRITE_MODE;
+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "events_only";
+
+ try {
+ const { processTracedEvents } = getInternalTracingHandler({
+ targetProjectId: "project-123",
+ traceId,
+ traceName: "agent-turn",
+ environment: "production",
+ aiFeatureOtelIngestion: true,
+ });
+
+ await processTracedEvents();
+
+ expect(mockProcessEventBatch).not.toHaveBeenCalled();
+ expect(mockPublishAiFeatureTrace).toHaveBeenCalledWith({
+ projectId: "project-123",
+ eventInputs: expect.any(Array),
+ });
+ } finally {
+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE = previousWriteMode;
+ }
+ });
+
+ it("keeps processEventBatch for AI-feature traces in legacy write mode", async () => {
+ const previousWriteMode = env.LANGFUSE_MIGRATION_V4_WRITE_MODE;
+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "legacy";
+
+ try {
+ const { processTracedEvents } = getInternalTracingHandler({
+ targetProjectId: "project-123",
+ traceId,
+ traceName: "agent-turn",
+ environment: "production",
+ aiFeatureOtelIngestion: true,
+ });
+
+ await processTracedEvents();
+
+ expect(mockProcessEventBatch).toHaveBeenCalled();
+ expect(mockPublishAiFeatureTrace).not.toHaveBeenCalled();
+ } finally {
+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE = previousWriteMode;
+ }
+ });
});