🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(in-app-agent): ingest AI feature traces (starting with in-app agent, other features will follow) via v4 OTel as production (#16697)
Lysander Kiesela19768430aeb
diff --git a/packages/shared/src/in-app-agent/constants.ts b/packages/shared/src/in-app-agent/constants.tsindex 7fbaa1b81d..268bc80338 100644--- a/packages/shared/src/in-app-agent/constants.ts+++ b/packages/shared/src/in-app-agent/constants.ts@@ -63,3 +63,7 @@ export const getInAppAgentLlmCallObservationId = (runId: string,stepNumber: number,) => `${runId}-llm-${stepNumber}`;++// Product traces and feedback scores for the in-app agent. Must match so+// scores attach to the same traces evaluators in the AI-features project see.+export const IN_APP_AGENT_PRODUCT_ENVIRONMENT = "production";diff --git a/packages/shared/src/server/ingestion/ingestionAttribution.ts b/packages/shared/src/server/ingestion/ingestionAttribution.tsindex 681fd2fd72..78ed508fa8 100644--- a/packages/shared/src/server/ingestion/ingestionAttribution.ts+++ b/packages/shared/src/server/ingestion/ingestionAttribution.ts@@ -15,6 +15,7 @@ export const UNKNOWN_INGESTION_SDK_VALUE = "unknown";export const INTERNAL_INGESTION_SDK_NAMES = ["langfuse-internal-ai-sdk","langfuse-internal-otel-writer",+ "langfuse-internal-ai-features",] as const;export const LANGFUSE_SDK_LATEST_MAJOR = {diff --git a/packages/shared/src/server/llm/getInternalTracingHandler.ts b/packages/shared/src/server/llm/getInternalTracingHandler.tsindex 55b1ff0b59..93bacaa015 100644--- a/packages/shared/src/server/llm/getInternalTracingHandler.ts+++ b/packages/shared/src/server/llm/getInternalTracingHandler.ts@@ -1,10 +1,12 @@import { Langfuse } from "langfuse";+import { env } from "../../env";import { ProcessedTraceEvent, TraceSinkParams } from "./types";import { buildInternalTraceEventInputs } from "./internalTraceEvents";import { processEventBatch } from "../ingestion/processEventBatch";import { createUnknownSdkIngestionAttribution } from "../ingestion/ingestionAttribution";import { logger } from "../logger";import { traceException } from "../instrumentation";+import { publishAiFeatureTraceViaOtelIngestion } from "../otel/aiFeatureOtelWriter";export function prepareInternalTraceEvents(params: {events: Array<{@@ -99,6 +101,39 @@ export function getInternalTracingHandler(traceSinkParams: TraceSinkParams): {prompt,});+ const useAiFeatureOtel =+ Boolean(traceSinkParams.aiFeatureOtelIngestion) &&+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE !== "legacy";++ if (useAiFeatureOtel) {+ try {+ const { eventInputs } = buildInternalTraceEventInputs({+ processedEvents,+ traceId: traceSinkParams.traceId,+ projectId: targetProjectId,+ userId: traceSinkParams.userId,+ sessionId: traceSinkParams.sessionId,+ });++ if (eventInputs.length > 0) {+ await publishAiFeatureTraceViaOtelIngestion({+ eventInputs,+ projectId: targetProjectId,+ });+ }+ } catch (otelError) {+ traceException(otelError);+ logger.error(+ "Failed to process AI-feature traces via OTel ingestion",+ {+ error: otelError,+ },+ );+ }++ return;+ }+// Legacy write to traces/observations tablestry {const auth = {@@ -134,6 +169,8 @@ export function getInternalTracingHandler(traceSinkParams: TraceSinkParams): {processedEvents,traceId: traceSinkParams.traceId,projectId: targetProjectId,+ userId: traceSinkParams.userId,+ sessionId: traceSinkParams.sessionId,experimentContext: eventsWriter.experimentContext,});diff --git a/packages/shared/src/server/llm/internalTraceEvents.ts b/packages/shared/src/server/llm/internalTraceEvents.tsindex df7895bba9..5b4496c711 100644--- a/packages/shared/src/server/llm/internalTraceEvents.ts+++ b/packages/shared/src/server/llm/internalTraceEvents.ts@@ -6,6 +6,10 @@ import {asStringArray,} from "../../utils/objects";import { stringifyValue } from "../../utils/stringChecks";+import {+ ObservationTypeDomain,+ type ObservationType,+} from "../../domain/observations";import {convertCallsToArrays,convertDefinitionsToMap,@@ -108,7 +112,7 @@ type InternalTraceSnapshot = {traceId: string;parentSpanId?: string;name?: string;- type: "SPAN" | "GENERATION";+ type: ObservationType;environment?: string;version?: string;release?: string;@@ -142,8 +146,18 @@ function isCreateEvent(type: string): boolean {return type.endsWith("-create");}-function getSnapshotType(eventType: string): "SPAN" | "GENERATION" {- return eventType.startsWith("generation-") ? "GENERATION" : "SPAN";+function getSnapshotType(eventType: string): ObservationType {+ const prefix = eventType.replace(/-(create|update)$/i, "").toUpperCase();+ const parsed = ObservationTypeDomain.safeParse(prefix);+ return parsed.success ? parsed.data : "SPAN";+}++function asIsoTimestamp(value: unknown): string | undefined {+ if (value instanceof Date && !Number.isNaN(value.getTime())) {+ return value.toISOString();+ }++ return asString(value);}function getEventTime(@@ -153,6 +167,10 @@ function getEventTime(const candidates = [body.startTime, body.timestamp, event.timestamp];for (const candidate of candidates) {+ if (candidate instanceof Date && !Number.isNaN(candidate.getTime())) {+ return candidate.getTime();+ }+if (typeof candidate === "string") {const parsed = new Date(candidate).getTime();if (!Number.isNaN(parsed)) {@@ -205,8 +223,9 @@ function mergeSnapshotEvent(event: ProcessedTraceEvent,): InternalTraceSnapshot {const { body } = event;- const startTime = asString(body.startTime);- const timestamp = asString(body.timestamp) ?? asString(event.timestamp);+ const startTime = asIsoTimestamp(body.startTime);+ const timestamp =+ asIsoTimestamp(body.timestamp) ?? asIsoTimestamp(event.timestamp);const metadata = asRecord(body.metadata);return {@@ -214,18 +233,16 @@ function mergeSnapshotEvent(traceId: asString(body.traceId) ?? snapshot.traceId,parentSpanId: asString(body.parentObservationId) ?? snapshot.parentSpanId,type:- snapshot.type === "GENERATION"- ? snapshot.type- : getSnapshotType(event.type),+ snapshot.type !== "SPAN" ? snapshot.type : getSnapshotType(event.type),name: asString(body.name) ?? snapshot.name,environment: asString(body.environment) ?? snapshot.environment,version: asString(body.version) ?? snapshot.version,release: asString(body.release) ?? snapshot.release,startTimeISO:startTime ?? snapshot.startTimeISO ?? timestamp ?? snapshot.startTimeISO,- endTimeISO: asString(body.endTime) ?? snapshot.endTimeISO,+ endTimeISO: asIsoTimestamp(body.endTime) ?? snapshot.endTimeISO,completionStartTime:- asString(body.completionStartTime) ?? snapshot.completionStartTime,+ asIsoTimestamp(body.completionStartTime) ?? snapshot.completionStartTime,level: asString(body.level) ?? snapshot.level,statusMessage: asString(body.statusMessage) ?? snapshot.statusMessage,promptName: asString(body.promptName) ?? snapshot.promptName,@@ -311,6 +328,8 @@ export function buildInternalTraceEventInputs(params: {processedEvents: ProcessedTraceEvent[];traceId: string;projectId: string;+ userId?: string;+ sessionId?: string;experimentContext?: InternalTraceExperimentContext;}): {rootSpanId: string;@@ -375,8 +394,9 @@ export function buildInternalTraceEventInputs(params: {tags: rootSnapshot.tags ?? [],bookmarked: rootSnapshot.bookmarked,public: rootSnapshot.public,- userId: rootSnapshot.userId,- sessionId: rootSnapshot.sessionId,+ userId: snapshot.userId ?? rootSnapshot.userId ?? params.userId,+ sessionId:+ snapshot.sessionId ?? rootSnapshot.sessionId ?? params.sessionId,level: snapshot.level ?? "DEFAULT",statusMessage: snapshot.statusMessage,promptName: snapshot.promptName,diff --git a/packages/shared/src/server/llm/types.ts b/packages/shared/src/server/llm/types.tsindex 8d0ade6257..182bf3323e 100644--- a/packages/shared/src/server/llm/types.ts+++ b/packages/shared/src/server/llm/types.ts@@ -541,6 +541,7 @@ export type TraceSinkParams = {// NOTE: These strings must be whitelisted in the TS SDK to allow ingestion of traces by Langfuse. Please mirror edits to this string in https://github.com/langfuse/langfuse-js/blob/main/langfuse-core/src/index.ts.environment: string;userId?: string;+ sessionId?: string;metadata?: Record<string, unknown>;prompt?: {name: string;@@ -553,4 +554,12 @@ export type TraceSinkParams = {* events are still written to the legacy traces/observations tables.*/eventsWriter?: InternalEventsWriter;+ /**+ * When true, v4 write modes (`dual` / `events_only`) publish through+ * `publishAiFeatureTraceViaOtelIngestion` instead of `processEventBatch`.+ * Used by in-app agent product traces. Those traces use environment+ * `production` (not `langfuse-*`) so they stay eligible as observation-eval+ * targets. Legacy write mode keeps `processEventBatch`.+ */+ aiFeatureOtelIngestion?: boolean;};diff --git a/packages/shared/src/server/otel/aiFeatureOtelWriter.test.ts b/packages/shared/src/server/otel/aiFeatureOtelWriter.test.tsnew file mode 100644index 0000000000..f43b0f6786--- /dev/null+++ b/packages/shared/src/server/otel/aiFeatureOtelWriter.test.ts@@ -0,0 +1,199 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++import { buildInternalTraceEventInputs } from "../llm/internalTraceEvents";+import {+ AI_FEATURE_OTEL_SDK_NAME,+ publishAiFeatureTraceViaOtelIngestion,+} from "./aiFeatureOtelWriter";++const publishToOtelIngestionQueue = vi.fn().mockResolvedValue(undefined);+const processorConstructor = vi.fn();++vi.mock("./OtelIngestionProcessor", async (importOriginal) => {+ const actual =+ await importOriginal<typeof import("./OtelIngestionProcessor")>();+ return {+ ...actual,+ OtelIngestionProcessor: class {+ constructor(config: unknown) {+ processorConstructor(config);+ }+ publishToOtelIngestionQueue = publishToOtelIngestionQueue;+ },+ };+});++const START_ISO = "2026-08-27T12:00:00.000Z";+const END_ISO = "2026-08-27T12:00:01.000Z";+const RUN_ID = "arun_01234567-89ab-cdef-0123-456789abcdef";+const TRACE_ID = `${RUN_ID}-trace`;++const processedEvents = [+ {+ type: "trace-create",+ timestamp: START_ISO,+ body: {+ id: TRACE_ID,+ traceId: TRACE_ID,+ name: "agent-turn",+ environment: "production",+ tags: ["in-app-agent"],+ userId: "user-1",+ sessionId: "conversation-1",+ startTime: START_ISO,+ endTime: END_ISO,+ input: "hello",+ output: "hi",+ },+ },+ {+ type: "agent-create",+ timestamp: START_ISO,+ body: {+ id: RUN_ID,+ traceId: TRACE_ID,+ name: "agent-turn",+ environment: "production",+ startTime: START_ISO,+ endTime: END_ISO,+ input: { messages: [{ role: "user", content: "hello" }] },+ output: { text: "hi" },+ metadata: { langfuse_ai_feature: "in-app-agent" },+ },+ },+ {+ type: "tool-create",+ timestamp: START_ISO,+ body: {+ id: "tool-call-1",+ traceId: TRACE_ID,+ parentObservationId: RUN_ID,+ name: "langfuse_listPrompts",+ environment: "production",+ startTime: START_ISO,+ endTime: END_ISO,+ },+ },+ {+ type: "generation-create",+ timestamp: START_ISO,+ body: {+ id: `${RUN_ID}-llm-0`,+ traceId: TRACE_ID,+ parentObservationId: RUN_ID,+ name: "invoke-model",+ environment: "production",+ startTime: new Date(START_ISO),+ endTime: new Date(END_ISO),+ model: "claude-opus",+ usageDetails: { input: 10, output: 5, total: 15 },+ },+ },+];++const processPublishedSpans = async () => {+ expect(publishToOtelIngestionQueue).toHaveBeenCalledTimes(1);+ const resourceSpans = publishToOtelIngestionQueue.mock.calls[0][0];++ const { OtelIngestionProcessor } = await vi.importActual<+ typeof import("./OtelIngestionProcessor")+ >("./OtelIngestionProcessor");+ return new OtelIngestionProcessor({+ projectId: "project-1",+ publicKey: "",+ sdkName: AI_FEATURE_OTEL_SDK_NAME,+ sdkVersion: "unknown",+ ingestionVersion: "4",+ isLangfuseInternal: false,+ }).processToEvent(resourceSpans);+};++beforeEach(() => {+ vi.clearAllMocks();+});++describe("publishAiFeatureTraceViaOtelIngestion", () => {+ it("preserves agent and tool observation types from SDK event types", async () => {+ const { eventInputs } = buildInternalTraceEventInputs({+ processedEvents,+ traceId: TRACE_ID,+ projectId: "project-1",+ });++ await publishAiFeatureTraceViaOtelIngestion({+ projectId: "project-1",+ eventInputs,+ });++ expect(processorConstructor).toHaveBeenCalledWith(+ expect.objectContaining({+ ingestionVersion: "4",+ isLangfuseInternal: false,+ sdkName: AI_FEATURE_OTEL_SDK_NAME,+ }),+ );++ const events = await processPublishedSpans();+ expect(events).toHaveLength(4);++ const wrappingRoot = events.find((event) => event.spanId === TRACE_ID);+ const agent = events.find((event) => event.spanId === RUN_ID);+ const tool = events.find((event) => event.spanId === "tool-call-1");+ const generation = events.find(+ (event) => event.spanId === `${RUN_ID}-llm-0`,+ );++ expect(wrappingRoot).toMatchObject({+ traceId: TRACE_ID,+ parentSpanId: null,+ name: "agent-turn",+ type: "SPAN",+ environment: "production",+ userId: "user-1",+ sessionId: "conversation-1",+ });+ expect(wrappingRoot?.input).toBe("hello");+ expect(wrappingRoot?.output).toBe("hi");+ expect(wrappingRoot?.tags).toEqual(["in-app-agent"]);+ expect(agent).toMatchObject({+ traceId: TRACE_ID,+ parentSpanId: TRACE_ID,+ name: "agent-turn",+ type: "AGENT",+ environment: "production",+ });+ expect(generation).toMatchObject({+ traceId: TRACE_ID,+ parentSpanId: RUN_ID,+ name: "invoke-model",+ type: "GENERATION",+ environment: "production",+ modelName: "claude-opus",+ userId: "user-1",+ sessionId: "conversation-1",+ });+ expect(tool).toMatchObject({+ traceId: TRACE_ID,+ parentSpanId: RUN_ID,+ name: "langfuse_listPrompts",+ type: "TOOL",+ environment: "production",+ userId: "user-1",+ sessionId: "conversation-1",+ });+ expect(generation?.providedUsageDetails).toMatchObject({+ input: 10,+ output: 5,+ total: 15,+ });+ });++ it("skips an empty input", async () => {+ await publishAiFeatureTraceViaOtelIngestion({+ projectId: "project-1",+ eventInputs: [],+ });++ expect(publishToOtelIngestionQueue).not.toHaveBeenCalled();+ });+});diff --git a/packages/shared/src/server/otel/aiFeatureOtelWriter.ts b/packages/shared/src/server/otel/aiFeatureOtelWriter.tsnew file mode 100644index 0000000000..6e6ca81c80--- /dev/null+++ b/packages/shared/src/server/otel/aiFeatureOtelWriter.ts@@ -0,0 +1,121 @@+import type { InternalTraceEventInput } from "../llm/internalTraceEvents";+import { type ResourceSpan } from "./OtelIngestionProcessor";+import {+ internalTraceEventToOtelAttributes,+ publishOtelResourceSpans,+} from "./internalTraceOtelWriter";++export const AI_FEATURE_OTEL_SDK_NAME = "langfuse-internal-ai-features";+const AI_FEATURE_OTEL_SCOPE = "langfuse-ai-features-writer";+const AI_FEATURE_SERVICE_NAME = "langfuse-ai-features";++type OtelAttribute = {+ key: string;+ value: { stringValue: string };+};++function timestampToUnixNano(timestamp: string): string {+ const milliseconds = Date.parse(timestamp);+ if (Number.isNaN(milliseconds)) {+ throw new Error(`Invalid AI-feature trace timestamp: ${timestamp}`);+ }++ return `${BigInt(milliseconds) * 1_000_000n}`;+}++function attributesToOtelList(+ attributes: ReturnType<typeof internalTraceEventToOtelAttributes>,+): OtelAttribute[] {+ return Object.entries(attributes)+ .map(([key, value]) => {+ if (value === undefined || value === null) return undefined;+ return {+ key,+ value: {+ stringValue:+ typeof value === "string" ? value : JSON.stringify(value),+ },+ };+ })+ .filter((attribute): attribute is OtelAttribute => Boolean(attribute));+}++/**+ * Keep the SDK `trace-create` span (id === trace id). Legacy ingestion stored+ * that as a TRACE row; event propagation then materializes it as the wrapping+ * `t-{traceId}` SPAN with conversation input/output. Emitting it here gives+ * OTel-native assistant traces the same root span above the AGENT turn.+ */+function aiFeatureTraceInputsToResourceSpans(+ eventInputs: InternalTraceEventInput[],+): ResourceSpan[] {+ const spans = eventInputs.map((eventInput) => {+ const isRoot = !eventInput.parentSpanId;++ return {+ traceId: eventInput.traceId,+ spanId: eventInput.spanId,+ ...(eventInput.parentSpanId+ ? { parentSpanId: eventInput.parentSpanId }+ : {}),+ name: eventInput.name ?? "span",+ kind: 1,+ startTimeUnixNano: timestampToUnixNano(eventInput.startTimeISO),+ endTimeUnixNano: timestampToUnixNano(eventInput.endTimeISO),+ attributes: attributesToOtelList(+ internalTraceEventToOtelAttributes(eventInput, { isRoot }),+ ),+ status: {+ ...(eventInput.level === "ERROR" ? { code: 2 } : {}),+ ...(eventInput.statusMessage+ ? { message: eventInput.statusMessage }+ : {}),+ },+ };+ });++ return [+ {+ resource: {+ attributes: [+ {+ key: "service.name",+ value: { stringValue: AI_FEATURE_SERVICE_NAME },+ },+ ],+ },+ scopeSpans: [+ {+ scope: {+ name: AI_FEATURE_OTEL_SCOPE,+ version: "unknown",+ },+ spans,+ },+ ],+ },+ ] as ResourceSpan[];+}++/**+ * Publishes AI-features product traces through the same OTel ingestion queue+ * as `writeInternalTraceViaOtelIngestion`. Span construction stays here so+ * non-W3C ids (`arun_*`, `${runId}-llm-N`) and `production` survive; attribute+ * names and observation types come from `internalTraceEventToOtelAttributes`.+ */+export async function publishAiFeatureTraceViaOtelIngestion(params: {+ eventInputs: InternalTraceEventInput[];+ projectId: string;+}): Promise<void> {+ if (params.eventInputs.length === 0) return;++ await publishOtelResourceSpans({+ resourceSpans: aiFeatureTraceInputsToResourceSpans(params.eventInputs),+ projectId: params.projectId,+ sdkName: AI_FEATURE_OTEL_SDK_NAME,+ ingestionVersion: "4",+ // Product traces must look like customer data: the public schema keeps+ // `production`, and observation evals are allowed to match them.+ isLangfuseInternal: false,+ });+}diff --git a/packages/shared/src/server/otel/index.ts b/packages/shared/src/server/otel/index.tsindex 92dae9b006..5ec05d7406 100644--- a/packages/shared/src/server/otel/index.ts+++ b/packages/shared/src/server/otel/index.ts@@ -1,3 +1,4 @@export * from "./OtelIngestionProcessor";export * from "./OtelMediaProcessor";export * from "./internalTraceOtelWriter";+export * from "./aiFeatureOtelWriter";diff --git a/packages/shared/src/server/otel/internalTraceOtelWriter.ts b/packages/shared/src/server/otel/internalTraceOtelWriter.tsindex 14ae5d81a1..1ade651eb0 100644--- a/packages/shared/src/server/otel/internalTraceOtelWriter.ts+++ b/packages/shared/src/server/otel/internalTraceOtelWriter.ts@@ -15,15 +15,195 @@ import {type ReadableSpan,} from "@opentelemetry/sdk-trace-base";+import { ObservationTypeDomain } from "../../domain/observations";import type { InternalTraceEventInput } from "../llm/internalTraceEvents";import { logger } from "../logger";import { LangfuseOtelSpanAttributes } from "./attributes";-import { OtelIngestionProcessor } from "./OtelIngestionProcessor";+import {+ OtelIngestionProcessor,+ type ResourceSpan,+} from "./OtelIngestionProcessor";const INTERNAL_TRACE_WRITER_SDK_NAME = "langfuse-internal-otel-writer";const INTERNAL_TRACE_WRITER_SCOPE = "langfuse-internal-trace-writer";const W3C_TRACE_ID_PATTERN = /^[0-9a-f]{32}$/;+function asOtelAttributeString(value: unknown): string | undefined {+ if (value === undefined || value === null) return undefined;+ return typeof value === "string" ? value : JSON.stringify(value);+}++function observationTypeToOtelAttribute(+ type: string | undefined,+): string | undefined {+ const parsed = ObservationTypeDomain.safeParse(type?.toUpperCase());+ return parsed.success ? parsed.data.toLowerCase() : undefined;+}++/**+ * Maps an internal trace event onto the Langfuse OTel attribute names the+ * ingestion processor already understands (`langfuse.observation.type`,+ * generation fields, trace I/O on the root). Shared so type matching is not+ * re-implemented per caller.+ */+export function internalTraceEventToOtelAttributes(+ eventInput: Pick<+ InternalTraceEventInput,+ | "environment"+ | "level"+ | "statusMessage"+ | "input"+ | "output"+ | "metadata"+ | "type"+ | "modelName"+ | "modelParameters"+ | "providedUsageDetails"+ | "providedCostDetails"+ | "promptName"+ | "promptVersion"+ | "completionStartTime"+ | "userId"+ | "sessionId"+ | "tags"+ | "public"+ | "release"+ | "traceName"+ >,+ options: { isRoot: boolean },+): Attributes {+ const observationType = observationTypeToOtelAttribute(eventInput.type);+ const metadataJson = asOtelAttributeString(eventInput.metadata);+ const { isRoot } = options;++ return {+ ...(eventInput.environment !== undefined+ ? { [LangfuseOtelSpanAttributes.ENVIRONMENT]: eventInput.environment }+ : {}),+ ...(observationType+ ? { [LangfuseOtelSpanAttributes.OBSERVATION_TYPE]: observationType }+ : {}),+ ...(eventInput.level !== undefined+ ? { [LangfuseOtelSpanAttributes.OBSERVATION_LEVEL]: eventInput.level }+ : {}),+ ...(eventInput.statusMessage !== undefined+ ? {+ [LangfuseOtelSpanAttributes.OBSERVATION_STATUS_MESSAGE]:+ eventInput.statusMessage,+ }+ : {}),+ ...(eventInput.input !== undefined+ ? { [LangfuseOtelSpanAttributes.OBSERVATION_INPUT]: eventInput.input }+ : {}),+ ...(eventInput.output !== undefined+ ? { [LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT]: eventInput.output }+ : {}),+ ...(metadataJson !== undefined+ ? { [LangfuseOtelSpanAttributes.OBSERVATION_METADATA]: metadataJson }+ : {}),+ ...(eventInput.modelName !== undefined+ ? { [LangfuseOtelSpanAttributes.OBSERVATION_MODEL]: eventInput.modelName }+ : {}),+ ...(eventInput.modelParameters !== undefined+ ? {+ [LangfuseOtelSpanAttributes.OBSERVATION_MODEL_PARAMETERS]:+ JSON.stringify(eventInput.modelParameters),+ }+ : {}),+ ...(eventInput.providedUsageDetails !== undefined+ ? {+ [LangfuseOtelSpanAttributes.OBSERVATION_USAGE_DETAILS]:+ JSON.stringify(eventInput.providedUsageDetails),+ }+ : {}),+ ...(eventInput.providedCostDetails !== undefined+ ? {+ [LangfuseOtelSpanAttributes.OBSERVATION_COST_DETAILS]: JSON.stringify(+ eventInput.providedCostDetails,+ ),+ }+ : {}),+ ...(eventInput.promptName !== undefined+ ? {+ [LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_NAME]:+ eventInput.promptName,+ }+ : {}),+ ...(eventInput.promptVersion !== undefined+ ? {+ [LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_VERSION]:+ eventInput.promptVersion,+ }+ : {}),+ ...(eventInput.completionStartTime !== undefined+ ? {+ [LangfuseOtelSpanAttributes.OBSERVATION_COMPLETION_START_TIME]:+ eventInput.completionStartTime,+ }+ : {}),+ ...(eventInput.userId !== undefined+ ? { [LangfuseOtelSpanAttributes.TRACE_USER_ID]: eventInput.userId }+ : {}),+ ...(eventInput.sessionId !== undefined+ ? { [LangfuseOtelSpanAttributes.TRACE_SESSION_ID]: eventInput.sessionId }+ : {}),+ ...(eventInput.tags !== undefined+ ? {+ [LangfuseOtelSpanAttributes.TRACE_TAGS]: JSON.stringify(+ eventInput.tags,+ ),+ }+ : {}),+ ...(eventInput.public !== undefined+ ? {+ [LangfuseOtelSpanAttributes.TRACE_PUBLIC]: String(eventInput.public),+ }+ : {}),+ ...(eventInput.release !== undefined+ ? { [LangfuseOtelSpanAttributes.RELEASE]: eventInput.release }+ : {}),+ ...(isRoot ? { [LangfuseOtelSpanAttributes.IS_APP_ROOT]: "true" } : {}),+ ...(isRoot && eventInput.traceName !== undefined+ ? { [LangfuseOtelSpanAttributes.TRACE_NAME]: eventInput.traceName }+ : {}),+ ...(isRoot && eventInput.input !== undefined+ ? { [LangfuseOtelSpanAttributes.TRACE_INPUT]: eventInput.input }+ : {}),+ ...(isRoot && eventInput.output !== undefined+ ? { [LangfuseOtelSpanAttributes.TRACE_OUTPUT]: eventInput.output }+ : {}),+ ...(isRoot && metadataJson !== undefined+ ? { [LangfuseOtelSpanAttributes.TRACE_METADATA]: metadataJson }+ : {}),+ };+}++/**+ * Enqueues already-built OTLP resource spans on the regular OTel ingestion+ * path. Callers choose `isLangfuseInternal` and `sdkName`; judge/code-eval+ * stay internal, AI-feature product traces do not.+ */+export async function publishOtelResourceSpans(params: {+ resourceSpans: ResourceSpan[];+ projectId: string;+ sdkName: string;+ ingestionVersion?: string;+ isLangfuseInternal: boolean;+}): Promise<void> {+ if (params.resourceSpans.length === 0) return;++ const processor = new OtelIngestionProcessor({+ projectId: params.projectId,+ publicKey: "", // internal ingestion has no API key; mirrors internal event writes+ sdkName: params.sdkName,+ sdkVersion: "unknown",+ ingestionVersion: params.ingestionVersion,+ isLangfuseInternal: params.isLangfuseInternal,+ });++ await processor.publishToOtelIngestionQueue(params.resourceSpans);+}+/*** Publishes internally captured OTel spans through the regular OTel ingestion* pipeline (same S3 + queue path as the public /api/public/otel/v1/traces@@ -44,11 +224,10 @@ export async function publishInternalOtelSpans(params: {if (!resourceSpans || resourceSpans.length === 0) return;- const processor = new OtelIngestionProcessor({+ await publishOtelResourceSpans({+ resourceSpans,projectId: params.projectId,- publicKey: "", // internal ingestion has no API key; mirrors internal event writessdkName: params.sdkName,- sdkVersion: "unknown",// Opt into the v4-native direct events write like a modern SDK batch:// only that path runs processToEvent -> createEventRecord, which is// the sole extractor of langfuse.experiment.* into experiment_*@@ -63,8 +242,6 @@ export async function publishInternalOtelSpans(params: {// trace-upsert eval-loop guard.isLangfuseInternal: true,});-- await processor.publishToOtelIngestionQueue(resourceSpans);}/**@@ -153,38 +330,9 @@ export async function writeInternalTraceViaOtelIngestion(trace: {for (const eventInput of eventInputs) {currentTraceId = eventInput.traceId.toLowerCase();const isRoot = !eventInput.parentSpanId;- const metadataJson = JSON.stringify(eventInput.metadata);-- const attributes: Attributes = {- ...(eventInput.environment !== undefined- ? { [LangfuseOtelSpanAttributes.ENVIRONMENT]: eventInput.environment }- : {}),- ...(eventInput.level !== undefined- ? { [LangfuseOtelSpanAttributes.OBSERVATION_LEVEL]: eventInput.level }- : {}),- ...(eventInput.statusMessage !== undefined- ? {- [LangfuseOtelSpanAttributes.OBSERVATION_STATUS_MESSAGE]:- eventInput.statusMessage,- }- : {}),- ...(eventInput.input !== undefined- ? { [LangfuseOtelSpanAttributes.OBSERVATION_INPUT]: eventInput.input }- : {}),- ...(eventInput.output !== undefined- ? {- [LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT]:- eventInput.output,- }- : {}),- [LangfuseOtelSpanAttributes.OBSERVATION_METADATA]: metadataJson,- ...(isRoot && eventInput.traceName !== undefined- ? { [LangfuseOtelSpanAttributes.TRACE_NAME]: eventInput.traceName }- : {}),- ...(isRoot- ? { [LangfuseOtelSpanAttributes.TRACE_METADATA]: metadataJson }- : {}),- };+ const attributes = internalTraceEventToOtelAttributes(eventInput, {+ isRoot,+ });let context = ROOT_CONTEXT;if (eventInput.parentSpanId) {diff --git a/web/src/features/in-app-agent/server/router.ts b/web/src/features/in-app-agent/server/router.tsindex f1ccc84c79..77e477c6fe 100644--- a/web/src/features/in-app-agent/server/router.ts+++ b/web/src/features/in-app-agent/server/router.ts@@ -16,6 +16,7 @@ import {AgUiContextSchema,getInAppAgentInstrumentationObservationId,getInAppAgentInstrumentationTraceId,+ IN_APP_AGENT_PRODUCT_ENVIRONMENT,} from "@langfuse/shared/in-app-agent";import { InAppAgentMessageFeedbackValueSchema } from "../schema";import {@@ -95,7 +96,7 @@ const SubmitFeedbackInput = ConversationIdInput.extend({});const IN_APP_AGENT_FEEDBACK_SCORE_NAME = "in_app_agent_feedback";-const IN_APP_AGENT_FEEDBACK_ENVIRONMENT = "langfuse-in-app-agent";+const IN_APP_AGENT_FEEDBACK_ENVIRONMENT = IN_APP_AGENT_PRODUCT_ENVIRONMENT;export const inAppAgentRouter = createTRPCRouter({// Each row carries its newest run summary for the activity poll and badges.diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.tsindex e9d2b11eb9..71d6cff860 100644--- a/worker/src/features/in-app-agent/executeInAppAgentRun.ts+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.ts@@ -16,6 +16,7 @@ import {InAppAgentRunRequestSchema,InAppAgentRunStatus,getInAppAgentInstrumentationTraceId,+ IN_APP_AGENT_PRODUCT_ENVIRONMENT,parseInAppAgentInterruptEvent,type AgUiEvent,type InAppAgentRunRequest,@@ -158,7 +159,7 @@ export async function executeInAppAgentRun(params: {// The uncovered durability window: an approved mutation may have started// but its result never persisted. Never generically retried. Hoisted to// function scope because both the loop callbacks and the outer catch can- // record the terminal state (failStream races the drain rejection).+ // record the terminal state.const failureCode = () =>isApprovedContinuation && !approvedToolResultPersisted? {@@ -168,9 +169,9 @@ export async function executeInAppAgentRun(params: {}: undefined;- // Single-flight (same shape as web's withInAppAgentMcpApiKeyCleanup): the- // loop's onFinish and the outer catch can invoke this concurrently because- // failStream errors the stream without awaiting its onError/onFinish chain.+ // Single-flight (same shape as web's withInAppAgentMcpApiKeyCleanup):+ // onFinish cleans up before the stream errors, then the outer catch+ // runs after drainStream rejects and may call this again.const cleanupMcpApiKey = (): Promise<void> => {if (!mcpApiKey) return Promise.resolve();const keyId = mcpApiKey.id;@@ -629,12 +630,12 @@ export async function executeInAppAgentRun(params: {await drainStream(stream);} catch (error) {// Two classes land here: pre-loop failures (revalidation, input build,- // key minting) and loop failures surfaced through the errored stream —- // failStream fires the loop's onError without awaiting it and errors the- // stream synchronously, so this catch can RACE that onError and win the- // terminal CAS. Both writers therefore classify identically: the- // durability check first, then agent_error for loop-phase failures and- // init_failed only before the loop existed.+ // key minting) and loop failures surfaced through the errored stream+ // after tracing flush, onError, and onFinish. finishClaimedRun is+ // idempotent, so a second write is a no-op if onError already recorded+ // FAILED. Both writers classify identically: the durability check first,+ // then agent_error for loop-phase failures and init_failed only before+ // the loop existed.await finishClaimedRun({prisma,projectId,@@ -768,7 +769,7 @@ function buildTracingConfig(params: {}const traceSinkParams = getLangfuseAITraceSinkParams({- environment: "langfuse-in-app-agent",+ environment: IN_APP_AGENT_PRODUCT_ENVIRONMENT,feature: "in-app-agent",projectId: params.projectId,traceId: getInAppAgentInstrumentationTraceId(params.runId),diff --git a/worker/src/features/in-app-agent/runtime/agent.test.ts b/worker/src/features/in-app-agent/runtime/agent.test.tsindex b3e0bb7503..f99ddd1a6f 100644--- a/worker/src/features/in-app-agent/runtime/agent.test.ts+++ b/worker/src/features/in-app-agent/runtime/agent.test.ts@@ -1309,6 +1309,71 @@ describe("createAgUiStream", () => {expect(instrumentationMocks.instrumentation.flush).toHaveBeenCalled();});+ it("waits for tracing flush before finishing an aborted agent stream", async () => {+ const { createAgUiStream } = await import("./agent");+ let resolveFlush: (() => void) | undefined;+ instrumentationMocks.instrumentation.flush.mockReturnValue(+ new Promise<void>((resolve) => {+ resolveFlush = resolve;+ }),+ );+ const input = {+ threadId: "conversation-1",+ runId: "run-flush-abort",+ messages: [+ {+ id: "user-message-1",+ role: "user" as const,+ content: "hello",+ },+ ],+ tools: [],+ context: [],+ state: {+ type: "existingConversation" as const,+ projectId: "project-1",+ conversationId: "conversation-1",+ },+ forwardedProps: {},+ };+ adapterEvents.items = [];+ const onAbort = vi.fn();+ const abortController = new AbortController();+ abortController.abort("worker_shutdown");++ const stream = await createAgUiStream({+ input,+ signal: abortController.signal,+ options: {+ onAbort,+ model: testBedrockModel("test-model"),+ langfuseMcp: {+ url: "https://example.com/api/public/mcp",+ publicKey: "pk",+ secretKey: "sk",+ toolPolicy: defaultInAppAgentToolPolicy,+ },+ redirectAction: { projectId: "project-1", isV4Enabled: false },+ langfuseClient: {+ getPrompt: promptMocks.getPrompt,+ } as unknown as Langfuse,+ useLocalPrompt: false,+ langfuseTracing: createTestTracingConfig(),+ },+ });++ const streamDone = readStream(stream);+ await vi.waitFor(() => {+ expect(instrumentationMocks.instrumentation.flush).toHaveBeenCalled();+ });+ expect(onAbort).not.toHaveBeenCalled();++ resolveFlush?.();+ await streamDone;++ expect(onAbort).toHaveBeenCalledOnce();+ });+it("does not enable Bedrock reasoning for non-Claude models", async () => {const { createAgUiStream } = await import("./agent");const input = {diff --git a/worker/src/features/in-app-agent/runtime/agent.ts b/worker/src/features/in-app-agent/runtime/agent.tsindex 755384169d..01b244d50a 100644--- a/worker/src/features/in-app-agent/runtime/agent.ts+++ b/worker/src/features/in-app-agent/runtime/agent.ts@@ -340,14 +340,21 @@ export async function createAgUiStream(params: {operation: string,callback: (activeInstrumentation: NonNullable<typeof instrumentation>,- ) => void,- ) => {+ ) => void | Promise<void>,+ ): Promise<void> => {if (!instrumentation) {- return;+ return Promise.resolve();}try {- callback(instrumentation);+ return Promise.resolve(callback(instrumentation)).catch((error) => {+ logger.warn("Failed to record in-app agent Langfuse tracing", {+ error,+ operation,+ runId: params.input.runId,+ threadId: params.input.threadId,+ });+ });} catch (error) {logger.warn("Failed to record in-app agent Langfuse tracing", {error,@@ -355,6 +362,7 @@ export async function createAgUiStream(params: {runId: params.input.runId,threadId: params.input.threadId,});+ return Promise.resolve();}};recordInstrumentation("recordAvailableSkills", (instrumentation) =>@@ -473,7 +481,7 @@ export async function createAgUiStream(params: {recordInstrumentation("endWithError", (instrumentation) =>instrumentation.endWithError(error),);- recordInstrumentation("flush", (instrumentation) =>+ const flushPromise = recordInstrumentation("flush", (instrumentation) =>instrumentation.flush(),);ending = true;@@ -491,18 +499,25 @@ export async function createAgUiStream(params: {});runTerminalCallback(- () => params.options.onError?.(error),- "Error while marking agent stream as failed",+ () => flushPromise,+ "Error while flushing agent stream tracing after failure",)+ .then(() =>+ runTerminalCallback(+ () => params.options.onError?.(error),+ "Error while marking agent stream as failed",+ ),+ ).then(() =>runTerminalCallback(runOnFinish,"Error while running agent stream finish callback after failure",),)+ .then(() => {+ controller.error(error);+ }).finally(finish);-- controller.error(error);};const enqueueEvent = (@@ -582,7 +597,7 @@ export async function createAgUiStream(params: {recordInstrumentation("end", (instrumentation) =>instrumentation.end({ aborted: true }),);- recordInstrumentation("flush", (instrumentation) =>+ const flushPromise = recordInstrumentation("flush", (instrumentation) =>instrumentation.flush(),);ending = true;@@ -591,6 +606,12 @@ export async function createAgUiStream(params: {interruptAdapter?.();subscription?.unsubscribe();eventQueue+ .then(() =>+ runTerminalCallback(+ () => flushPromise,+ "Error while flushing agent stream tracing after abort",+ ),+ ).then(() =>runTerminalCallback(() => params.options.onAbort?.(),@@ -826,8 +847,8 @@ export async function createAgUiStream(params: {}if (streamedRunError !== null) {- closeController(() => {- recordInstrumentation("flush", (instrumentation) =>+ closeController(async () => {+ await recordInstrumentation("flush", (instrumentation) =>instrumentation.flush(),);return handleStreamedRunError();@@ -866,7 +887,7 @@ export async function createAgUiStream(params: {closeController(streamedRunError === null- ? () => {+ ? async () => {const truncatedByStepLimit =isTruncatedByStepLimit(stepLimitState);recordInstrumentation("end", (instrumentation) =>@@ -881,7 +902,7 @@ export async function createAgUiStream(params: {: {},),);- recordInstrumentation("flush", (instrumentation) =>+ await recordInstrumentation("flush", (instrumentation) =>instrumentation.flush(),);return params.options.onComplete?.({@@ -889,8 +910,8 @@ export async function createAgUiStream(params: {truncatedByStepLimit,});}- : () => {- recordInstrumentation("flush", (instrumentation) =>+ : async () => {+ await recordInstrumentation("flush", (instrumentation) =>instrumentation.flush(),);return handleStreamedRunError();@@ -936,7 +957,7 @@ export async function createAgUiStream(params: {recordInstrumentation("end", (instrumentation) =>instrumentation.end({ aborted: true }),);- recordInstrumentation("flush", (instrumentation) =>+ const flushPromise = recordInstrumentation("flush", (instrumentation) =>instrumentation.flush(),);shouldEnqueue = false;@@ -944,6 +965,12 @@ export async function createAgUiStream(params: {interruptAdapter?.();subscription?.unsubscribe();eventQueue+ .then(() =>+ runTerminalCallback(+ () => flushPromise,+ "Error while flushing agent stream tracing after cancel",+ ),+ ).then(() =>runTerminalCallback(() => params.options.onAbort?.(),diff --git a/worker/src/features/in-app-agent/runtime/instrumentation.test.ts b/worker/src/features/in-app-agent/runtime/instrumentation.test.tsindex b2731e6756..fd6cdf3c9f 100644--- a/worker/src/features/in-app-agent/runtime/instrumentation.test.ts+++ b/worker/src/features/in-app-agent/runtime/instrumentation.test.ts@@ -64,6 +64,7 @@ const input: AgUiRunAgentInput = {const expectedAgentRunInput = {messages: [{ role: "user", content: "hello" }],};+const expectedWrappingRootInput = "hello";describe("InAppAgentInstrumentation", () => {beforeEach(() => {@@ -139,6 +140,7 @@ describe("InAppAgentInstrumentation", () => {traceId,traceName: "agent-turn",userId: "user-1",+ sessionId: "conversation-1",}),);expect(mocks.handler.langfuse.trace).toHaveBeenCalledWith(@@ -146,7 +148,7 @@ describe("InAppAgentInstrumentation", () => {);expect(mocks.handler.langfuse.trace).toHaveBeenCalledWith(expect.objectContaining({- input: expectedAgentRunInput,+ input: expectedWrappingRootInput,}),);expect(mocks.trace.generation).not.toHaveBeenCalled();@@ -157,6 +159,8 @@ describe("InAppAgentInstrumentation", () => {traceId,parentObservationId: agentRunObservationId,name: "listObservations",+ userId: "user-1",+ sessionId: "conversation-1",input: { limit: 10 },output: toolOutput,startTime: expect.any(Date),@@ -210,24 +214,8 @@ describe("InAppAgentInstrumentation", () => {);expect(mocks.trace.update).toHaveBeenCalledWith(expect.objectContaining({- input: expectedAgentRunInput,- output: {- messages: [- { role: "assistant", content: "hi there" },- {- role: "assistant",- content: "",- tool_calls: [toolCall],- },- {- role: "tool",- tool_call_id: "tool-1",- content: toolOutput,- },- ],- text: "hi there",- tool_calls: [toolCall],- },+ input: expectedWrappingRootInput,+ output: "hi there",}),);});@@ -785,11 +773,8 @@ describe("InAppAgentInstrumentation", () => {);expect(mocks.trace.update).toHaveBeenCalledWith(expect.objectContaining({- input: expectedAgentRunInput,- output: {- messages: [{ role: "assistant", content: "second turn output" }],- text: "second turn output",- },+ input: expectedWrappingRootInput,+ output: "second turn output",}),);});@@ -815,6 +800,12 @@ describe("InAppAgentInstrumentation", () => {expect(mocks.trace.generation).not.toHaveBeenCalled();expect(mocks.handler.langfuse.trace).toHaveBeenCalledWith(+ expect.objectContaining({+ input: expectedWrappingRootInput,+ }),+ );+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "agent-create",expect.objectContaining({input: {messages: [{ role: "user", content: "hello" }],@@ -875,6 +866,12 @@ describe("InAppAgentInstrumentation", () => {expect(mocks.handler.langfuse.trace).toHaveBeenCalledWith(expect.objectContaining({name: "agent-turn",+ input: expectedWrappingRootInput,+ }),+ );+ expect(mocks.handler.langfuse.enqueue).toHaveBeenCalledWith(+ "agent-create",+ expect.objectContaining({input: {messages: [{ role: "user", content: "hello" }],},@@ -985,7 +982,7 @@ describe("InAppAgentInstrumentation", () => {name: "agent-turn",sessionId: "conversation-1",timestamp: new Date(traceStartedAt),- input: expectedAgentRunInput,+ input: expectedWrappingRootInput,tags: ["in-app-agent"],metadata: expect.objectContaining({approval_continuation_count: 2,@@ -1046,7 +1043,7 @@ describe("InAppAgentInstrumentation", () => {);expect(mocks.trace.update).toHaveBeenCalledWith(expect.objectContaining({- input: expectedAgentRunInput,+ input: expectedWrappingRootInput,metadata: expect.objectContaining({approval_continuation_count: 2,}),@@ -1106,6 +1103,12 @@ describe("InAppAgentInstrumentation", () => {}),}),);+ expect(mocks.trace.update).toHaveBeenCalledWith(+ expect.objectContaining({+ input: expectedWrappingRootInput,+ output: "I need your approval.\n\nThe tool completed.",+ }),+ );});it("compacts text message chunks before recording output", () => {@@ -1585,7 +1588,7 @@ describe("InAppAgentInstrumentation", () => {expect(toolCreates[0]?.[1]).toEqual(expect.objectContaining({id: "tool-a",- parentObservationId: agentRunObservationId,+ parentObservationId: getInAppAgentLlmCallObservationId(runId, 1),startTime: toolAStart,endTime: toolAEnd,}),@@ -1593,7 +1596,7 @@ describe("InAppAgentInstrumentation", () => {expect(toolCreates[1]?.[1]).toEqual(expect.objectContaining({id: "tool-b",- parentObservationId: agentRunObservationId,+ parentObservationId: getInAppAgentLlmCallObservationId(runId, 1),startTime: toolBStart,endTime: toolBEnd,}),diff --git a/worker/src/features/in-app-agent/runtime/instrumentation.ts b/worker/src/features/in-app-agent/runtime/instrumentation.tsindex 4f708e93e6..da4ed7d4a2 100644--- a/worker/src/features/in-app-agent/runtime/instrumentation.ts+++ b/worker/src/features/in-app-agent/runtime/instrumentation.ts@@ -104,6 +104,8 @@ type ObservationBody = {metadata?: Record<string, unknown>;promptName?: string;promptVersion?: number;+ userId?: string;+ sessionId?: string;};type ToolCallApprovalStatus = "approved" | "rejected";type AgentRunToolSpan = {@@ -177,14 +179,18 @@ export function createInAppAgentInstrumentation({export class InAppAgentInstrumentation {private readonly processTracedEvents: () => Promise<void>;+ private flushPromise?: Promise<void>;private readonly langfuse: InAppAgentLangfuse;private readonly trace: InAppAgentTrace;private readonly runId: string;private readonly traceId: string;private readonly rootObservationId: string;+ private readonly userId: string;+ private readonly sessionId: string;private readonly traceStartTime: Date;private readonly agentRunStartTime: Date;private agentRunInput: unknown;+ private readonly wrappingRootInput: string | undefined;private readonly approvalContinuation?: ApprovalContinuation;private readonly prompt?: InAppAgentPromptMetadata;private readonly toolSpans = new Map<string, AgentRunToolSpan>();@@ -197,6 +203,9 @@ export class InAppAgentInstrumentation {InAppAgentToolApprovalSource>();private readonly toolExecutionTimes = new Map<string, ToolExecutionTimes>();+ private readonly toolCallToModelObservationId = new Map<string, string>();+ private openModelObservationId?: string;+ private lastModelObservationId?: string;private readonly metadata: Record<string, unknown>;private readonly agentRunOutputMessages: AgentRunChatMessage[] = [];private readonly agentRunToolCalls: AgentRunToolCall[] = [];@@ -243,9 +252,12 @@ export class InAppAgentInstrumentation {: {}),};this.agentRunInput = getAgentRunInput(params.input);+ this.wrappingRootInput = getWrappingRootUserText(params.input);this.prompt = params.prompt;this.model = params.model;this.runId = params.runId;+ this.userId = params.userId;+ this.sessionId = params.input.threadId;const rootRunId = this.approvalContinuation?.rootRunId ?? params.runId;this.traceId = getInAppAgentInstrumentationTraceId(rootRunId);this.rootObservationId =@@ -260,8 +272,10 @@ export class InAppAgentInstrumentation {traceName: IN_APP_AGENT_TURN_NAME,environment: params.environment,userId: params.userId,+ sessionId: params.input.threadId,metadata: this.metadata,prompt: params.prompt,+ aiFeatureOtelIngestion: true,};const { handler, processTracedEvents } =getInternalTracingHandler(traceSinkParams);@@ -274,7 +288,7 @@ export class InAppAgentInstrumentation {userId: params.userId,sessionId: params.input.threadId,timestamp: this.traceStartTime,- input: this.getTraceInput(),+ input: this.wrappingRootInput,metadata: this.metadata,tags: ["in-app-agent"],});@@ -352,8 +366,14 @@ export class InAppAgentInstrumentation {}const now = new Date();+ const callNumber =+ this.openModelCall?.callNumber ?? ++this.nextModelCallNumber;+ this.openModelObservationId = getInAppAgentLlmCallObservationId(+ this.runId,+ callNumber,+ );this.openModelCall = {- callNumber: this.openModelCall?.callNumber ?? ++this.nextModelCallNumber,+ callNumber,startTime: this.openModelCall?.startTime ?? now,input: getModelCallInput(options),textDeltas: [],@@ -450,8 +470,8 @@ export class InAppAgentInstrumentation {},});this.trace.update({- input: this.getTraceInput(),- output: this.getAgentRunOutput(),+ input: this.wrappingRootInput,+ output: this.getWrappingRootOutput(),metadata: { ...this.metadata, error: message },});this.ended = true;@@ -476,17 +496,19 @@ export class InAppAgentInstrumentation {};this.emitRootAgentObservation({ metadata });this.trace.update({- input: this.getTraceInput(),- output: this.getAgentRunOutput(),+ input: this.wrappingRootInput,+ output: this.getWrappingRootOutput(),metadata,});this.ended = true;}- flush() {- this.processTracedEvents().catch((error) => {+ flush(): Promise<void> {+ this.flushPromise ??= this.processTracedEvents().catch((error) => {logger.warn("Failed to flush in-app agent Langfuse tracing", error);});++ return this.flushPromise;}private recordEvent(event: AgUiEvent) {@@ -693,7 +715,7 @@ export class InAppAgentInstrumentation {const body: ObservationBody = {id: toolCallId,traceId: this.traceId,- parentObservationId: this.rootObservationId,+ parentObservationId: this.getToolParentObservationId(toolCallId),name: tool.name,startTime,endTime,@@ -761,6 +783,11 @@ export class InAppAgentInstrumentation {const finishReason = getModelFinishReason(finish);const model = call.modelId ?? this.model;const id = getInAppAgentLlmCallObservationId(this.runId, call.callNumber);+ this.lastModelObservationId = id;+ this.openModelObservationId = undefined;+ for (const toolCall of call.toolCalls) {+ this.toolCallToModelObservationId.set(toolCall.toolCallId, id);+ }const text = call.textDeltas.join("");const output =text || call.toolCalls.length > 0@@ -868,12 +895,25 @@ export class InAppAgentInstrumentation {}}+ private getToolParentObservationId(toolCallId: string): string {+ return (+ this.toolCallToModelObservationId.get(toolCallId) ??+ this.openModelObservationId ??+ this.lastModelObservationId ??+ this.rootObservationId+ );+ }+private enqueueObservation(type: string, body: ObservationBody) {(this.langfuse as unknown as {enqueue: (type: string, body: ObservationBody) => void;}- ).enqueue(type, body);+ ).enqueue(type, {+ ...body,+ userId: this.userId,+ sessionId: this.sessionId,+ });}// Reasoning that no assistant message followed (e.g. aborted or errored@@ -970,17 +1010,34 @@ export class InAppAgentInstrumentation {}}+ private getWrappingRootOutput(): string | undefined {+ const texts = this.getAgentRunOutputMessages().flatMap((message) => {+ if (message.role !== "assistant") {+ return [];+ }++ const text = getRenderedMessageText(message.content);+ return text.trim() ? [text] : [];+ });++ if (texts.length === 0) {+ return undefined;+ }++ return texts.join("\n\n");+ }+private getAgentRunOutput() {- const priorMessages =- this.approvalContinuation && isRecord(this.agentRunInput)- ? getPriorAgentRunOutputMessages(this.agentRunInput.messages)- : [];- const messages = [...priorMessages, ...this.agentRunOutputMessages];+ const messages = this.getAgentRunOutputMessages();if (messages.length === 0) {return undefined;}+ const priorMessages = messages.slice(+ 0,+ messages.length - this.agentRunOutputMessages.length,+ );const priorToolCalls = priorMessages.flatMap((message) => message.tool_calls ?? [],);@@ -993,6 +1050,15 @@ export class InAppAgentInstrumentation {};}+ private getAgentRunOutputMessages(): AgentRunChatMessage[] {+ const priorMessages =+ this.approvalContinuation && isRecord(this.agentRunInput)+ ? getPriorAgentRunOutputMessages(this.agentRunInput.messages)+ : [];++ return [...priorMessages, ...this.agentRunOutputMessages];+ }+private emitApprovalWaitObservation() {const continuation = this.approvalContinuation;if (!continuation) {@@ -1088,6 +1154,37 @@ function getPriorAgentRunOutputMessages();}+function getWrappingRootUserText(input: AgUiRunAgentInput): string | undefined {+ const lastUserMessage = input.messages.findLast(+ (message) => message.role === "user",+ );++ if (!lastUserMessage) {+ return undefined;+ }++ const text = getRenderedMessageText(lastUserMessage.content);+ return text.length > 0 ? text : undefined;+}++function getRenderedMessageText(content: unknown): string {+ if (typeof content === "string") {+ return content;+ }++ if (Array.isArray(content)) {+ return content+ .flatMap((part) =>+ isRecord(part) && part.type === "text" && typeof part.text === "string"+ ? [part.text]+ : [],+ )+ .join("");+ }++ return "";+}+function getAgentRunInput(input: AgUiRunAgentInput): unknown {const messages = getAgentRunMessages(getCurrentTurnMessages(input.messages));const context = getAgentRunContext(input);diff --git a/worker/src/features/internal-tracing/__tests__/getInternalTracingHandler.test.ts b/worker/src/features/internal-tracing/__tests__/getInternalTracingHandler.test.tsindex bc03e62fec..f5c9629a34 100644--- a/worker/src/features/internal-tracing/__tests__/getInternalTracingHandler.test.ts+++ b/worker/src/features/internal-tracing/__tests__/getInternalTracingHandler.test.ts@@ -1,12 +1,16 @@import { beforeEach, describe, expect, it, vi } from "vitest";-const { mockExportLocalEvents, mockProcessEventBatch, mockWrite } = vi.hoisted(- () => ({- mockExportLocalEvents: vi.fn(),- mockProcessEventBatch: vi.fn(),- mockWrite: vi.fn(),- }),-);+const {+ mockExportLocalEvents,+ mockProcessEventBatch,+ mockWrite,+ mockPublishAiFeatureTrace,+} = vi.hoisted(() => ({+ mockExportLocalEvents: vi.fn(),+ mockProcessEventBatch: vi.fn(),+ mockWrite: vi.fn(),+ mockPublishAiFeatureTrace: vi.fn(),+}));vi.mock("langfuse", () => {return {@@ -23,8 +27,16 @@ vi.mock(}),);+vi.mock(+ "../../../../../packages/shared/src/server/otel/aiFeatureOtelWriter",+ () => ({+ publishAiFeatureTraceViaOtelIngestion: mockPublishAiFeatureTrace,+ }),+);+import { getInternalTracingHandler } from "../../../../../packages/shared/src/server/llm/getInternalTracingHandler";import { LangfuseInternalTraceEnvironment } from "@langfuse/shared";+import { env } from "../../../../../packages/shared/src/env";const traceId = "trace-123";const processedEventsFixture = [@@ -80,6 +92,7 @@ describe("getInternalTracingHandler", () => {errors: [],});mockWrite.mockResolvedValue(undefined);+ mockPublishAiFeatureTrace.mockResolvedValue(undefined);});it("disables staging propagation when direct event write is enabled", async () => {@@ -132,4 +145,51 @@ describe("getInternalTracingHandler", () => {);expect(mockWrite).not.toHaveBeenCalled();});++ it("publishes AI-feature traces via OTel on v4 write modes", async () => {+ const previousWriteMode = env.LANGFUSE_MIGRATION_V4_WRITE_MODE;+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "events_only";++ try {+ const { processTracedEvents } = getInternalTracingHandler({+ targetProjectId: "project-123",+ traceId,+ traceName: "agent-turn",+ environment: "production",+ aiFeatureOtelIngestion: true,+ });++ await processTracedEvents();++ expect(mockProcessEventBatch).not.toHaveBeenCalled();+ expect(mockPublishAiFeatureTrace).toHaveBeenCalledWith({+ projectId: "project-123",+ eventInputs: expect.any(Array),+ });+ } finally {+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE = previousWriteMode;+ }+ });++ it("keeps processEventBatch for AI-feature traces in legacy write mode", async () => {+ const previousWriteMode = env.LANGFUSE_MIGRATION_V4_WRITE_MODE;+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "legacy";++ try {+ const { processTracedEvents } = getInternalTracingHandler({+ targetProjectId: "project-123",+ traceId,+ traceName: "agent-turn",+ environment: "production",+ aiFeatureOtelIngestion: true,+ });++ await processTracedEvents();++ expect(mockProcessEventBatch).toHaveBeenCalled();+ expect(mockPublishAiFeatureTrace).not.toHaveBeenCalled();+ } finally {+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE = previousWriteMode;+ }+ });});