🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(ai-features): ingest Ask AI and titles as production OTel (#16717)
Lysander Kiesele65f2e0693b0
diff --git a/packages/shared/src/in-app-agent/server/persistence.ts b/packages/shared/src/in-app-agent/server/persistence.tsindex 6155931e03..6363a40d9b 100644--- a/packages/shared/src/in-app-agent/server/persistence.ts+++ b/packages/shared/src/in-app-agent/server/persistence.ts@@ -6,12 +6,7 @@ import {InAppAgentRunErrorCode,InAppAgentRunStatus,} from "../../features/inAppAgent/types";-import {- ChatMessageRole,- ChatMessageType,- LangfuseInternalTraceEnvironment,- logger,-} from "../../server";+import { ChatMessageRole, ChatMessageType, logger } from "../../server";import { isSettledInAppAgentRunStatus } from "../constants";import { recordRunTerminalOutcome } from "./runMetrics";import { Prisma } from "../../db";@@ -478,6 +473,10 @@ export async function maybeInferAndPersistConversationTitle(params: {return;}+ if (!isUnsetConversationTitle(conversation.title)) {+ return;+ }+const transcript = buildConversationTitleTranscript(await getConversationMessages({prisma: params.prisma,@@ -548,7 +547,6 @@ ${JSON.stringify(transcript, null, 2)}maxTokens: 1000,traceSinkParams: params.aiTelemetryEnabled? getLangfuseAITraceSinkParams({- environment: LangfuseInternalTraceEnvironment.InAppAgent,feature: "in-app-agent-conversation-title",projectId: params.projectId,traceName: "in-app-agent-conversation-title",@@ -1504,6 +1502,14 @@ function getDefaultConversationTitle(date: Date) {return `Chat on ${weekday} at ${hours}:${minutes}`;}+function isUnsetConversationTitle(title: string | null) {+ if (!title) {+ return true;+ }++ return /^Chat on [A-Za-z]+ at \d{2}:\d{2}$/.test(title);+}+export function buildConversationTitleTranscript(messages: readonly AgUiMessage[],) {diff --git a/packages/shared/src/server/llm/ai-sdk/telemetry.ts b/packages/shared/src/server/llm/ai-sdk/telemetry.tsindex dbac53db05..995feae50e 100644--- a/packages/shared/src/server/llm/ai-sdk/telemetry.ts+++ b/packages/shared/src/server/llm/ai-sdk/telemetry.ts@@ -24,6 +24,7 @@ import { stringifyValue } from "../../../utils/stringChecks";import { traceException } from "../../instrumentation";import { logger } from "../../logger";import { LangfuseOtelSpanAttributes } from "../../otel/attributes";+import { AI_FEATURE_OTEL_SDK_NAME } from "../../otel/internalAiFeatureOtelWriter";import { publishInternalOtelSpans } from "../../otel/internalTraceOtelWriter";import type { InternalTraceExperimentContext } from "../internalTraceEvents";import type { TraceSinkParams } from "../types";@@ -79,7 +80,8 @@ export type AiSdkTelemetryCapture = {* experiment observation evals.** Returns `undefined` (no tracing) when the environment is not- * langfuse-prefixed (the eval-loop safeguard) or when the trace ID is not a+ * langfuse-prefixed (the eval-loop safeguard) unless this is an AI-feature+ * product trace (`aiFeatureOtelIngestion`), or when the trace ID is not a* valid W3C trace ID.*/export function createAiSdkTelemetryCapture(params: {@@ -88,11 +90,19 @@ export function createAiSdkTelemetryCapture(params: {rootInput?: unknown;}): AiSdkTelemetryCapture | undefined {const { traceSinkParams, rootInput } = params;+ const isAiFeatureProductTrace = Boolean(+ traceSinkParams.aiFeatureOtelIngestion,+ );// Safeguard: All internal traces must use LangfuseInternalTraceEnvironment enum values// This prevents infinite eval loops (user trace → eval → eval trace → another eval)// See corresponding check in worker/src/features/evaluation/evalService.ts createEvalJobs()- if (!traceSinkParams.environment?.startsWith("langfuse")) {+ // AI-feature product traces are the exception: they use environment+ // `production` so observation evals can match them.+ if (+ !isAiFeatureProductTrace &&+ !traceSinkParams.environment?.startsWith("langfuse")+ ) {logger.warn("Skipping trace creation: internal traces must use LangfuseInternalTraceEnvironment enum",{@@ -270,7 +280,10 @@ export function createAiSdkTelemetryCapture(params: {await publishInternalOtelSpans({spans: matchingSpans,projectId: traceSinkParams.targetProjectId,- sdkName: INTERNAL_SDK_NAME,+ sdkName: isAiFeatureProductTrace+ ? AI_FEATURE_OTEL_SDK_NAME+ : INTERNAL_SDK_NAME,+ isLangfuseInternal: !isAiFeatureProductTrace,});} catch (e) {traceException(e);diff --git a/packages/shared/src/server/llm/getInternalTracingHandler.ts b/packages/shared/src/server/llm/getInternalTracingHandler.tsindex 93bacaa015..1cc0e873d3 100644--- a/packages/shared/src/server/llm/getInternalTracingHandler.ts+++ b/packages/shared/src/server/llm/getInternalTracingHandler.ts@@ -6,7 +6,7 @@ import { processEventBatch } from "../ingestion/processEventBatch";import { createUnknownSdkIngestionAttribution } from "../ingestion/ingestionAttribution";import { logger } from "../logger";import { traceException } from "../instrumentation";-import { publishAiFeatureTraceViaOtelIngestion } from "../otel/aiFeatureOtelWriter";+import { publishAiFeatureTraceViaOtelIngestion } from "../otel/internalAiFeatureOtelWriter";export function prepareInternalTraceEvents(params: {events: Array<{diff --git a/packages/shared/src/server/llm/langfuseAiCompletion.ts b/packages/shared/src/server/llm/langfuseAiCompletion.tsindex 0007d357f2..181105caed 100644--- a/packages/shared/src/server/llm/langfuseAiCompletion.ts+++ b/packages/shared/src/server/llm/langfuseAiCompletion.ts@@ -14,8 +14,10 @@ export function isLangfuseAITracingConfigured() {return Boolean(env.LANGFUSE_AI_FEATURES_PROJECT_ID);}+/** Product telemetry in the AI-features project. Eligible as an eval target. */+const LANGFUSE_AI_FEATURE_ENVIRONMENT = "production";+export function getLangfuseAITraceSinkParams(params: {- environment: TraceSinkParams["environment"];feature: string;projectId: string;traceId?: string;@@ -29,7 +31,7 @@ export function getLangfuseAITraceSinkParams(params: {}return {- environment: params.environment,+ environment: LANGFUSE_AI_FEATURE_ENVIRONMENT,traceName: params.traceName,traceId: params.traceId ?? randomBytes(16).toString("hex"),targetProjectId: env.LANGFUSE_AI_FEATURES_PROJECT_ID,@@ -40,6 +42,7 @@ export function getLangfuseAITraceSinkParams(params: {...params.metadata,},prompt: params.prompt,+ aiFeatureOtelIngestion: true,};}diff --git a/packages/shared/src/server/llm/types.ts b/packages/shared/src/server/llm/types.tsindex 182bf3323e..caf1e8af41 100644--- a/packages/shared/src/server/llm/types.ts+++ b/packages/shared/src/server/llm/types.ts@@ -556,10 +556,12 @@ export type TraceSinkParams = {eventsWriter?: InternalEventsWriter;/*** When true, v4 write modes (`dual` / `events_only`) publish through- * `publishAiFeatureTraceViaOtelIngestion` instead of `processEventBatch`.- * Used by in-app agent product traces. Those traces use environment- * `production` (not `langfuse-*`) so they stay eligible as observation-eval- * targets. Legacy write mode keeps `processEventBatch`.+ * `publishAiFeatureTraceViaOtelIngestion` instead of `processEventBatch`+ * (LangChain) or with `isLangfuseInternal: false` (AI SDK). Used by+ * in-app agent, Ask AI, and conversation-title product traces. Those+ * traces use environment `production` (not `langfuse-*`) so they stay+ * eligible as observation-eval targets. Legacy write mode keeps+ * `processEventBatch` for the LangChain path.*/aiFeatureOtelIngestion?: boolean;};diff --git a/packages/shared/src/server/otel/index.ts b/packages/shared/src/server/otel/index.tsindex 5ec05d7406..36d8e0a6a5 100644--- a/packages/shared/src/server/otel/index.ts+++ b/packages/shared/src/server/otel/index.ts@@ -1,4 +1,4 @@export * from "./OtelIngestionProcessor";export * from "./OtelMediaProcessor";export * from "./internalTraceOtelWriter";-export * from "./aiFeatureOtelWriter";+export * from "./internalAiFeatureOtelWriter";diff --git a/packages/shared/src/server/otel/aiFeatureOtelWriter.test.ts b/packages/shared/src/server/otel/internalAiFeatureOtelWriter.test.tssimilarity index 99%rename from packages/shared/src/server/otel/aiFeatureOtelWriter.test.tsrename to packages/shared/src/server/otel/internalAiFeatureOtelWriter.test.tsindex f43b0f6786..d4cb4dbb90 100644--- a/packages/shared/src/server/otel/aiFeatureOtelWriter.test.ts+++ b/packages/shared/src/server/otel/internalAiFeatureOtelWriter.test.ts@@ -4,7 +4,7 @@ import { buildInternalTraceEventInputs } from "../llm/internalTraceEvents";import {AI_FEATURE_OTEL_SDK_NAME,publishAiFeatureTraceViaOtelIngestion,-} from "./aiFeatureOtelWriter";+} from "./internalAiFeatureOtelWriter";const publishToOtelIngestionQueue = vi.fn().mockResolvedValue(undefined);const processorConstructor = vi.fn();diff --git a/packages/shared/src/server/otel/aiFeatureOtelWriter.ts b/packages/shared/src/server/otel/internalAiFeatureOtelWriter.tssimilarity index 100%rename from packages/shared/src/server/otel/aiFeatureOtelWriter.tsrename to packages/shared/src/server/otel/internalAiFeatureOtelWriter.tsdiff --git a/packages/shared/src/server/otel/internalTraceOtelWriter.ts b/packages/shared/src/server/otel/internalTraceOtelWriter.tsindex 1ade651eb0..325235cc02 100644--- a/packages/shared/src/server/otel/internalTraceOtelWriter.ts+++ b/packages/shared/src/server/otel/internalTraceOtelWriter.ts@@ -216,6 +216,12 @@ export async function publishInternalOtelSpans(params: {spans: ReadableSpan[];projectId: string;sdkName: string;+ /**+ * Judge/code-eval traces stay internal (`true`, the default). AI-feature+ * product traces (Ask AI, titles) pass `false` so observation evals can+ * match them.+ */+ isLangfuseInternal?: boolean;}): Promise<void> {const serialized = JsonTraceSerializer.serializeRequest(params.spans);if (!serialized) return;@@ -240,7 +246,7 @@ export async function publishInternalOtelSpans(params: {// schema; the public schema strips the "langfuse-" environment prefix,// exposing internal traces as user environments and bypassing the// trace-upsert eval-loop guard.- isLangfuseInternal: true,+ isLangfuseInternal: params.isLangfuseInternal ?? true,});}diff --git a/web/src/__tests__/server/in-app-agent-persistence.servertest.ts b/web/src/__tests__/server/in-app-agent-persistence.servertest.tsindex adb0b520c9..c1a940ac86 100644--- a/web/src/__tests__/server/in-app-agent-persistence.servertest.ts+++ b/web/src/__tests__/server/in-app-agent-persistence.servertest.ts@@ -592,6 +592,39 @@ describe("in-app agent persistence", () => {}});+ it("does not regenerate an already inferred conversation title", async () => {+ const originalBedrockSmallModel = env.LANGFUSE_AI_SMALL_MODEL;+ const { projectId, userId } = await createCaller();+ const conversation = await createConversation({ projectId, userId });++ await prisma.inAppAgentConversation.update({+ where: { id_projectId: { id: conversation.id, projectId } },+ data: { title: "Cluster traces by tags" },+ });++ try {+ (env as any).LANGFUSE_AI_SMALL_MODEL = "small-title-model";++ await maybeInferAndPersistConversationTitle({+ prisma,+ projectId,+ conversationId: conversation.id,+ userId,+ aiTelemetryEnabled: false,+ });++ await expect(+ prisma.inAppAgentConversation.findUniqueOrThrow({+ where: { id_projectId: { id: conversation.id, projectId } },+ select: { title: true },+ }),+ ).resolves.toEqual({ title: "Cluster traces by tags" });+ expect(mockGenerateLLMText).not.toHaveBeenCalled();+ } finally {+ (env as any).LANGFUSE_AI_SMALL_MODEL = originalBedrockSmallModel;+ }+ });+it("keeps the default title when title generation fails", async () => {const originalBedrockSmallModel = env.LANGFUSE_AI_SMALL_MODEL;const { projectId, userId } = await createCaller();diff --git a/web/src/features/natural-language-filters/server/router.ts b/web/src/features/natural-language-filters/server/router.tsindex 648300fd0c..4a9568b195 100644--- a/web/src/features/natural-language-filters/server/router.ts+++ b/web/src/features/natural-language-filters/server/router.ts@@ -6,7 +6,6 @@ import { TRPCError } from "@trpc/server";import {type ChatMessage,ChatMessageType,- LangfuseInternalTraceEnvironment,logger,generateLangfuseAIText,getClientInitiatedNonStreamingLlmTimeoutMs,@@ -133,8 +132,6 @@ export const naturalLanguageFilterRouter = createTRPCRouter({timeout: getClientInitiatedNonStreamingLlmTimeoutMs(),traceSinkParams: aiTelemetryEnabled? getLangfuseAITraceSinkParams({- environment:- LangfuseInternalTraceEnvironment.NaturalLanguageFilter,feature: "natural-language-filter",projectId: ctx.session.projectId,traceName: "natural-language-filter",diff --git a/web/src/features/search-bar/server/router.ts b/web/src/features/search-bar/server/router.tsindex b6304aa854..792d010fd3 100644--- a/web/src/features/search-bar/server/router.ts+++ b/web/src/features/search-bar/server/router.ts@@ -22,7 +22,6 @@ import {type ChatMessage,ChatMessageRole,ChatMessageType,- LangfuseInternalTraceEnvironment,logger,generateLangfuseAIText,getClientInitiatedNonStreamingLlmTimeoutMs,@@ -224,8 +223,6 @@ export const searchBarRouter = createTRPCRouter({traceSinkParams: aiTelemetryEnabled? getLangfuseAITraceSinkParams({traceId,- environment:- LangfuseInternalTraceEnvironment.NaturalLanguageFilter,feature: "search-bar-filter",projectId: ctx.session.projectId,traceName: "search-bar-filter",diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.tsindex 71d6cff860..774c1f0e96 100644--- a/worker/src/features/in-app-agent/executeInAppAgentRun.ts+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.ts@@ -16,7 +16,6 @@ import {InAppAgentRunRequestSchema,InAppAgentRunStatus,getInAppAgentInstrumentationTraceId,- IN_APP_AGENT_PRODUCT_ENVIRONMENT,parseInAppAgentInterruptEvent,type AgUiEvent,type InAppAgentRunRequest,@@ -769,7 +768,6 @@ function buildTracingConfig(params: {}const traceSinkParams = getLangfuseAITraceSinkParams({- environment: IN_APP_AGENT_PRODUCT_ENVIRONMENT,feature: "in-app-agent",projectId: params.projectId,traceId: getInAppAgentInstrumentationTraceId(params.runId),diff --git a/worker/src/features/internal-tracing/__tests__/getInternalTracingHandler.test.ts b/worker/src/features/internal-tracing/__tests__/getInternalTracingHandler.test.tsindex f5c9629a34..64a9c41549 100644--- a/worker/src/features/internal-tracing/__tests__/getInternalTracingHandler.test.ts+++ b/worker/src/features/internal-tracing/__tests__/getInternalTracingHandler.test.ts@@ -28,7 +28,7 @@ vi.mock();vi.mock(- "../../../../../packages/shared/src/server/otel/aiFeatureOtelWriter",+ "../../../../../packages/shared/src/server/otel/internalAiFeatureOtelWriter",() => ({publishAiFeatureTraceViaOtelIngestion: mockPublishAiFeatureTrace,}),