Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(ai-features): ingest Ask AI and titles as production OTel (#16717)

Lysander Kiesele65f2e0693b0
diff --git a/packages/shared/src/in-app-agent/server/persistence.ts b/packages/shared/src/in-app-agent/server/persistence.ts
index 6155931e03..6363a40d9b 100644
--- a/packages/shared/src/in-app-agent/server/persistence.ts
+++ b/packages/shared/src/in-app-agent/server/persistence.ts
@@ -6,12 +6,7 @@ import {
InAppAgentRunErrorCode,
InAppAgentRunStatus,
} from "../../features/inAppAgent/types";
-import {
- ChatMessageRole,
- ChatMessageType,
- LangfuseInternalTraceEnvironment,
- logger,
-} from "../../server";
+import { ChatMessageRole, ChatMessageType, logger } from "../../server";
import { isSettledInAppAgentRunStatus } from "../constants";
import { recordRunTerminalOutcome } from "./runMetrics";
import { Prisma } from "../../db";
@@ -478,6 +473,10 @@ export async function maybeInferAndPersistConversationTitle(params: {
return;
}
+ if (!isUnsetConversationTitle(conversation.title)) {
+ return;
+ }
+
const transcript = buildConversationTitleTranscript(
await getConversationMessages({
prisma: params.prisma,
@@ -548,7 +547,6 @@ ${JSON.stringify(transcript, null, 2)}
maxTokens: 1000,
traceSinkParams: params.aiTelemetryEnabled
? getLangfuseAITraceSinkParams({
- environment: LangfuseInternalTraceEnvironment.InAppAgent,
feature: "in-app-agent-conversation-title",
projectId: params.projectId,
traceName: "in-app-agent-conversation-title",
@@ -1504,6 +1502,14 @@ function getDefaultConversationTitle(date: Date) {
return `Chat on ${weekday} at ${hours}:${minutes}`;
}
+function isUnsetConversationTitle(title: string | null) {
+ if (!title) {
+ return true;
+ }
+
+ return /^Chat on [A-Za-z]+ at \d{2}:\d{2}$/.test(title);
+}
+
export function buildConversationTitleTranscript(
messages: readonly AgUiMessage[],
) {
diff --git a/packages/shared/src/server/llm/ai-sdk/telemetry.ts b/packages/shared/src/server/llm/ai-sdk/telemetry.ts
index dbac53db05..995feae50e 100644
--- a/packages/shared/src/server/llm/ai-sdk/telemetry.ts
+++ b/packages/shared/src/server/llm/ai-sdk/telemetry.ts
@@ -24,6 +24,7 @@ import { stringifyValue } from "../../../utils/stringChecks";
import { traceException } from "../../instrumentation";
import { logger } from "../../logger";
import { LangfuseOtelSpanAttributes } from "../../otel/attributes";
+import { AI_FEATURE_OTEL_SDK_NAME } from "../../otel/internalAiFeatureOtelWriter";
import { publishInternalOtelSpans } from "../../otel/internalTraceOtelWriter";
import type { InternalTraceExperimentContext } from "../internalTraceEvents";
import type { TraceSinkParams } from "../types";
@@ -79,7 +80,8 @@ export type AiSdkTelemetryCapture = {
* experiment observation evals.
*
* Returns `undefined` (no tracing) when the environment is not
- * langfuse-prefixed (the eval-loop safeguard) or when the trace ID is not a
+ * langfuse-prefixed (the eval-loop safeguard) unless this is an AI-feature
+ * product trace (`aiFeatureOtelIngestion`), or when the trace ID is not a
* valid W3C trace ID.
*/
export function createAiSdkTelemetryCapture(params: {
@@ -88,11 +90,19 @@ export function createAiSdkTelemetryCapture(params: {
rootInput?: unknown;
}): AiSdkTelemetryCapture | undefined {
const { traceSinkParams, rootInput } = params;
+ const isAiFeatureProductTrace = Boolean(
+ traceSinkParams.aiFeatureOtelIngestion,
+ );
// Safeguard: All internal traces must use LangfuseInternalTraceEnvironment enum values
// This prevents infinite eval loops (user trace → eval → eval trace → another eval)
// See corresponding check in worker/src/features/evaluation/evalService.ts createEvalJobs()
- if (!traceSinkParams.environment?.startsWith("langfuse")) {
+ // AI-feature product traces are the exception: they use environment
+ // `production` so observation evals can match them.
+ if (
+ !isAiFeatureProductTrace &&
+ !traceSinkParams.environment?.startsWith("langfuse")
+ ) {
logger.warn(
"Skipping trace creation: internal traces must use LangfuseInternalTraceEnvironment enum",
{
@@ -270,7 +280,10 @@ export function createAiSdkTelemetryCapture(params: {
await publishInternalOtelSpans({
spans: matchingSpans,
projectId: traceSinkParams.targetProjectId,
- sdkName: INTERNAL_SDK_NAME,
+ sdkName: isAiFeatureProductTrace
+ ? AI_FEATURE_OTEL_SDK_NAME
+ : INTERNAL_SDK_NAME,
+ isLangfuseInternal: !isAiFeatureProductTrace,
});
} catch (e) {
traceException(e);
diff --git a/packages/shared/src/server/llm/getInternalTracingHandler.ts b/packages/shared/src/server/llm/getInternalTracingHandler.ts
index 93bacaa015..1cc0e873d3 100644
--- a/packages/shared/src/server/llm/getInternalTracingHandler.ts
+++ b/packages/shared/src/server/llm/getInternalTracingHandler.ts
@@ -6,7 +6,7 @@ import { processEventBatch } from "../ingestion/processEventBatch";
import { createUnknownSdkIngestionAttribution } from "../ingestion/ingestionAttribution";
import { logger } from "../logger";
import { traceException } from "../instrumentation";
-import { publishAiFeatureTraceViaOtelIngestion } from "../otel/aiFeatureOtelWriter";
+import { publishAiFeatureTraceViaOtelIngestion } from "../otel/internalAiFeatureOtelWriter";
export function prepareInternalTraceEvents(params: {
events: Array<{
diff --git a/packages/shared/src/server/llm/langfuseAiCompletion.ts b/packages/shared/src/server/llm/langfuseAiCompletion.ts
index 0007d357f2..181105caed 100644
--- a/packages/shared/src/server/llm/langfuseAiCompletion.ts
+++ b/packages/shared/src/server/llm/langfuseAiCompletion.ts
@@ -14,8 +14,10 @@ export function isLangfuseAITracingConfigured() {
return Boolean(env.LANGFUSE_AI_FEATURES_PROJECT_ID);
}
+/** Product telemetry in the AI-features project. Eligible as an eval target. */
+const LANGFUSE_AI_FEATURE_ENVIRONMENT = "production";
+
export function getLangfuseAITraceSinkParams(params: {
- environment: TraceSinkParams["environment"];
feature: string;
projectId: string;
traceId?: string;
@@ -29,7 +31,7 @@ export function getLangfuseAITraceSinkParams(params: {
}
return {
- environment: params.environment,
+ environment: LANGFUSE_AI_FEATURE_ENVIRONMENT,
traceName: params.traceName,
traceId: params.traceId ?? randomBytes(16).toString("hex"),
targetProjectId: env.LANGFUSE_AI_FEATURES_PROJECT_ID,
@@ -40,6 +42,7 @@ export function getLangfuseAITraceSinkParams(params: {
...params.metadata,
},
prompt: params.prompt,
+ aiFeatureOtelIngestion: true,
};
}
diff --git a/packages/shared/src/server/llm/types.ts b/packages/shared/src/server/llm/types.ts
index 182bf3323e..caf1e8af41 100644
--- a/packages/shared/src/server/llm/types.ts
+++ b/packages/shared/src/server/llm/types.ts
@@ -556,10 +556,12 @@ export type TraceSinkParams = {
eventsWriter?: InternalEventsWriter;
/**
* When true, v4 write modes (`dual` / `events_only`) publish through
- * `publishAiFeatureTraceViaOtelIngestion` instead of `processEventBatch`.
- * Used by in-app agent product traces. Those traces use environment
- * `production` (not `langfuse-*`) so they stay eligible as observation-eval
- * targets. Legacy write mode keeps `processEventBatch`.
+ * `publishAiFeatureTraceViaOtelIngestion` instead of `processEventBatch`
+ * (LangChain) or with `isLangfuseInternal: false` (AI SDK). Used by
+ * in-app agent, Ask AI, and conversation-title product traces. Those
+ * traces use environment `production` (not `langfuse-*`) so they stay
+ * eligible as observation-eval targets. Legacy write mode keeps
+ * `processEventBatch` for the LangChain path.
*/
aiFeatureOtelIngestion?: boolean;
};
diff --git a/packages/shared/src/server/otel/index.ts b/packages/shared/src/server/otel/index.ts
index 5ec05d7406..36d8e0a6a5 100644
--- a/packages/shared/src/server/otel/index.ts
+++ b/packages/shared/src/server/otel/index.ts
@@ -1,4 +1,4 @@
export * from "./OtelIngestionProcessor";
export * from "./OtelMediaProcessor";
export * from "./internalTraceOtelWriter";
-export * from "./aiFeatureOtelWriter";
+export * from "./internalAiFeatureOtelWriter";
diff --git a/packages/shared/src/server/otel/aiFeatureOtelWriter.test.ts b/packages/shared/src/server/otel/internalAiFeatureOtelWriter.test.ts
similarity index 99%
rename from packages/shared/src/server/otel/aiFeatureOtelWriter.test.ts
rename to packages/shared/src/server/otel/internalAiFeatureOtelWriter.test.ts
index f43b0f6786..d4cb4dbb90 100644
--- a/packages/shared/src/server/otel/aiFeatureOtelWriter.test.ts
+++ b/packages/shared/src/server/otel/internalAiFeatureOtelWriter.test.ts
@@ -4,7 +4,7 @@ import { buildInternalTraceEventInputs } from "../llm/internalTraceEvents";
import {
AI_FEATURE_OTEL_SDK_NAME,
publishAiFeatureTraceViaOtelIngestion,
-} from "./aiFeatureOtelWriter";
+} from "./internalAiFeatureOtelWriter";
const publishToOtelIngestionQueue = vi.fn().mockResolvedValue(undefined);
const processorConstructor = vi.fn();
diff --git a/packages/shared/src/server/otel/aiFeatureOtelWriter.ts b/packages/shared/src/server/otel/internalAiFeatureOtelWriter.ts
similarity index 100%
rename from packages/shared/src/server/otel/aiFeatureOtelWriter.ts
rename to packages/shared/src/server/otel/internalAiFeatureOtelWriter.ts
diff --git a/packages/shared/src/server/otel/internalTraceOtelWriter.ts b/packages/shared/src/server/otel/internalTraceOtelWriter.ts
index 1ade651eb0..325235cc02 100644
--- a/packages/shared/src/server/otel/internalTraceOtelWriter.ts
+++ b/packages/shared/src/server/otel/internalTraceOtelWriter.ts
@@ -216,6 +216,12 @@ export async function publishInternalOtelSpans(params: {
spans: ReadableSpan[];
projectId: string;
sdkName: string;
+ /**
+ * Judge/code-eval traces stay internal (`true`, the default). AI-feature
+ * product traces (Ask AI, titles) pass `false` so observation evals can
+ * match them.
+ */
+ isLangfuseInternal?: boolean;
}): Promise<void> {
const serialized = JsonTraceSerializer.serializeRequest(params.spans);
if (!serialized) return;
@@ -240,7 +246,7 @@ export async function publishInternalOtelSpans(params: {
// schema; the public schema strips the "langfuse-" environment prefix,
// exposing internal traces as user environments and bypassing the
// trace-upsert eval-loop guard.
- isLangfuseInternal: true,
+ isLangfuseInternal: params.isLangfuseInternal ?? true,
});
}
diff --git a/web/src/__tests__/server/in-app-agent-persistence.servertest.ts b/web/src/__tests__/server/in-app-agent-persistence.servertest.ts
index adb0b520c9..c1a940ac86 100644
--- a/web/src/__tests__/server/in-app-agent-persistence.servertest.ts
+++ b/web/src/__tests__/server/in-app-agent-persistence.servertest.ts
@@ -592,6 +592,39 @@ describe("in-app agent persistence", () => {
}
});
+ it("does not regenerate an already inferred conversation title", async () => {
+ const originalBedrockSmallModel = env.LANGFUSE_AI_SMALL_MODEL;
+ const { projectId, userId } = await createCaller();
+ const conversation = await createConversation({ projectId, userId });
+
+ await prisma.inAppAgentConversation.update({
+ where: { id_projectId: { id: conversation.id, projectId } },
+ data: { title: "Cluster traces by tags" },
+ });
+
+ try {
+ (env as any).LANGFUSE_AI_SMALL_MODEL = "small-title-model";
+
+ await maybeInferAndPersistConversationTitle({
+ prisma,
+ projectId,
+ conversationId: conversation.id,
+ userId,
+ aiTelemetryEnabled: false,
+ });
+
+ await expect(
+ prisma.inAppAgentConversation.findUniqueOrThrow({
+ where: { id_projectId: { id: conversation.id, projectId } },
+ select: { title: true },
+ }),
+ ).resolves.toEqual({ title: "Cluster traces by tags" });
+ expect(mockGenerateLLMText).not.toHaveBeenCalled();
+ } finally {
+ (env as any).LANGFUSE_AI_SMALL_MODEL = originalBedrockSmallModel;
+ }
+ });
+
it("keeps the default title when title generation fails", async () => {
const originalBedrockSmallModel = env.LANGFUSE_AI_SMALL_MODEL;
const { projectId, userId } = await createCaller();
diff --git a/web/src/features/natural-language-filters/server/router.ts b/web/src/features/natural-language-filters/server/router.ts
index 648300fd0c..4a9568b195 100644
--- a/web/src/features/natural-language-filters/server/router.ts
+++ b/web/src/features/natural-language-filters/server/router.ts
@@ -6,7 +6,6 @@ import { TRPCError } from "@trpc/server";
import {
type ChatMessage,
ChatMessageType,
- LangfuseInternalTraceEnvironment,
logger,
generateLangfuseAIText,
getClientInitiatedNonStreamingLlmTimeoutMs,
@@ -133,8 +132,6 @@ export const naturalLanguageFilterRouter = createTRPCRouter({
timeout: getClientInitiatedNonStreamingLlmTimeoutMs(),
traceSinkParams: aiTelemetryEnabled
? getLangfuseAITraceSinkParams({
- environment:
- LangfuseInternalTraceEnvironment.NaturalLanguageFilter,
feature: "natural-language-filter",
projectId: ctx.session.projectId,
traceName: "natural-language-filter",
diff --git a/web/src/features/search-bar/server/router.ts b/web/src/features/search-bar/server/router.ts
index b6304aa854..792d010fd3 100644
--- a/web/src/features/search-bar/server/router.ts
+++ b/web/src/features/search-bar/server/router.ts
@@ -22,7 +22,6 @@ import {
type ChatMessage,
ChatMessageRole,
ChatMessageType,
- LangfuseInternalTraceEnvironment,
logger,
generateLangfuseAIText,
getClientInitiatedNonStreamingLlmTimeoutMs,
@@ -224,8 +223,6 @@ export const searchBarRouter = createTRPCRouter({
traceSinkParams: aiTelemetryEnabled
? getLangfuseAITraceSinkParams({
traceId,
- environment:
- LangfuseInternalTraceEnvironment.NaturalLanguageFilter,
feature: "search-bar-filter",
projectId: ctx.session.projectId,
traceName: "search-bar-filter",
diff --git a/worker/src/features/in-app-agent/executeInAppAgentRun.ts b/worker/src/features/in-app-agent/executeInAppAgentRun.ts
index 71d6cff860..774c1f0e96 100644
--- a/worker/src/features/in-app-agent/executeInAppAgentRun.ts
+++ b/worker/src/features/in-app-agent/executeInAppAgentRun.ts
@@ -16,7 +16,6 @@ import {
InAppAgentRunRequestSchema,
InAppAgentRunStatus,
getInAppAgentInstrumentationTraceId,
- IN_APP_AGENT_PRODUCT_ENVIRONMENT,
parseInAppAgentInterruptEvent,
type AgUiEvent,
type InAppAgentRunRequest,
@@ -769,7 +768,6 @@ function buildTracingConfig(params: {
}
const traceSinkParams = getLangfuseAITraceSinkParams({
- environment: IN_APP_AGENT_PRODUCT_ENVIRONMENT,
feature: "in-app-agent",
projectId: params.projectId,
traceId: getInAppAgentInstrumentationTraceId(params.runId),
diff --git a/worker/src/features/internal-tracing/__tests__/getInternalTracingHandler.test.ts b/worker/src/features/internal-tracing/__tests__/getInternalTracingHandler.test.ts
index f5c9629a34..64a9c41549 100644
--- a/worker/src/features/internal-tracing/__tests__/getInternalTracingHandler.test.ts
+++ b/worker/src/features/internal-tracing/__tests__/getInternalTracingHandler.test.ts
@@ -28,7 +28,7 @@ vi.mock(
);
vi.mock(
- "../../../../../packages/shared/src/server/otel/aiFeatureOtelWriter",
+ "../../../../../packages/shared/src/server/otel/internalAiFeatureOtelWriter",
() => ({
publishAiFeatureTraceViaOtelIngestion: mockPublishAiFeatureTrace,
}),