🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(otel): map cma session create as span (#18756)
Nimarb321aa5de75d
diff --git a/packages/shared/src/server/otel/ObservationTypeMapper.ts b/packages/shared/src/server/otel/ObservationTypeMapper.tsindex 89d1639332..30e5a9e71c 100644--- a/packages/shared/src/server/otel/ObservationTypeMapper.ts+++ b/packages/shared/src/server/otel/ObservationTypeMapper.ts@@ -454,10 +454,23 @@ export class ObservationTypeMapperRegistry {},),- // Priority 10: Model-based fallback+ // Priority 10: Claude Managed Agents session creation is a lifecycle span.new CustomAttributeMapper(- "ModelBased",+ "ClaudeManagedAgentSessionCreation",10,+ (attributes, _resourceAttributes, scopeData, spanName) =>+ isClaudeManagedAgentSessionCreation(+ attributes,+ scopeData?.name,+ spanName,+ ),+ () => "SPAN",+ ),++ // Priority 11: Model-based fallback+ new CustomAttributeMapper(+ "ModelBased",+ 11,(attributes, _resourceAttributes, _scopeData) => {const modelKeys = [LangfuseOtelSpanAttributes.OBSERVATION_MODEL,@@ -511,3 +524,20 @@ export class ObservationTypeMapperRegistry {return [...this.mappers];}}++/** CMA session creation configures a model without executing a model call. */+export function isClaudeManagedAgentSessionCreation(+ attributes: Record<string, unknown>,+ scopeName: unknown,+ spanName?: string,+): boolean {+ const sessionId = attributes["anthropic.session.id"];+ return (+ spanName === "anthropic.sessions.create" &&+ (scopeName === "com.anthropic.claude-platform" ||+ scopeName === "com.anthropic.sdk.typescript") &&+ !hasMeaningfulValue(attributes["gen_ai.operation.name"]) &&+ typeof sessionId === "string" &&+ sessionId.trim().length > 0+ );+}diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.tsindex 5c1ae62ac8..6d978eff7f 100644--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts@@ -36,7 +36,10 @@ import {AI_FEATURE_OTEL_SDK_NAME,LangfuseOtelSpanAttributes,} from "./attributes";-import { ObservationTypeMapperRegistry } from "./ObservationTypeMapper";+import {+ isClaudeManagedAgentSessionCreation,+ ObservationTypeMapperRegistry,+} from "./ObservationTypeMapper";import { env } from "../../env";import { OtelIngestionQueue } from "../redis/otelIngestionQueue";import { isValidDateString, flattenJsonToPathArrays } from "./utils";@@ -528,13 +531,26 @@ export class OtelIngestionProcessor {source: "event" as const,};+ const observationType =+ observationTypeMapper.mapToObservationType(+ spanAttributes,+ resourceAttributes,+ scopeSpan?.scope,+ span.name,+ );// AI SDK agent spans carry aggregate usage duplicating their// child model-call spans — skip model/usage/cost for them.- const isAiSdkAgentSpan =- this.isAiSdkAgentOperation(spanAttributes);+ // Also Claude Managed Agent create_session spans have a model logged+ // but no usage+ const skipModelUsageAndCost = this.shouldSkipModelUsageAndCost(+ spanAttributes,+ observationType,+ scopeName,+ span.name,+ );const usageDetails = UsageDetails.safeParse(- isAiSdkAgentSpan+ skipModelUsageAndCost? {}: this.extractUsageDetails(spanAttributes,@@ -561,13 +577,6 @@ export class OtelIngestionProcessor {? normalizedTools.toolCallNames: undefined;- const observationType =- observationTypeMapper.mapToObservationType(- spanAttributes,- resourceAttributes,- scopeSpan?.scope,- span.name,- );// Prompts can only be linked to GENERATION observationsconst canLinkPrompt =observationType === ObservationType.GENERATION;@@ -632,7 +641,7 @@ export class OtelIngestionProcessor {spanAttributes,scopeSpan?.scope?.name ?? "",),- modelName: isAiSdkAgentSpan+ modelName: skipModelUsageAndCost? undefined: this.extractModelName(spanAttributes),completionStartTime: this.extractCompletionStartTime(@@ -644,7 +653,7 @@ export class OtelIngestionProcessor {providedUsageDetails: usageDetails.success? usageDetails.data: undefined,- providedCostDetails: isAiSdkAgentSpan+ providedCostDetails: skipModelUsageAndCost? {}: this.extractCostDetails(spanAttributes, spanContext),@@ -1267,16 +1276,18 @@ export class OtelIngestionProcessor {metadata,);- // AI SDK agent spans carry aggregate usage duplicating their child- // model-call spans — skip model/usage/cost for them.- const isAiSdkAgentSpan = this.isAiSdkAgentOperation(attributes);-const mappedObservationType = observationTypeMapper.mapToObservationType(attributes,resourceAttributes,scopeSpan?.scope,span.name,);+ const skipModelUsageAndCost = this.shouldSkipModelUsageAndCost(+ attributes,+ mappedObservationType,+ instrumentationScopeName,+ span.name,+ );// Prompts can only be linked to GENERATION observationsconst canLinkPrompt = mappedObservationType === ObservationType.GENERATION;@@ -1320,7 +1331,9 @@ export class OtelIngestionProcessor {attributes,instrumentationScopeName,) as any,- model: isAiSdkAgentSpan ? undefined : this.extractModelName(attributes),+ model: skipModelUsageAndCost+ ? undefined+ : this.extractModelName(attributes),promptName: canLinkPrompt? (attributes?.[LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_NAME] ??attributes["langfuse.prompt.name"] ??@@ -1330,14 +1343,14 @@ export class OtelIngestionProcessor {promptVersion: canLinkPrompt? this.extractPromptVersion(attributes): null,- usageDetails: isAiSdkAgentSpan+ usageDetails: skipModelUsageAndCost? {}: this.extractUsageDetails(attributes,instrumentationScopeName,observationContext,),- costDetails: isAiSdkAgentSpan+ costDetails: skipModelUsageAndCost? {}: this.extractCostDetails(attributes, observationContext),input: normalizedToolMetadata.input,@@ -2884,23 +2897,27 @@ export class OtelIngestionProcessor {}/**- * The Vercel AI SDK OTel integration (@ai-sdk/otel) emits an- * `invoke_agent` span (and `agent_step` child spans) that carry aggregate- * `gen_ai.usage.*` token counts duplicating the per-call usage on the- * grandchild model-call span (`gen_ai.operation.name: "chat"`). Populating- * model/usage/cost on the agent spans as well as the model-call span would- * classify both as generations and double every trace's cost, so model,- * usage, and cost extraction is skipped for these spans. Agent-type spans- * from other instrumentations (e.g. OpenInference `AGENT` spans) are not- * affected because the gate is on the operation name, not the observation- * type.+ * The Vercel AI SDK OTel integration (@ai-sdk/otel) emits `invoke_agent`+ * and `agent_step` spans carrying aggregate usage from child model calls.+ * Skip their model/usage/cost to avoid double counting. Other AGENT spans+ * are unaffected unless they have one of these operation names.+ *+ * CMA session creation only configures a model and must not trigger enrichment.*/- private isAiSdkAgentOperation(attributes: Record<string, unknown>): boolean {+ private shouldSkipModelUsageAndCost(+ attributes: Record<string, unknown>,+ observationType: ObservationType,+ scopeName: string | undefined,+ spanName: string,+ ): boolean {const operationName = attributes["gen_ai.operation.name"];+ if (operationName === "invoke_agent" || operationName === "agent_step") {+ return true;+ }return (- typeof operationName === "string" &&- ["invoke_agent", "agent_step"].includes(operationName)+ observationType === ObservationType.SPAN &&+ isClaudeManagedAgentSessionCreation(attributes, scopeName, spanName));}diff --git a/web/src/__tests__/server/api/otel/otelMapping.servertest.ts b/web/src/__tests__/server/api/otel/otelMapping.servertest.tsindex 93391b532e..f5e7616bb5 100644--- a/web/src/__tests__/server/api/otel/otelMapping.servertest.ts+++ b/web/src/__tests__/server/api/otel/otelMapping.servertest.ts@@ -5780,6 +5780,129 @@ describe("OTel Resource Span Mapping", () => {).toBeUndefined();});+ it.each([+ {+ scenario: "maps platform session creation to a span",+ expectedType: "SPAN",+ expectedModel: undefined,+ },+ {+ scenario: "maps SDK session creation to a span",+ scopeName: "com.anthropic.sdk.typescript",+ expectedType: "SPAN",+ expectedModel: undefined,+ },+ {+ scenario: "preserves other instrumentation scopes",+ scopeName: "other-sdk",+ expectedType: "GENERATION",+ expectedModel: "claude-test-model",+ },+ {+ scenario: "requires a managed session ID",+ sessionId: null,+ expectedType: "GENERATION",+ expectedModel: "claude-test-model",+ },+ {+ scenario: "preserves other model spans",+ spanName: "anthropic.messages.create",+ expectedType: "GENERATION",+ expectedModel: "claude-test-model",+ },+ {+ scenario: "respects an explicit generation type",+ extraAttributes: { "langfuse.observation.type": "generation" },+ expectedType: "GENERATION",+ expectedModel: "claude-test-model",+ },+ {+ scenario: "preserves model fields on explicitly typed chat spans",+ extraAttributes: {+ "langfuse.observation.type": "span",+ "gen_ai.operation.name": "chat",+ },+ expectedType: "SPAN",+ expectedModel: "claude-test-model",+ },+ ])(+ "CMA session creation: $scenario",+ async ({+ scopeName = "com.anthropic.claude-platform",+ spanName = "anthropic.sessions.create",+ sessionId = "session-example",+ extraAttributes = {},+ expectedType,+ expectedModel,+ }) => {+ const attributes: Record<string, string> = {+ "gen_ai.request.model": "claude-test-model",+ "gen_ai.usage.input_tokens": "12",+ "langfuse.observation.cost_details": JSON.stringify({ total: 0.12 }),+ };+ Object.assign(attributes, extraAttributes);+ if (sessionId !== null) {+ attributes["anthropic.session.id"] = sessionId;+ }+ const resourceSpan: ResourceSpan = {+ scopeSpans: [+ {+ scope: { name: scopeName },+ spans: [+ {+ ...defaultSpanProps,+ traceId: Buffer.from(defaultSpanProps.traceId.data),+ spanId: Buffer.from(defaultSpanProps.spanId.data),+ name: spanName,+ attributes: Object.entries(attributes).map(+ ([key, value]) => ({+ key,+ value: { stringValue: value },+ }),+ ),+ },+ ],+ },+ ],+ };++ const events = await convertOtelSpanToIngestionEvent(+ resourceSpan,+ new Set(),+ );+ const observation = events.find(+ (event) => event.type !== "trace-create",+ );+ const expectedUsage = expectedModel ? { input: 12 } : undefined;+ const expectedCost = expectedModel ? { total: 0.12 } : {};+ expect(observation).toMatchObject({+ type: `${expectedType.toLowerCase()}-create`,+ body: {+ model: expectedModel,+ metadata: {+ attributes: { "gen_ai.request.model": "claude-test-model" },+ },+ },+ });+ expect(observation?.body.usageDetails).toEqual(expectedUsage ?? {});+ expect(observation?.body).toHaveProperty("costDetails", expectedCost);++ const eventInputs = createTestOtelProcessor().processToEvent([+ resourceSpan,+ ]);+ expect(eventInputs).toHaveLength(1);+ expect(eventInputs[0]).toMatchObject({+ type: expectedType,+ modelName: expectedModel,+ metadata: {+ attributes: { "gen_ai.request.model": "claude-test-model" },+ },+ });+ expect(eventInputs[0].providedUsageDetails).toEqual(expectedUsage);+ expect(eventInputs[0].providedCostDetails).toEqual(expectedCost);+ },+ );+it.each([{scenario: "names the agent",