🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(otel): map flat Anthropic cache token aliases to canonical usage buckets (#15411)
Milana Gurbanovabebd9bdf483d
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.tsindex 94be47aefe..a39c9315d8 100644--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts@@ -2841,6 +2841,7 @@ export class OtelIngestionProcessor {rawUsageDetails["total_tokens"] ?? rawUsageDetails["total"];const cacheReadTokens =rawUsageDetails["cache_read.input_tokens"] ??+ rawUsageDetails["cache_read_input_tokens"] ??rawUsageDetails["cache_read_tokens"] ??rawUsageDetails["details.cache_read_tokens"] ??rawUsageDetails["details.cache_read_input_tokens"] ??@@ -2848,6 +2849,7 @@ export class OtelIngestionProcessor {rawUsageDetails["input_cached_tokens"];const cacheCreationTokens =rawUsageDetails["cache_creation.input_tokens"] ??+ rawUsageDetails["cache_creation_input_tokens"] ??rawUsageDetails["cache_write_tokens"] ??rawUsageDetails["details.cache_write_tokens"] ??rawUsageDetails["details.cache_creation_input_tokens"] ??@@ -2873,12 +2875,14 @@ export class OtelIngestionProcessor {"total_tokens","total","cache_read.input_tokens",+ "cache_read_input_tokens","cache_read_tokens","details.cache_read_tokens","details.cache_read_input_tokens","prompt_details.cache_read","input_cached_tokens","cache_creation.input_tokens",+ "cache_creation_input_tokens","cache_write_tokens","details.cache_write_tokens","details.cache_creation_input_tokens",diff --git a/web/src/__tests__/server/api/otel/otelMapping.servertest.ts b/web/src/__tests__/server/api/otel/otelMapping.servertest.tsindex 7f1bd6739b..43aa561691 100644--- a/web/src/__tests__/server/api/otel/otelMapping.servertest.ts+++ b/web/src/__tests__/server/api/otel/otelMapping.servertest.ts@@ -7969,6 +7969,261 @@ describe("OTel Resource Span Mapping", () => {expect(usageDetails.output_reasoning_tokens).toBe(25);expect(usageDetails["reasoning.output_tokens"]).toBeUndefined();});+ it("should normalize raw Anthropic cache_read_input_tokens / cache_creation_input_tokens into Langfuse canonical keys", async () => {+ // flat Anthropic cache spellings must map to cache aliases, not opaque passthrough buckets+ const traceId = "abcdef1234567890abcdef1234567893";++ const litellmAnthropicSpan = {+ resource: {+ attributes: [+ {+ key: "service.name",+ value: { stringValue: "test-service" },+ },+ ],+ },+ scopeSpans: [+ {+ scope: {+ name: "litellm",+ version: "1.0.0",+ },+ spans: [+ {+ traceId: Buffer.from(traceId, "hex"),+ spanId: Buffer.from("1234567890abcde2", "hex"),+ name: "anthropic-raw-cache-usage",+ kind: 1,+ startTimeUnixNano: {+ low: 1000000,+ high: 406528574,+ unsigned: true,+ },+ endTimeUnixNano: {+ low: 2000000,+ high: 406528574,+ unsigned: true,+ },+ attributes: [+ {+ key: "gen_ai.usage.input_tokens",+ value: { intValue: { low: 100, high: 0, unsigned: false } },+ },+ {+ key: "gen_ai.usage.output_tokens",+ value: { intValue: { low: 40, high: 0, unsigned: false } },+ },+ {+ key: "gen_ai.usage.cache_read_input_tokens",+ value: { intValue: { low: 20, high: 0, unsigned: false } },+ },+ {+ key: "gen_ai.usage.cache_creation_input_tokens",+ value: { intValue: { low: 10, high: 0, unsigned: false } },+ },+ ],+ status: {},+ },+ ],+ },+ ],+ };++ const events = await convertOtelSpanToIngestionEvent(+ litellmAnthropicSpan,+ new Set(),+ );+ const observationEvent = events.find(+ (e) => e.type === "generation-create" || e.type === "span-create",+ );++ expect(observationEvent).toBeDefined();+ // input must be the uncached remainder: 100 - 20 - 10 = 70+ expect(observationEvent?.body.usageDetails.input).toBe(70);+ expect(observationEvent?.body.usageDetails.output).toBe(40);+ expect(observationEvent?.body.usageDetails.input_cached_tokens).toBe(20);+ expect(observationEvent?.body.usageDetails.input_cache_creation).toBe(10);+ // sum of all input* buckets must equal the reported (inclusive) input count+ const inputSum = Object.entries(+ observationEvent?.body.usageDetails as Record<string, number>,+ )+ .filter(([key]) => key.startsWith("input"))+ .reduce((acc, [, value]) => acc + value, 0);+ expect(inputSum).toBe(100);+ // The raw Anthropic keys must NOT be passed through after normalization+ expect(+ observationEvent?.body.usageDetails["cache_read_input_tokens"],+ ).toBeUndefined();+ expect(+ observationEvent?.body.usageDetails["cache_creation_input_tokens"],+ ).toBeUndefined();+ });++ it("should subtract cache tokens only once when dotted and flat Anthropic spellings arrive together", async () => {+ // dotted and flat spellings for the same value must subtract exactly once+ const traceId = "abcdef1234567890abcdef1234567896";++ const bothSpellingsSpan = {+ resource: {+ attributes: [+ {+ key: "service.name",+ value: { stringValue: "test-service" },+ },+ ],+ },+ scopeSpans: [+ {+ scope: {+ name: "litellm",+ version: "1.0.0",+ },+ spans: [+ {+ traceId: Buffer.from(traceId, "hex"),+ spanId: Buffer.from("1234567890abcde5", "hex"),+ name: "anthropic-both-cache-spellings",+ kind: 1,+ startTimeUnixNano: {+ low: 1000000,+ high: 406528574,+ unsigned: true,+ },+ endTimeUnixNano: {+ low: 2000000,+ high: 406528574,+ unsigned: true,+ },+ attributes: [+ {+ key: "gen_ai.usage.input_tokens",+ value: { intValue: { low: 100, high: 0, unsigned: false } },+ },+ {+ key: "gen_ai.usage.cache_read.input_tokens",+ value: { intValue: { low: 20, high: 0, unsigned: false } },+ },+ {+ key: "gen_ai.usage.cache_read_input_tokens",+ value: { intValue: { low: 20, high: 0, unsigned: false } },+ },+ {+ key: "gen_ai.usage.cache_creation.input_tokens",+ value: { intValue: { low: 10, high: 0, unsigned: false } },+ },+ {+ key: "gen_ai.usage.cache_creation_input_tokens",+ value: { intValue: { low: 10, high: 0, unsigned: false } },+ },+ ],+ status: {},+ },+ ],+ },+ ],+ };++ const events = await convertOtelSpanToIngestionEvent(+ bothSpellingsSpan,+ new Set(),+ );+ const observationEvent = events.find(+ (e) => e.type === "generation-create" || e.type === "span-create",+ );++ expect(observationEvent).toBeDefined();+ const usageDetails = observationEvent?.body.usageDetails as Record<+ string,+ number+ >;+ // single subtraction only: 100 - 20 - 10 = 70, not 100 - 2*20 - 2*10 = 40+ expect(usageDetails.input).toBe(70);+ expect(usageDetails.input_cached_tokens).toBe(20);+ expect(usageDetails.input_cache_creation).toBe(10);+ // neither raw spelling may be passed through as its own bucket+ expect(usageDetails["cache_read.input_tokens"]).toBeUndefined();+ expect(usageDetails["cache_read_input_tokens"]).toBeUndefined();+ expect(usageDetails["cache_creation.input_tokens"]).toBeUndefined();+ expect(usageDetails["cache_creation_input_tokens"]).toBeUndefined();+ });++ it("should let a dotted cache value of 0 shadow the flat spelling (?? alias chain)", async () => {+ // pins pre-existing ??-chain behavior: a dotted 0 shadows the flat spelling+ const traceId = "abcdef1234567890abcdef1234567897";++ const dottedZeroSpan = {+ resource: {+ attributes: [+ {+ key: "service.name",+ value: { stringValue: "test-service" },+ },+ ],+ },+ scopeSpans: [+ {+ scope: {+ name: "litellm",+ version: "1.0.0",+ },+ spans: [+ {+ traceId: Buffer.from(traceId, "hex"),+ spanId: Buffer.from("1234567890abcde6", "hex"),+ name: "anthropic-dotted-zero-cache",+ kind: 1,+ startTimeUnixNano: {+ low: 1000000,+ high: 406528574,+ unsigned: true,+ },+ endTimeUnixNano: {+ low: 2000000,+ high: 406528574,+ unsigned: true,+ },+ attributes: [+ {+ key: "gen_ai.usage.input_tokens",+ value: { intValue: { low: 100, high: 0, unsigned: false } },+ },+ {+ key: "gen_ai.usage.cache_read.input_tokens",+ value: { intValue: { low: 0, high: 0, unsigned: false } },+ },+ {+ key: "gen_ai.usage.cache_read_input_tokens",+ value: { intValue: { low: 20, high: 0, unsigned: false } },+ },+ ],+ status: {},+ },+ ],+ },+ ],+ };++ const events = await convertOtelSpanToIngestionEvent(+ dottedZeroSpan,+ new Set(),+ );+ const observationEvent = events.find(+ (e) => e.type === "generation-create" || e.type === "span-create",+ );++ expect(observationEvent).toBeDefined();+ const usageDetails = observationEvent?.body.usageDetails as Record<+ string,+ number+ >;+ // input stays 100: the dotted 0 wins the ?? chain, the flat 20 is never subtracted+ expect(usageDetails.input).toBe(100);+ // the winning dotted 0 materializes as a zero-valued cache bucket+ expect(usageDetails.input_cached_tokens).toBe(0);+ // neither raw spelling is passed through as its own bucket+ expect(usageDetails["cache_read.input_tokens"]).toBeUndefined();+ expect(usageDetails["cache_read_input_tokens"]).toBeUndefined();+ });});describe("Google ADK blob-derived cache usage", () => {