🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(otel): stop double counting the cache-write TTL breakdown in generic gen_ai usage mapping (#19420)
Hassieb Pakzade0fad7b26221
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.tsindex e697dd4cbc..7ed785b3ea 100644--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts@@ -3364,6 +3364,21 @@ export class OtelIngestionProcessor {rawUsageDetails["details.cache_creation_input_tokens"] ??rawUsageDetails["prompt_details.cache_write"] ??rawUsageDetails["input_cache_creation"];+ // TTL breakdown of the cache-write total; only the unattributed remainder+ // stays in input_cache_creation so each write is counted and priced once.+ const cacheCreation5mTokens =+ rawUsageDetails["cache_creation.5m_input_tokens"];+ const cacheCreation1hTokens =+ rawUsageDetails["cache_creation.1h_input_tokens"];+ const hasCacheCreationBreakdown =+ cacheCreation5mTokens !== undefined ||+ cacheCreation1hTokens !== undefined;+ const cacheCreationBreakdownTokens =+ (cacheCreation5mTokens ?? 0) + (cacheCreation1hTokens ?? 0);+ const cacheCreationRemainderTokens =+ cacheCreationTokens !== undefined && hasCacheCreationBreakdown+ ? Math.max(cacheCreationTokens - cacheCreationBreakdownTokens, 0)+ : cacheCreationTokens;// Reasoning/audio details are included in the emitted output token count// and are therefore subtracted from output below to avoid double counting.const outputReasoningTokens =@@ -3398,6 +3413,8 @@ export class OtelIngestionProcessor {"details.cache_creation_input_tokens","prompt_details.cache_write","input_cache_creation",+ "cache_creation.5m_input_tokens",+ "cache_creation.1h_input_tokens","reasoning.output_tokens","completion_details.reasoning","completion_details.audio",@@ -3418,7 +3435,10 @@ export class OtelIngestionProcessor {if (inputTokens !== undefined) {normalizedUsageDetails.input = Math.max(- inputTokens - (cacheReadTokens ?? 0) - (cacheCreationTokens ?? 0),+ inputTokens -+ (cacheReadTokens ?? 0) -+ (cacheCreationRemainderTokens ?? 0) -+ cacheCreationBreakdownTokens,0,);}@@ -3438,8 +3458,17 @@ export class OtelIngestionProcessor {normalizedUsageDetails.input_cached_tokens = cacheReadTokens;}- if (cacheCreationTokens !== undefined) {- normalizedUsageDetails.input_cache_creation = cacheCreationTokens;+ if (cacheCreationRemainderTokens !== undefined) {+ normalizedUsageDetails.input_cache_creation =+ cacheCreationRemainderTokens;+ }++ if (cacheCreation5mTokens !== undefined) {+ normalizedUsageDetails.input_cache_creation_5m = cacheCreation5mTokens;+ }++ if (cacheCreation1hTokens !== undefined) {+ normalizedUsageDetails.input_cache_creation_1h = cacheCreation1hTokens;}if (outputReasoningTokens !== undefined) {diff --git a/web/src/__tests__/server/api/otel/otelMapping.servertest.ts b/web/src/__tests__/server/api/otel/otelMapping.servertest.tsindex 06bbf1562f..030d07d0f1 100644--- a/web/src/__tests__/server/api/otel/otelMapping.servertest.ts+++ b/web/src/__tests__/server/api/otel/otelMapping.servertest.ts@@ -9122,6 +9122,99 @@ describe("OTel Resource Span Mapping", () => {expect(usageDetails["cache_creation_input_tokens"]).toBeUndefined();});+ it.each([+ {+ name: "5m-only breakdown covering the whole cache write",+ attributes: {+ "gen_ai.usage.input_tokens": 8756,+ "gen_ai.usage.output_tokens": 665,+ "gen_ai.usage.cache_read.input_tokens": 4842,+ "gen_ai.usage.cache_creation.input_tokens": 3910,+ "gen_ai.usage.cache_creation.5m_input_tokens": 3910,+ },+ expected: {+ input: 4,+ input_cached_tokens: 4842,+ input_cache_creation: 0,+ input_cache_creation_5m: 3910,+ output: 665,+ },+ },+ {+ name: "5m and 1h breakdown with an unattributed remainder",+ attributes: {+ "gen_ai.usage.input_tokens": 6000,+ "gen_ai.usage.cache_creation.input_tokens": 5000,+ "gen_ai.usage.cache_creation.5m_input_tokens": 3000,+ "gen_ai.usage.cache_creation.1h_input_tokens": 1500,+ },+ expected: {+ input: 1000,+ input_cache_creation: 500,+ input_cache_creation_5m: 3000,+ input_cache_creation_1h: 1500,+ },+ },+ ])(+ "should split the cache-write TTL breakdown out of input_cache_creation ($name)",+ async ({ attributes, expected }) => {+ const traceId = "abcdef1234567890abcdef1234567898";++ const span = {+ resource: {+ attributes: [+ {+ key: "service.name",+ value: { stringValue: "test-service" },+ },+ ],+ },+ scopeSpans: [+ {+ scope: {+ name: "@mastra/langfuse",+ version: "1.0.0",+ },+ spans: [+ {+ traceId: Buffer.from(traceId, "hex"),+ spanId: Buffer.from("1234567890abcde6", "hex"),+ name: "cache-write-ttl-breakdown",+ kind: 1,+ startTimeUnixNano: {+ low: 1000000,+ high: 406528574,+ unsigned: true,+ },+ endTimeUnixNano: {+ low: 2000000,+ high: 406528574,+ unsigned: true,+ },+ attributes: Object.entries(attributes).map(+ ([key, value]) => ({+ key,+ value: {+ intValue: { low: value, high: 0, unsigned: false },+ },+ }),+ ),+ status: {},+ },+ ],+ },+ ],+ };++ const events = await convertOtelSpanToIngestionEvent(span, new Set());+ const observationEvent = events.find(+ (e) => e.type === "generation-create" || e.type === "span-create",+ );++ expect(observationEvent?.body.usageDetails).toEqual(expected);+ },+ );+it("should let a dotted cache value of 0 shadow the flat spelling (?? alias chain)", async () => {// pins pre-existing ??-chain behavior: a dotted 0 shadows the flat spellingconst traceId = "abcdef1234567890abcdef1234567897";