Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(otel): stop double counting the cache-write TTL breakdown in generic gen_ai usage mapping (#19420)

Hassieb Pakzade0fad7b26221
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
index e697dd4cbc..7ed785b3ea 100644
--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts
+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
@@ -3364,6 +3364,21 @@ export class OtelIngestionProcessor {
rawUsageDetails["details.cache_creation_input_tokens"] ??
rawUsageDetails["prompt_details.cache_write"] ??
rawUsageDetails["input_cache_creation"];
+ // TTL breakdown of the cache-write total; only the unattributed remainder
+ // stays in input_cache_creation so each write is counted and priced once.
+ const cacheCreation5mTokens =
+ rawUsageDetails["cache_creation.5m_input_tokens"];
+ const cacheCreation1hTokens =
+ rawUsageDetails["cache_creation.1h_input_tokens"];
+ const hasCacheCreationBreakdown =
+ cacheCreation5mTokens !== undefined ||
+ cacheCreation1hTokens !== undefined;
+ const cacheCreationBreakdownTokens =
+ (cacheCreation5mTokens ?? 0) + (cacheCreation1hTokens ?? 0);
+ const cacheCreationRemainderTokens =
+ cacheCreationTokens !== undefined && hasCacheCreationBreakdown
+ ? Math.max(cacheCreationTokens - cacheCreationBreakdownTokens, 0)
+ : cacheCreationTokens;
// Reasoning/audio details are included in the emitted output token count
// and are therefore subtracted from output below to avoid double counting.
const outputReasoningTokens =
@@ -3398,6 +3413,8 @@ export class OtelIngestionProcessor {
"details.cache_creation_input_tokens",
"prompt_details.cache_write",
"input_cache_creation",
+ "cache_creation.5m_input_tokens",
+ "cache_creation.1h_input_tokens",
"reasoning.output_tokens",
"completion_details.reasoning",
"completion_details.audio",
@@ -3418,7 +3435,10 @@ export class OtelIngestionProcessor {
if (inputTokens !== undefined) {
normalizedUsageDetails.input = Math.max(
- inputTokens - (cacheReadTokens ?? 0) - (cacheCreationTokens ?? 0),
+ inputTokens -
+ (cacheReadTokens ?? 0) -
+ (cacheCreationRemainderTokens ?? 0) -
+ cacheCreationBreakdownTokens,
0,
);
}
@@ -3438,8 +3458,17 @@ export class OtelIngestionProcessor {
normalizedUsageDetails.input_cached_tokens = cacheReadTokens;
}
- if (cacheCreationTokens !== undefined) {
- normalizedUsageDetails.input_cache_creation = cacheCreationTokens;
+ if (cacheCreationRemainderTokens !== undefined) {
+ normalizedUsageDetails.input_cache_creation =
+ cacheCreationRemainderTokens;
+ }
+
+ if (cacheCreation5mTokens !== undefined) {
+ normalizedUsageDetails.input_cache_creation_5m = cacheCreation5mTokens;
+ }
+
+ if (cacheCreation1hTokens !== undefined) {
+ normalizedUsageDetails.input_cache_creation_1h = cacheCreation1hTokens;
}
if (outputReasoningTokens !== undefined) {
diff --git a/web/src/__tests__/server/api/otel/otelMapping.servertest.ts b/web/src/__tests__/server/api/otel/otelMapping.servertest.ts
index 06bbf1562f..030d07d0f1 100644
--- a/web/src/__tests__/server/api/otel/otelMapping.servertest.ts
+++ b/web/src/__tests__/server/api/otel/otelMapping.servertest.ts
@@ -9122,6 +9122,99 @@ describe("OTel Resource Span Mapping", () => {
expect(usageDetails["cache_creation_input_tokens"]).toBeUndefined();
});
+ it.each([
+ {
+ name: "5m-only breakdown covering the whole cache write",
+ attributes: {
+ "gen_ai.usage.input_tokens": 8756,
+ "gen_ai.usage.output_tokens": 665,
+ "gen_ai.usage.cache_read.input_tokens": 4842,
+ "gen_ai.usage.cache_creation.input_tokens": 3910,
+ "gen_ai.usage.cache_creation.5m_input_tokens": 3910,
+ },
+ expected: {
+ input: 4,
+ input_cached_tokens: 4842,
+ input_cache_creation: 0,
+ input_cache_creation_5m: 3910,
+ output: 665,
+ },
+ },
+ {
+ name: "5m and 1h breakdown with an unattributed remainder",
+ attributes: {
+ "gen_ai.usage.input_tokens": 6000,
+ "gen_ai.usage.cache_creation.input_tokens": 5000,
+ "gen_ai.usage.cache_creation.5m_input_tokens": 3000,
+ "gen_ai.usage.cache_creation.1h_input_tokens": 1500,
+ },
+ expected: {
+ input: 1000,
+ input_cache_creation: 500,
+ input_cache_creation_5m: 3000,
+ input_cache_creation_1h: 1500,
+ },
+ },
+ ])(
+ "should split the cache-write TTL breakdown out of input_cache_creation ($name)",
+ async ({ attributes, expected }) => {
+ const traceId = "abcdef1234567890abcdef1234567898";
+
+ const span = {
+ resource: {
+ attributes: [
+ {
+ key: "service.name",
+ value: { stringValue: "test-service" },
+ },
+ ],
+ },
+ scopeSpans: [
+ {
+ scope: {
+ name: "@mastra/langfuse",
+ version: "1.0.0",
+ },
+ spans: [
+ {
+ traceId: Buffer.from(traceId, "hex"),
+ spanId: Buffer.from("1234567890abcde6", "hex"),
+ name: "cache-write-ttl-breakdown",
+ kind: 1,
+ startTimeUnixNano: {
+ low: 1000000,
+ high: 406528574,
+ unsigned: true,
+ },
+ endTimeUnixNano: {
+ low: 2000000,
+ high: 406528574,
+ unsigned: true,
+ },
+ attributes: Object.entries(attributes).map(
+ ([key, value]) => ({
+ key,
+ value: {
+ intValue: { low: value, high: 0, unsigned: false },
+ },
+ }),
+ ),
+ status: {},
+ },
+ ],
+ },
+ ],
+ };
+
+ const events = await convertOtelSpanToIngestionEvent(span, new Set());
+ const observationEvent = events.find(
+ (e) => e.type === "generation-create" || e.type === "span-create",
+ );
+
+ expect(observationEvent?.body.usageDetails).toEqual(expected);
+ },
+ );
+
it("should let a dotted cache value of 0 shadow the flat spelling (?? alias chain)", async () => {
// pins pre-existing ??-chain behavior: a dotted 0 shadows the flat spelling
const traceId = "abcdef1234567890abcdef1234567897";