Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(ingestion): price Anthropic 1-hour cache writes at the 1-hour rate (#17958)

Hassieb Pakzad311ec57cee52
diff --git a/packages/shared/src/server/ingestion/types.ts b/packages/shared/src/server/ingestion/types.ts
index e68db1d95c..c999aca48a 100644
--- a/packages/shared/src/server/ingestion/types.ts
+++ b/packages/shared/src/server/ingestion/types.ts
@@ -102,9 +102,49 @@ const RawUsageDetails = z.record(z.string(), z.unknown()).transform((val) => {
}
}
+ splitAnthropicCacheCreation(val.cache_creation, result);
+
return Object.keys(result).length > 0 ? result : undefined;
});
+/**
+ * Native Anthropic usage nests cache writes by TTL under `cache_creation`.
+ * The 5-minute and 1-hour writes are priced differently, so they are stored
+ * as `input_cache_creation_5m` / `input_cache_creation_1h`; the aggregate
+ * `cache_creation_input_tokens` keeps only the unattributed remainder so no
+ * write is priced twice.
+ */
+function splitAnthropicCacheCreation(
+ cacheCreation: unknown,
+ result: Record<string, number>,
+) {
+ if (typeof cacheCreation !== "object" || cacheCreation === null) return;
+
+ const split = cacheCreation as Record<string, unknown>;
+ const isCount = (value: unknown): value is number =>
+ typeof value === "number" && Number.isInteger(value) && value >= 0;
+ const fiveMinute = split["ephemeral_5m_input_tokens"];
+ const oneHour = split["ephemeral_1h_input_tokens"];
+ if (!isCount(fiveMinute) && !isCount(oneHour)) return;
+
+ let attributed = 0;
+ if (isCount(fiveMinute)) {
+ result["input_cache_creation_5m"] = fiveMinute;
+ attributed += fiveMinute;
+ }
+ if (isCount(oneHour)) {
+ result["input_cache_creation_1h"] = oneHour;
+ attributed += oneHour;
+ }
+
+ const remainder = (result["cache_creation_input_tokens"] ?? 0) - attributed;
+ if (remainder > 0) {
+ result["cache_creation_input_tokens"] = remainder;
+ } else {
+ delete result["cache_creation_input_tokens"];
+ }
+}
+
const OpenAICompletionUsageSchema = z
.object({
prompt_tokens: z.number().int().nonnegative(),
diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.ts
index dc10131fbe..a44ba13158 100644
--- a/worker/src/services/IngestionService/index.ts
+++ b/worker/src/services/IngestionService/index.ts
@@ -159,6 +159,17 @@ function withSerializedEventByteLength(
};
}
+// Models priced before TTL-split cache writes existed (e.g. custom models)
+// only carry an aggregate cache-write price; TTL-split usage falls back to it.
+const TTL_CACHE_CREATION_USAGE_TYPES = new Set([
+ "input_cache_creation_5m",
+ "input_cache_creation_1h",
+]);
+const AGGREGATE_CACHE_CREATION_USAGE_TYPES = [
+ "cache_creation_input_tokens",
+ "input_cache_creation",
+];
+
const immutableEntityKeys: {
[TableName.Traces]: (keyof TraceRecordInsertType)[];
[TableName.Scores]: (keyof ScoreRecordInsertType)[];
@@ -1706,7 +1717,13 @@ export class IngestionService {
const finalCostEntries: [string, number][] = [];
for (const [key, units] of Object.entries(usageUnits)) {
- const price = modelPrices?.find((price) => price.usageType === key);
+ const price =
+ modelPrices?.find((price) => price.usageType === key) ??
+ (TTL_CACHE_CREATION_USAGE_TYPES.has(key)
+ ? AGGREGATE_CACHE_CREATION_USAGE_TYPES.map((usageType) =>
+ modelPrices?.find((price) => price.usageType === usageType),
+ ).find(Boolean)
+ : undefined);
if (units != null && price) {
finalCostEntries.push([key, price.price.mul(units).toNumber()]);
diff --git a/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts b/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts
index 03e7c9ea2f..eaf47c8a87 100644
--- a/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts
+++ b/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts
@@ -607,6 +607,30 @@ describe("Ingestion end-to-end tests", () => {
output_reasoning_tokens: 4,
},
},
+ // Native Anthropic Messages usage with TTL-split cache writes
+ {
+ usage: null,
+ usageDetails: {
+ input_tokens: 3,
+ output_tokens: 94,
+ cache_creation_input_tokens: 12256,
+ cache_read_input_tokens: 16399,
+ cache_creation: {
+ ephemeral_5m_input_tokens: 2000,
+ ephemeral_1h_input_tokens: 10256,
+ },
+ server_tool_use: { web_search_requests: 0 },
+ service_tier: "standard",
+ },
+ expectedUsageDetails: {
+ input_tokens: 3,
+ output_tokens: 94,
+ cache_read_input_tokens: 16399,
+ input_cache_creation_5m: 2000,
+ input_cache_creation_1h: 10256,
+ },
+ absentUsageDetailKeys: ["cache_creation_input_tokens"],
+ },
].forEach((testConfig) => {
it(`should create trace, generation and score without matching models ${JSON.stringify(
testConfig,
@@ -769,6 +793,9 @@ describe("Ingestion end-to-end tests", () => {
expect(generation.usage_details).toMatchObject(
testConfig.expectedUsageDetails,
);
+ for (const key of testConfig.absentUsageDetailKeys ?? []) {
+ expect(generation.usage_details).not.toHaveProperty(key);
+ }
expect(generation.output).toEqual(
JSON.stringify({
key: "this is a great gpt output",
diff --git a/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts b/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts
index 3fe13e29b3..2b2a7ee46e 100644
--- a/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts
+++ b/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts
@@ -206,6 +206,24 @@ describe("Token Cost Calculation", () => {
expect(costs.total_cost).toBe(0.00009375);
});
+ it("prices TTL-split cache writes with the aggregate cache-write price when no TTL price exists", () => {
+ const costs = (IngestionService as any).calculateUsageCosts(
+ [
+ {
+ price: new Decimal(0.00000375),
+ usageType: "cache_creation_input_tokens",
+ },
+ { price: new Decimal(0.000006), usageType: "input_cache_creation_1h" },
+ ],
+ { provided_cost_details: {} },
+ { input_cache_creation_5m: 2000, input_cache_creation_1h: 10256 },
+ );
+
+ expect(costs.cost_details.input_cache_creation_5m).toBe(0.0075);
+ expect(costs.cost_details.input_cache_creation_1h).toBeCloseTo(0.061536, 9);
+ expect(costs.total_cost).toBeCloseTo(0.069036, 9);
+ });
+
it("should correctly calculate token costs with user provided costs", async () => {
const prices = await prisma.price.findMany({
where: {