🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(ingestion): price Anthropic 1-hour cache writes at the 1-hour rate (#17958)
Hassieb Pakzad311ec57cee52
diff --git a/packages/shared/src/server/ingestion/types.ts b/packages/shared/src/server/ingestion/types.tsindex e68db1d95c..c999aca48a 100644--- a/packages/shared/src/server/ingestion/types.ts+++ b/packages/shared/src/server/ingestion/types.ts@@ -102,9 +102,49 @@ const RawUsageDetails = z.record(z.string(), z.unknown()).transform((val) => {}}+ splitAnthropicCacheCreation(val.cache_creation, result);+return Object.keys(result).length > 0 ? result : undefined;});+/**+ * Native Anthropic usage nests cache writes by TTL under `cache_creation`.+ * The 5-minute and 1-hour writes are priced differently, so they are stored+ * as `input_cache_creation_5m` / `input_cache_creation_1h`; the aggregate+ * `cache_creation_input_tokens` keeps only the unattributed remainder so no+ * write is priced twice.+ */+function splitAnthropicCacheCreation(+ cacheCreation: unknown,+ result: Record<string, number>,+) {+ if (typeof cacheCreation !== "object" || cacheCreation === null) return;++ const split = cacheCreation as Record<string, unknown>;+ const isCount = (value: unknown): value is number =>+ typeof value === "number" && Number.isInteger(value) && value >= 0;+ const fiveMinute = split["ephemeral_5m_input_tokens"];+ const oneHour = split["ephemeral_1h_input_tokens"];+ if (!isCount(fiveMinute) && !isCount(oneHour)) return;++ let attributed = 0;+ if (isCount(fiveMinute)) {+ result["input_cache_creation_5m"] = fiveMinute;+ attributed += fiveMinute;+ }+ if (isCount(oneHour)) {+ result["input_cache_creation_1h"] = oneHour;+ attributed += oneHour;+ }++ const remainder = (result["cache_creation_input_tokens"] ?? 0) - attributed;+ if (remainder > 0) {+ result["cache_creation_input_tokens"] = remainder;+ } else {+ delete result["cache_creation_input_tokens"];+ }+}+const OpenAICompletionUsageSchema = z.object({prompt_tokens: z.number().int().nonnegative(),diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.tsindex dc10131fbe..a44ba13158 100644--- a/worker/src/services/IngestionService/index.ts+++ b/worker/src/services/IngestionService/index.ts@@ -159,6 +159,17 @@ function withSerializedEventByteLength(};}+// Models priced before TTL-split cache writes existed (e.g. custom models)+// only carry an aggregate cache-write price; TTL-split usage falls back to it.+const TTL_CACHE_CREATION_USAGE_TYPES = new Set([+ "input_cache_creation_5m",+ "input_cache_creation_1h",+]);+const AGGREGATE_CACHE_CREATION_USAGE_TYPES = [+ "cache_creation_input_tokens",+ "input_cache_creation",+];+const immutableEntityKeys: {[TableName.Traces]: (keyof TraceRecordInsertType)[];[TableName.Scores]: (keyof ScoreRecordInsertType)[];@@ -1706,7 +1717,13 @@ export class IngestionService {const finalCostEntries: [string, number][] = [];for (const [key, units] of Object.entries(usageUnits)) {- const price = modelPrices?.find((price) => price.usageType === key);+ const price =+ modelPrices?.find((price) => price.usageType === key) ??+ (TTL_CACHE_CREATION_USAGE_TYPES.has(key)+ ? AGGREGATE_CACHE_CREATION_USAGE_TYPES.map((usageType) =>+ modelPrices?.find((price) => price.usageType === usageType),+ ).find(Boolean)+ : undefined);if (units != null && price) {finalCostEntries.push([key, price.price.mul(units).toNumber()]);diff --git a/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts b/worker/src/services/IngestionService/tests/IngestionService.integration.test.tsindex 03e7c9ea2f..eaf47c8a87 100644--- a/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts+++ b/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts@@ -607,6 +607,30 @@ describe("Ingestion end-to-end tests", () => {output_reasoning_tokens: 4,},},+ // Native Anthropic Messages usage with TTL-split cache writes+ {+ usage: null,+ usageDetails: {+ input_tokens: 3,+ output_tokens: 94,+ cache_creation_input_tokens: 12256,+ cache_read_input_tokens: 16399,+ cache_creation: {+ ephemeral_5m_input_tokens: 2000,+ ephemeral_1h_input_tokens: 10256,+ },+ server_tool_use: { web_search_requests: 0 },+ service_tier: "standard",+ },+ expectedUsageDetails: {+ input_tokens: 3,+ output_tokens: 94,+ cache_read_input_tokens: 16399,+ input_cache_creation_5m: 2000,+ input_cache_creation_1h: 10256,+ },+ absentUsageDetailKeys: ["cache_creation_input_tokens"],+ },].forEach((testConfig) => {it(`should create trace, generation and score without matching models ${JSON.stringify(testConfig,@@ -769,6 +793,9 @@ describe("Ingestion end-to-end tests", () => {expect(generation.usage_details).toMatchObject(testConfig.expectedUsageDetails,);+ for (const key of testConfig.absentUsageDetailKeys ?? []) {+ expect(generation.usage_details).not.toHaveProperty(key);+ }expect(generation.output).toEqual(JSON.stringify({key: "this is a great gpt output",diff --git a/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts b/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.tsindex 3fe13e29b3..2b2a7ee46e 100644--- a/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts+++ b/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts@@ -206,6 +206,24 @@ describe("Token Cost Calculation", () => {expect(costs.total_cost).toBe(0.00009375);});+ it("prices TTL-split cache writes with the aggregate cache-write price when no TTL price exists", () => {+ const costs = (IngestionService as any).calculateUsageCosts(+ [+ {+ price: new Decimal(0.00000375),+ usageType: "cache_creation_input_tokens",+ },+ { price: new Decimal(0.000006), usageType: "input_cache_creation_1h" },+ ],+ { provided_cost_details: {} },+ { input_cache_creation_5m: 2000, input_cache_creation_1h: 10256 },+ );++ expect(costs.cost_details.input_cache_creation_5m).toBe(0.0075);+ expect(costs.cost_details.input_cache_creation_1h).toBeCloseTo(0.061536, 9);+ expect(costs.total_cost).toBeCloseTo(0.069036, 9);+ });+it("should correctly calculate token costs with user provided costs", async () => {const prices = await prisma.price.findMany({where: {