🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(ingestion): keep OpenAI usage normalization when providers add extra usage fields (#18302)
Hassieb Pakzad4460bc2e0d82
diff --git a/packages/shared/src/server/ingestion/types.ts b/packages/shared/src/server/ingestion/types.tsindex c999aca48a..f14486b739 100644--- a/packages/shared/src/server/ingestion/types.ts+++ b/packages/shared/src/server/ingestion/types.ts@@ -89,6 +89,11 @@ const CostDetails = zconst RawUsageDetails = z.record(z.string(), z.unknown()).transform((val) => {if (!val) return;+ const result = extractUsageCounts(val);+ return Object.keys(result).length > 0 ? result : undefined;+});++function extractUsageCounts(val: Record<string, unknown>) {const result: Record<string, number> = {};for (const [key, value] of Object.entries(val)) {@@ -104,8 +109,8 @@ const RawUsageDetails = z.record(z.string(), z.unknown()).transform((val) => {splitAnthropicCacheCreation(val.cache_creation, result);- return Object.keys(result).length > 0 ? result : undefined;-});+ return result;+}/*** Native Anthropic usage nests cache writes by TTL under `cache_creation`.@@ -145,6 +150,10 @@ function splitAnthropicCacheCreation(}}+// The OpenAI schemas accept unknown top-level keys: OpenAI-compatible providers+// add fields such as `cost`, and rejecting those would route the payload to+// `RawUsageDetails` and lose the normalization. Unknown keys are kept under the+// same rules as `RawUsageDetails`; normalized keys take precedence.const OpenAICompletionUsageSchema = z.object({prompt_tokens: z.number().int().nonnegative(),@@ -157,7 +166,7 @@ const OpenAICompletionUsageSchema = z.record(z.string(), z.number().int().nonnegative().nullish()).nullish(),})- .strict()+ .loose().transform((v) => {if (!v) return;@@ -167,12 +176,14 @@ const OpenAICompletionUsageSchema = ztotal_tokens,prompt_tokens_details,completion_tokens_details,+ ...unknownFields} = v;const result: z.infer<typeof RawUsageDetails> & {input: number;output: number;total: number;} = {+ ...extractUsageCounts(unknownFields),input: prompt_tokens,output: completion_tokens,total: total_tokens,@@ -212,7 +223,7 @@ const OpenAIResponseUsageSchema = z.record(z.string(), z.number().int().nonnegative().nullish()).nullish(),})- .strict()+ .loose().transform((v) => {if (!v) return;@@ -222,12 +233,14 @@ const OpenAIResponseUsageSchema = ztotal_tokens,input_tokens_details,output_tokens_details,+ ...unknownFields} = v;const result: z.infer<typeof RawUsageDetails> & {input: number;output: number;total: number;} = {+ ...extractUsageCounts(unknownFields),input: input_tokens,output: output_tokens,total: total_tokens,diff --git a/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts b/worker/src/services/IngestionService/tests/IngestionService.integration.test.tsindex bdebea3325..7796da84e0 100644--- a/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts+++ b/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts@@ -549,6 +549,31 @@ describe("Ingestion end-to-end tests", () => {output_reasoning_tokens: 4,},},+ // OpenAI-compatible provider extensions (e.g. OpenRouter cost fields)+ {+ usage: null,+ usageDetails: {+ prompt_tokens: 194,+ completion_tokens: 2,+ total_tokens: 196,+ prompt_tokens_details: { cached_tokens: 4 },+ completion_tokens_details: { reasoning_tokens: 1 },+ cost: 0.0001,+ cost_details: { upstream_inference_cost: 0.0001 },+ },+ expectedUsageDetails: {+ input: 190,+ output: 1,+ total: 196,+ input_cached_tokens: 4,+ output_reasoning_tokens: 1,+ },+ absentUsageDetailKeys: [+ "prompt_tokens",+ "completion_tokens",+ "total_tokens",+ ],+ },// OpenAI Response API format{usage: null,@@ -604,6 +629,42 @@ describe("Ingestion end-to-end tests", () => {output_reasoning_tokens: 4,},},+ {+ usage: null,+ usageDetails: {+ input_tokens: 194,+ output_tokens: 2,+ total_tokens: 196,+ input_tokens_details: { cached_tokens: 4 },+ output_tokens_details: { reasoning_tokens: 1 },+ cost: 0.0001,+ },+ expectedUsageDetails: {+ input: 190,+ output: 1,+ total: 196,+ input_cached_tokens: 4,+ output_reasoning_tokens: 1,+ },+ absentUsageDetailKeys: ["input_tokens", "output_tokens", "total_tokens"],+ },+ {+ usage: null,+ usageDetails: {+ input_tokens: 3,+ output_tokens: 94,+ total_tokens: 16496,+ cache_read_input_tokens: 16399,+ service_tier: "standard",+ },+ expectedUsageDetails: {+ input: 3,+ output: 94,+ total: 16496,+ cache_read_input_tokens: 16399,+ },+ absentUsageDetailKeys: ["input_tokens", "output_tokens"],+ },// Native Anthropic Messages usage with TTL-split cache writes{usage: null,