Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(ingestion): keep OpenAI usage normalization when providers add extra usage fields (#18302)

Hassieb Pakzad4460bc2e0d82
diff --git a/packages/shared/src/server/ingestion/types.ts b/packages/shared/src/server/ingestion/types.ts
index c999aca48a..f14486b739 100644
--- a/packages/shared/src/server/ingestion/types.ts
+++ b/packages/shared/src/server/ingestion/types.ts
@@ -89,6 +89,11 @@ const CostDetails = z
const RawUsageDetails = z.record(z.string(), z.unknown()).transform((val) => {
if (!val) return;
+ const result = extractUsageCounts(val);
+ return Object.keys(result).length > 0 ? result : undefined;
+});
+
+function extractUsageCounts(val: Record<string, unknown>) {
const result: Record<string, number> = {};
for (const [key, value] of Object.entries(val)) {
@@ -104,8 +109,8 @@ const RawUsageDetails = z.record(z.string(), z.unknown()).transform((val) => {
splitAnthropicCacheCreation(val.cache_creation, result);
- return Object.keys(result).length > 0 ? result : undefined;
-});
+ return result;
+}
/**
* Native Anthropic usage nests cache writes by TTL under `cache_creation`.
@@ -145,6 +150,10 @@ function splitAnthropicCacheCreation(
}
}
+// The OpenAI schemas accept unknown top-level keys: OpenAI-compatible providers
+// add fields such as `cost`, and rejecting those would route the payload to
+// `RawUsageDetails` and lose the normalization. Unknown keys are kept under the
+// same rules as `RawUsageDetails`; normalized keys take precedence.
const OpenAICompletionUsageSchema = z
.object({
prompt_tokens: z.number().int().nonnegative(),
@@ -157,7 +166,7 @@ const OpenAICompletionUsageSchema = z
.record(z.string(), z.number().int().nonnegative().nullish())
.nullish(),
})
- .strict()
+ .loose()
.transform((v) => {
if (!v) return;
@@ -167,12 +176,14 @@ const OpenAICompletionUsageSchema = z
total_tokens,
prompt_tokens_details,
completion_tokens_details,
+ ...unknownFields
} = v;
const result: z.infer<typeof RawUsageDetails> & {
input: number;
output: number;
total: number;
} = {
+ ...extractUsageCounts(unknownFields),
input: prompt_tokens,
output: completion_tokens,
total: total_tokens,
@@ -212,7 +223,7 @@ const OpenAIResponseUsageSchema = z
.record(z.string(), z.number().int().nonnegative().nullish())
.nullish(),
})
- .strict()
+ .loose()
.transform((v) => {
if (!v) return;
@@ -222,12 +233,14 @@ const OpenAIResponseUsageSchema = z
total_tokens,
input_tokens_details,
output_tokens_details,
+ ...unknownFields
} = v;
const result: z.infer<typeof RawUsageDetails> & {
input: number;
output: number;
total: number;
} = {
+ ...extractUsageCounts(unknownFields),
input: input_tokens,
output: output_tokens,
total: total_tokens,
diff --git a/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts b/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts
index bdebea3325..7796da84e0 100644
--- a/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts
+++ b/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts
@@ -549,6 +549,31 @@ describe("Ingestion end-to-end tests", () => {
output_reasoning_tokens: 4,
},
},
+ // OpenAI-compatible provider extensions (e.g. OpenRouter cost fields)
+ {
+ usage: null,
+ usageDetails: {
+ prompt_tokens: 194,
+ completion_tokens: 2,
+ total_tokens: 196,
+ prompt_tokens_details: { cached_tokens: 4 },
+ completion_tokens_details: { reasoning_tokens: 1 },
+ cost: 0.0001,
+ cost_details: { upstream_inference_cost: 0.0001 },
+ },
+ expectedUsageDetails: {
+ input: 190,
+ output: 1,
+ total: 196,
+ input_cached_tokens: 4,
+ output_reasoning_tokens: 1,
+ },
+ absentUsageDetailKeys: [
+ "prompt_tokens",
+ "completion_tokens",
+ "total_tokens",
+ ],
+ },
// OpenAI Response API format
{
usage: null,
@@ -604,6 +629,42 @@ describe("Ingestion end-to-end tests", () => {
output_reasoning_tokens: 4,
},
},
+ {
+ usage: null,
+ usageDetails: {
+ input_tokens: 194,
+ output_tokens: 2,
+ total_tokens: 196,
+ input_tokens_details: { cached_tokens: 4 },
+ output_tokens_details: { reasoning_tokens: 1 },
+ cost: 0.0001,
+ },
+ expectedUsageDetails: {
+ input: 190,
+ output: 1,
+ total: 196,
+ input_cached_tokens: 4,
+ output_reasoning_tokens: 1,
+ },
+ absentUsageDetailKeys: ["input_tokens", "output_tokens", "total_tokens"],
+ },
+ {
+ usage: null,
+ usageDetails: {
+ input_tokens: 3,
+ output_tokens: 94,
+ total_tokens: 16496,
+ cache_read_input_tokens: 16399,
+ service_tier: "standard",
+ },
+ expectedUsageDetails: {
+ input: 3,
+ output: 94,
+ total: 16496,
+ cache_read_input_tokens: 16399,
+ },
+ absentUsageDetailKeys: ["input_tokens", "output_tokens"],
+ },
// Native Anthropic Messages usage with TTL-split cache writes
{
usage: null,