Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(translation): preserve Responses cache-write usage (#770)
nachiketb-nvidia0c3c4a556d57
diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rsindex ef3f078ed..844a31e43 100644--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs@@ -1933,15 +1933,30 @@ fn decode_responses_usage(value: Option<&Value>) -> Usage {.or_else(|| value.get("prompt_tokens_details")).and_then(|details| details.get("cached_tokens")).and_then(Value::as_u64);- let input_tokens = aggregate_input_tokens- .map(|tokens| tokens.saturating_sub(cached_input_tokens.unwrap_or(0)));+ let cache_creation_input_tokens = value+ .get("input_tokens_details")+ .and_then(|details| details.get("cache_write_tokens"))+ .and_then(Value::as_u64)+ .or_else(|| {+ value.get("prompt_tokens_details").and_then(|details| {+ details+ .get("cache_write_tokens")+ .and_then(Value::as_u64)+ .or_else(|| details.get("cache_creation_tokens").and_then(Value::as_u64))+ })+ });+ let input_tokens = aggregate_input_tokens.map(|tokens| {+ tokens+ .saturating_sub(cached_input_tokens.unwrap_or(0))+ .saturating_sub(cache_creation_input_tokens.unwrap_or(0))+ });let output_tokens = value.get("output_tokens").or_else(|| value.get("completion_tokens")).and_then(Value::as_u64);Usage {input_tokens,- cache: Usage::cache_details(cached_input_tokens, None),+ cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens),output_tokens,total_tokens: value.get("total_tokens")@@ -1979,7 +1994,10 @@ fn encode_responses_usage(usage: &Usage) -> Value {.total_tokens.or_else(|| Some(input_tokens + usage.output_tokens.unwrap_or(0))).unwrap_or(0),- "input_tokens_details": {"cached_tokens": usage.cached_input_tokens().unwrap_or(0)},+ "input_tokens_details": {+ "cached_tokens": usage.cached_input_tokens().unwrap_or(0),+ "cache_write_tokens": usage.cache_creation_input_tokens().unwrap_or(0),+ },"output_tokens_details": {"reasoning_tokens": usage.reasoning_tokens.unwrap_or(0)},})}diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rsindex 608d0f6a8..399b0c3e2 100644--- a/crates/switchyard-translation/src/codecs/responses/stream.rs+++ b/crates/switchyard-translation/src/codecs/responses/stream.rs@@ -991,12 +991,19 @@ fn responses_usage(usage: &serde_json::Map<String, Value>) -> Usage {.get("input_tokens_details").and_then(|details| details.get("cached_tokens")).and_then(Value::as_u64);- let input_tokens = aggregate_input_tokens- .map(|tokens| tokens.saturating_sub(cached_input_tokens.unwrap_or(0)));+ let cache_creation_input_tokens = usage+ .get("input_tokens_details")+ .and_then(|details| details.get("cache_write_tokens"))+ .and_then(Value::as_u64);+ let input_tokens = aggregate_input_tokens.map(|tokens| {+ tokens+ .saturating_sub(cached_input_tokens.unwrap_or(0))+ .saturating_sub(cache_creation_input_tokens.unwrap_or(0))+ });let output_tokens = usage.get("output_tokens").and_then(Value::as_u64);Usage {input_tokens,- cache: Usage::cache_details(cached_input_tokens, None),+ cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens),output_tokens,total_tokens: usage.get("total_tokens")@@ -1027,7 +1034,10 @@ fn responses_usage_value(usage: &Usage) -> Value {"total_tokens": usage.total_tokens.unwrap_or_else(|| {input_tokens + usage.output_tokens.unwrap_or(0)}),- "input_tokens_details": {"cached_tokens": usage.cached_input_tokens().unwrap_or(0)},+ "input_tokens_details": {+ "cached_tokens": usage.cached_input_tokens().unwrap_or(0),+ "cache_write_tokens": usage.cache_creation_input_tokens().unwrap_or(0),+ },"output_tokens_details": {"reasoning_tokens": usage.reasoning_tokens.unwrap_or(0)},})}diff --git a/crates/switchyard-translation/tests/response_translation.rs b/crates/switchyard-translation/tests/response_translation.rsindex c4ad3e83d..86276fcd7 100644--- a/crates/switchyard-translation/tests/response_translation.rs+++ b/crates/switchyard-translation/tests/response_translation.rs@@ -190,6 +190,7 @@ fn responses_reasoning_usage_translates_to_openai_chat_usage_details() -> TestRe"input_tokens": 10,"output_tokens": 5,"total_tokens": 15,+ "input_tokens_details": {"cached_tokens": 4, "cache_write_tokens": 2},"output_tokens_details": {"reasoning_tokens": 3}}});@@ -209,6 +210,64 @@ fn responses_reasoning_usage_translates_to_openai_chat_usage_details() -> TestReoutput["usage"]["completion_tokens_details"],json!({"reasoning_tokens": 3}));+ assert_eq!(+ output["usage"]["prompt_tokens_details"]["cache_creation_tokens"],+ 2+ );+ let decoded = engine+ .decode_response(WireFormat::OpenAiResponses, &body, &normalized_policy())?+ .response;+ assert_eq!(decoded.usage.input_tokens, Some(4));+ assert_eq!(decoded.usage.cached_input_tokens(), Some(4));+ assert_eq!(decoded.usage.cache_creation_input_tokens(), Some(2));+ assert_eq!(decoded.usage.total_tokens, Some(15));+ let mut aliased = body.clone();+ aliased["usage"] = output["usage"].clone();+ assert_eq!(+ engine+ .decode_response(WireFormat::OpenAiResponses, &aliased, &normalized_policy())?+ .response+ .usage,+ decoded.usage+ );+ let encoded = engine+ .encode_response(WireFormat::OpenAiResponses, &decoded, &normalized_policy())?+ .body;+ assert_eq!(encoded["usage"], body["usage"]);++ let mut state = switchyard_translation::StreamTranslationState::new(+ WireFormat::OpenAiResponses,+ WireFormat::OpenAiChat,+ );+ let events = engine.translate_event(+ &mut state,+ WireFormat::OpenAiResponses,+ WireFormat::OpenAiChat,+ &json!({"type": "response.completed", "response": body}),+ )?;+ assert_eq!(state.usage, decoded.usage);+ let chat = events+ .iter()+ .find(|event| event.get("usage").is_some())+ .ok_or("missing usage")?;+ assert_eq!(chat["usage"], output["usage"]);+ let mut state = switchyard_translation::StreamTranslationState::new(+ WireFormat::OpenAiChat,+ WireFormat::OpenAiResponses,+ );+ let mut events = engine.translate_event(+ &mut state,+ WireFormat::OpenAiChat,+ WireFormat::OpenAiResponses,+ &json!({"id": "chat_usage", "model": "model", "usage": output["usage"],+ "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}),+ )?;+ events.extend(engine.finish_stream(&mut state, WireFormat::OpenAiResponses)?);+ let completed = events+ .iter()+ .find(|event| event["type"] == "response.completed")+ .ok_or("missing completion")?;+ assert_eq!(completed["response"]["usage"], body["usage"]);Ok(())}@@ -298,7 +357,7 @@ fn openai_chat_cache_usage_translates_to_responses_usage_details() -> TestResultassert_eq!(output["usage"]["input_tokens"], 100);assert_eq!(output["usage"]["input_tokens_details"],- json!({"cached_tokens": 80})+ json!({"cached_tokens": 80, "cache_write_tokens": 0}));Ok(())}@@ -622,7 +681,7 @@ fn openai_chat_response_with_tool_call_translates_to_responses_output_item() ->"input_tokens": 4,"output_tokens": 3,"total_tokens": 7,- "input_tokens_details": {"cached_tokens": 0},+ "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},"output_tokens_details": {"reasoning_tokens": 0}}));@@ -700,7 +759,7 @@ fn openai_chat_usage_without_breakdowns_still_emits_responses_usage_details() ->"input_tokens": 41,"output_tokens": 3,"total_tokens": 44,- "input_tokens_details": {"cached_tokens": 0},+ "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},"output_tokens_details": {"reasoning_tokens": 0}}));@@ -739,7 +798,7 @@ fn openai_chat_cache_only_usage_still_emits_reasoning_details() -> TestResult {assert_eq!(output["usage"]["input_tokens_details"],- json!({"cached_tokens": 32})+ json!({"cached_tokens": 32, "cache_write_tokens": 0}));assert_eq!(output["usage"]["output_tokens_details"],diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rsindex c9de9b8d7..1613445fe 100644--- a/crates/switchyard-translation/tests/stream_translation.rs+++ b/crates/switchyard-translation/tests/stream_translation.rs@@ -882,7 +882,7 @@ fn openai_chat_stream_cache_usage_translates_to_responses_usage_details() -> Tesassert_eq!(completed["response"]["usage"]["input_tokens"], 100);assert_eq!(completed["response"]["usage"]["input_tokens_details"],- json!({"cached_tokens": 80})+ json!({"cached_tokens": 80, "cache_write_tokens": 0}));Ok(())}@@ -1581,7 +1581,7 @@ fn openai_chat_stream_usage_without_breakdowns_still_emits_responses_usage_detai};assert_eq!(completed["response"]["usage"]["input_tokens_details"],- json!({"cached_tokens": 0})+ json!({"cached_tokens": 0, "cache_write_tokens": 0}));assert_eq!(completed["response"]["usage"]["output_tokens_details"],