Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(translation): forward prompt-cache token usage across format translation (#117)

Brian Newsom097d0e4ae7c6
diff --git a/crates/protocol/src/llm.rs b/crates/protocol/src/llm.rs
index 635860b56..cda3eb825 100644
--- a/crates/protocol/src/llm.rs
+++ b/crates/protocol/src/llm.rs
@@ -232,12 +232,62 @@ pub struct LlmRequest {
/// Normalized token usage counts.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct Usage {
+ /// Non-cached input tokens. Provider codecs normalize aggregate OpenAI
+ /// input counts by subtracting the cache detail fields.
pub input_tokens: Option<u64>,
+ #[serde(flatten)]
+ pub cache: Option<Box<InputCacheUsage>>,
pub output_tokens: Option<u64>,
pub total_tokens: Option<u64>,
pub reasoning_tokens: Option<u64>,
}
+/// Optional cache-token detail kept out of the common, cache-free usage allocation.
+#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
+pub struct InputCacheUsage {
+ pub cached_input_tokens: Option<u64>,
+ pub cache_creation_input_tokens: Option<u64>,
+}
+
+impl Usage {
+ pub fn cache_details(
+ cached_input_tokens: Option<u64>,
+ cache_creation_input_tokens: Option<u64>,
+ ) -> Option<Box<InputCacheUsage>> {
+ if cached_input_tokens.is_none() && cache_creation_input_tokens.is_none() {
+ return None;
+ }
+ Some(Box::new(InputCacheUsage {
+ cached_input_tokens,
+ cache_creation_input_tokens,
+ }))
+ }
+
+ pub fn cached_input_tokens(&self) -> Option<u64> {
+ self.cache
+ .as_ref()
+ .and_then(|cache| cache.cached_input_tokens)
+ }
+
+ pub fn cache_creation_input_tokens(&self) -> Option<u64> {
+ self.cache
+ .as_ref()
+ .and_then(|cache| cache.cache_creation_input_tokens)
+ }
+
+ pub fn set_cached_input_tokens(&mut self, value: u64) {
+ self.cache
+ .get_or_insert_with(|| Box::new(InputCacheUsage::default()))
+ .cached_input_tokens = Some(value);
+ }
+
+ pub fn set_cache_creation_input_tokens(&mut self, value: u64) {
+ self.cache
+ .get_or_insert_with(|| Box::new(InputCacheUsage::default()))
+ .cache_creation_input_tokens = Some(value);
+ }
+}
+
/// Normalized reason a model stopped producing output.
#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index b4698b204..7cb1a9938 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -109,7 +109,12 @@ async fn upstream_chat(
"message": {"role": "assistant", "content": content},
"finish_reason": "stop"
}],
- "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}
+ "usage": {
+ "prompt_tokens": 10,
+ "completion_tokens": 2,
+ "total_tokens": 12,
+ "prompt_tokens_details": {"cached_tokens": 7}
+ }
}))
.into_response()
}
@@ -364,6 +369,18 @@ async fn all_inbound_formats_run_libsy_and_return_the_caller_format() -> TestRes
responses[2].json()?["output"][0]["content"][0]["text"],
"ok"
);
+ assert_eq!(responses[0].json()?["usage"]["prompt_tokens"], 10);
+ assert_eq!(
+ responses[0].json()?["usage"]["prompt_tokens_details"]["cached_tokens"],
+ 7
+ );
+ assert_eq!(responses[1].json()?["usage"]["input_tokens"], 3);
+ assert_eq!(responses[1].json()?["usage"]["cache_read_input_tokens"], 7);
+ assert_eq!(responses[2].json()?["usage"]["input_tokens"], 10);
+ assert_eq!(
+ responses[2].json()?["usage"]["input_tokens_details"]["cached_tokens"],
+ 7
+ );
for response in &responses {
assert_eq!(
response
diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs
index 0542082f6..e2deee864 100644
--- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs
@@ -852,13 +852,21 @@ fn decode_anthropic_usage(value: Option<&Value>) -> Usage {
return Usage::default();
};
let input_tokens = value.get("input_tokens").and_then(Value::as_u64);
+ let cached_input_tokens = value.get("cache_read_input_tokens").and_then(Value::as_u64);
+ let cache_creation_input_tokens = value
+ .get("cache_creation_input_tokens")
+ .and_then(Value::as_u64);
let output_tokens = value.get("output_tokens").and_then(Value::as_u64);
Usage {
input_tokens,
+ cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens),
output_tokens,
- total_tokens: input_tokens
- .zip(output_tokens)
- .map(|(input, output)| input + output),
+ total_tokens: input_tokens.zip(output_tokens).map(|(input, output)| {
+ input
+ + cached_input_tokens.unwrap_or(0)
+ + cache_creation_input_tokens.unwrap_or(0)
+ + output
+ }),
reasoning_tokens: value
.get("output_tokens_details")
.and_then(|details| details.get("reasoning_tokens"))
@@ -868,10 +876,17 @@ fn decode_anthropic_usage(value: Option<&Value>) -> Usage {
// Encodes normalized usage into Anthropic usage JSON.
fn encode_anthropic_usage(usage: &Usage) -> Value {
- json!({
+ let mut value = json!({
"input_tokens": usage.input_tokens.unwrap_or(0),
"output_tokens": usage.output_tokens.unwrap_or(0),
- })
+ });
+ if let Some(cached_tokens) = usage.cached_input_tokens() {
+ value["cache_read_input_tokens"] = json!(cached_tokens);
+ }
+ if let Some(cache_creation_tokens) = usage.cache_creation_input_tokens() {
+ value["cache_creation_input_tokens"] = json!(cache_creation_tokens);
+ }
+ value
}
// Maps Anthropic stop reasons to normalized stop reasons.
diff --git a/crates/switchyard-translation/src/codecs/anthropic/stream.rs b/crates/switchyard-translation/src/codecs/anthropic/stream.rs
index 077152752..8a6a6ee72 100644
--- a/crates/switchyard-translation/src/codecs/anthropic/stream.rs
+++ b/crates/switchyard-translation/src/codecs/anthropic/stream.rs
@@ -480,18 +480,21 @@ fn capture_anthropic_usage(state: &mut StreamTranslationState, usage: &Value) {
let Some(usage) = usage.as_object() else {
return;
};
- for key in [
- "input_tokens",
- "output_tokens",
- "cache_creation_input_tokens",
- "cache_read_input_tokens",
- ] {
- if let Some(value) = usage.get(key).and_then(Value::as_u64) {
- state.usage_extras.insert(key.to_string(), value);
- }
+ if let Some(value) = usage.get("input_tokens").and_then(Value::as_u64) {
+ state.usage.input_tokens = Some(value);
+ }
+ if let Some(value) = usage.get("output_tokens").and_then(Value::as_u64) {
+ state.usage.output_tokens = Some(value);
+ }
+ if let Some(value) = usage
+ .get("cache_creation_input_tokens")
+ .and_then(Value::as_u64)
+ {
+ state.usage.set_cache_creation_input_tokens(value);
+ }
+ if let Some(value) = usage.get("cache_read_input_tokens").and_then(Value::as_u64) {
+ state.usage.set_cached_input_tokens(value);
}
- state.usage.input_tokens = state.usage_extras.get("input_tokens").copied();
- state.usage.output_tokens = state.usage_extras.get("output_tokens").copied();
}
// Builds Anthropic usage payloads from normalized and provider-extra state.
@@ -508,11 +511,11 @@ fn anthropic_stream_usage(state: &StreamTranslationState) -> Value {
} else {
usage.insert("output_tokens".to_string(), json!(state.output_tokens_seen));
}
- for (key, value) in &state.usage_extras {
- if key == "input_tokens" || key == "output_tokens" {
- continue;
- }
- usage.insert(key.clone(), json!(value));
+ if let Some(value) = state.usage.cache_creation_input_tokens() {
+ usage.insert("cache_creation_input_tokens".to_string(), json!(value));
+ }
+ if let Some(value) = state.usage.cached_input_tokens() {
+ usage.insert("cache_read_input_tokens".to_string(), json!(value));
}
Value::Object(usage)
}
diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
index c6d823102..6c0ad424f 100644
--- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
@@ -1066,8 +1066,24 @@ pub(crate) fn decode_openai_usage(value: Option<&Value>) -> Usage {
let Some(value) = value.and_then(Value::as_object) else {
return Usage::default();
};
+ let cached_input_tokens = value
+ .get("prompt_tokens_details")
+ .and_then(|details| details.get("cached_tokens"))
+ .and_then(Value::as_u64);
+ let cache_creation_input_tokens = value
+ .get("prompt_tokens_details")
+ .and_then(|details| details.get("cache_creation_tokens"))
+ .and_then(Value::as_u64);
Usage {
- input_tokens: value.get("prompt_tokens").and_then(Value::as_u64),
+ input_tokens: value
+ .get("prompt_tokens")
+ .and_then(Value::as_u64)
+ .map(|tokens| {
+ tokens
+ .saturating_sub(cached_input_tokens.unwrap_or(0))
+ .saturating_sub(cache_creation_input_tokens.unwrap_or(0))
+ }),
+ cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens),
output_tokens: value.get("completion_tokens").and_then(Value::as_u64),
total_tokens: value.get("total_tokens").and_then(Value::as_u64),
reasoning_tokens: value
@@ -1084,14 +1100,23 @@ pub(crate) fn decode_openai_usage(value: Option<&Value>) -> Usage {
/// Encodes normalized usage into OpenAI Chat usage JSON.
pub(crate) fn encode_openai_usage(usage: &Usage) -> Value {
+ let prompt_tokens = usage.input_tokens.unwrap_or(0)
+ + usage.cached_input_tokens().unwrap_or(0)
+ + usage.cache_creation_input_tokens().unwrap_or(0);
let mut value = json!({
- "prompt_tokens": usage.input_tokens.unwrap_or(0),
+ "prompt_tokens": prompt_tokens,
"completion_tokens": usage.output_tokens.unwrap_or(0),
"total_tokens": usage
.total_tokens
- .or_else(|| Some(usage.input_tokens.unwrap_or(0) + usage.output_tokens.unwrap_or(0)))
+ .or_else(|| Some(prompt_tokens + usage.output_tokens.unwrap_or(0)))
.unwrap_or(0),
});
+ if usage.cached_input_tokens().is_some() || usage.cache_creation_input_tokens().is_some() {
+ value["prompt_tokens_details"] = json!({
+ "cached_tokens": usage.cached_input_tokens().unwrap_or(0),
+ "cache_creation_tokens": usage.cache_creation_input_tokens().unwrap_or(0),
+ });
+ }
if let Some(reasoning_tokens) = usage.reasoning_tokens {
value["completion_tokens_details"] = json!({
"reasoning_tokens": reasoning_tokens,
diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
index d2e5f6e74..d14bdc321 100644
--- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
+++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
@@ -70,7 +70,6 @@ fn decode_openai_chat_stream(
if let Some(usage) = object.get("usage").and_then(Value::as_object) {
let usage = openai_usage(usage);
- capture_openai_usage_extras(state, object.get("usage"));
state.usage = usage.clone();
state.saw_backend_usage = true;
out.push(LlmResponseChunk::Usage(usage));
@@ -225,8 +224,24 @@ fn finish_openai_chat_stream(state: &mut StreamTranslationState) -> Vec<Value> {
// Normalizes OpenAI token usage fields.
fn openai_usage(usage: &Map<String, Value>) -> Usage {
+ let cached_input_tokens = usage
+ .get("prompt_tokens_details")
+ .and_then(|details| details.get("cached_tokens"))
+ .and_then(Value::as_u64);
+ let cache_creation_input_tokens = usage
+ .get("prompt_tokens_details")
+ .and_then(|details| details.get("cache_creation_tokens"))
+ .and_then(Value::as_u64);
Usage {
- input_tokens: usage.get("prompt_tokens").and_then(Value::as_u64),
+ input_tokens: usage
+ .get("prompt_tokens")
+ .and_then(Value::as_u64)
+ .map(|tokens| {
+ tokens
+ .saturating_sub(cached_input_tokens.unwrap_or(0))
+ .saturating_sub(cache_creation_input_tokens.unwrap_or(0))
+ }),
+ cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens),
output_tokens: usage.get("completion_tokens").and_then(Value::as_u64),
total_tokens: usage.get("total_tokens").and_then(Value::as_u64),
reasoning_tokens: usage
@@ -241,19 +256,6 @@ fn openai_usage(usage: &Map<String, Value>) -> Usage {
}
}
-// Preserves OpenAI cache usage fields that have Anthropic equivalents.
-fn capture_openai_usage_extras(state: &mut StreamTranslationState, usage: Option<&Value>) {
- if let Some(cached_tokens) = usage
- .and_then(|usage| usage.get("prompt_tokens_details"))
- .and_then(|details| details.get("cached_tokens"))
- .and_then(Value::as_u64)
- {
- state
- .usage_extras
- .insert("cache_read_input_tokens".to_string(), cached_tokens);
- }
-}
-
// Builds a single OpenAI Chat stream chunk payload.
fn openai_stream_chunk(
state: &StreamTranslationState,
@@ -310,16 +312,8 @@ fn openai_tool_call_chunk(
// Builds OpenAI usage payloads from normalized and provider-extra state.
fn openai_usage_value(state: &StreamTranslationState) -> Value {
- let cache_creation_tokens = state
- .usage_extras
- .get("cache_creation_input_tokens")
- .copied()
- .unwrap_or(0);
- let cache_read_tokens = state
- .usage_extras
- .get("cache_read_input_tokens")
- .copied()
- .unwrap_or(0);
+ let cache_creation_tokens = state.usage.cache_creation_input_tokens().unwrap_or(0);
+ let cache_read_tokens = state.usage.cached_input_tokens().unwrap_or(0);
let prompt_tokens =
state.usage.input_tokens.unwrap_or(0) + cache_creation_tokens + cache_read_tokens;
let completion_tokens = state.usage.output_tokens.unwrap_or(0);
@@ -333,7 +327,9 @@ fn openai_usage_value(state: &StreamTranslationState) -> Value {
"reasoning_tokens": reasoning_tokens,
});
}
- if cache_creation_tokens > 0 || cache_read_tokens > 0 {
+ if state.usage.cache_creation_input_tokens().is_some()
+ || state.usage.cached_input_tokens().is_some()
+ {
usage["prompt_tokens_details"] = json!({
"cached_tokens": cache_read_tokens,
"cache_creation_tokens": cache_creation_tokens,
diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs
index 18e9125a7..196579f29 100644
--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs
@@ -1150,22 +1150,30 @@ fn decode_responses_usage(value: Option<&Value>) -> Usage {
let Some(value) = value.and_then(Value::as_object) else {
return Usage::default();
};
- let input_tokens = value
+ let aggregate_input_tokens = value
.get("input_tokens")
.or_else(|| value.get("prompt_tokens"))
.and_then(Value::as_u64);
+ let cached_input_tokens = value
+ .get("input_tokens_details")
+ .or_else(|| value.get("prompt_tokens_details"))
+ .and_then(|details| details.get("cached_tokens"))
+ .and_then(Value::as_u64);
+ let input_tokens = aggregate_input_tokens
+ .map(|tokens| tokens.saturating_sub(cached_input_tokens.unwrap_or(0)));
let output_tokens = value
.get("output_tokens")
.or_else(|| value.get("completion_tokens"))
.and_then(Value::as_u64);
Usage {
input_tokens,
+ cache: Usage::cache_details(cached_input_tokens, None),
output_tokens,
total_tokens: value
.get("total_tokens")
.and_then(Value::as_u64)
.or_else(|| {
- input_tokens
+ aggregate_input_tokens
.zip(output_tokens)
.map(|(input, output)| input + output)
}),
@@ -1183,14 +1191,20 @@ fn decode_responses_usage(value: Option<&Value>) -> Usage {
// Encodes normalized usage into Responses usage JSON.
fn encode_responses_usage(usage: &Usage) -> Value {
+ let input_tokens = usage.input_tokens.unwrap_or(0)
+ + usage.cached_input_tokens().unwrap_or(0)
+ + usage.cache_creation_input_tokens().unwrap_or(0);
let mut value = json!({
- "input_tokens": usage.input_tokens.unwrap_or(0),
+ "input_tokens": input_tokens,
"output_tokens": usage.output_tokens.unwrap_or(0),
"total_tokens": usage
.total_tokens
- .or_else(|| Some(usage.input_tokens.unwrap_or(0) + usage.output_tokens.unwrap_or(0)))
+ .or_else(|| Some(input_tokens + usage.output_tokens.unwrap_or(0)))
.unwrap_or(0),
});
+ if let Some(cached_tokens) = usage.cached_input_tokens() {
+ value["input_tokens_details"] = json!({"cached_tokens": cached_tokens});
+ }
if let Some(reasoning_tokens) = usage.reasoning_tokens {
value["output_tokens_details"] = json!({
"reasoning_tokens": reasoning_tokens,
diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rs
index 42234b9d2..e30190874 100644
--- a/crates/switchyard-translation/src/codecs/responses/stream.rs
+++ b/crates/switchyard-translation/src/codecs/responses/stream.rs
@@ -522,15 +522,22 @@ fn encode_responses_tool_delta(
// Normalizes OpenAI Responses token usage fields.
fn responses_usage(usage: &serde_json::Map<String, Value>) -> Usage {
- let input_tokens = usage.get("input_tokens").and_then(Value::as_u64);
+ let aggregate_input_tokens = usage.get("input_tokens").and_then(Value::as_u64);
+ let cached_input_tokens = usage
+ .get("input_tokens_details")
+ .and_then(|details| details.get("cached_tokens"))
+ .and_then(Value::as_u64);
+ let input_tokens = aggregate_input_tokens
+ .map(|tokens| tokens.saturating_sub(cached_input_tokens.unwrap_or(0)));
let output_tokens = usage.get("output_tokens").and_then(Value::as_u64);
Usage {
input_tokens,
+ cache: Usage::cache_details(cached_input_tokens, None),
output_tokens,
total_tokens: usage
.get("total_tokens")
.and_then(Value::as_u64)
- .or_else(|| Some(input_tokens.unwrap_or(0) + output_tokens.unwrap_or(0))),
+ .or_else(|| Some(aggregate_input_tokens.unwrap_or(0) + output_tokens.unwrap_or(0))),
reasoning_tokens: usage
.get("output_tokens_details")
.and_then(|details| details.get("reasoning_tokens"))
@@ -545,13 +552,19 @@ fn responses_usage(usage: &serde_json::Map<String, Value>) -> Usage {
// Builds OpenAI Responses usage payloads from normalized usage.
fn responses_usage_value(usage: &Usage) -> Value {
+ let input_tokens = usage.input_tokens.unwrap_or(0)
+ + usage.cached_input_tokens().unwrap_or(0)
+ + usage.cache_creation_input_tokens().unwrap_or(0);
let mut value = json!({
- "input_tokens": usage.input_tokens.unwrap_or(0),
+ "input_tokens": input_tokens,
"output_tokens": usage.output_tokens.unwrap_or(0),
"total_tokens": usage.total_tokens.unwrap_or_else(|| {
- usage.input_tokens.unwrap_or(0) + usage.output_tokens.unwrap_or(0)
+ input_tokens + usage.output_tokens.unwrap_or(0)
}),
});
+ if let Some(cached_tokens) = usage.cached_input_tokens() {
+ value["input_tokens_details"] = json!({"cached_tokens": cached_tokens});
+ }
if let Some(reasoning_tokens) = usage.reasoning_tokens {
value["output_tokens_details"] = json!({
"reasoning_tokens": reasoning_tokens,
diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rs
index 730e224b7..c2ddb7194 100644
--- a/crates/switchyard-translation/src/codecs/stream.rs
+++ b/crates/switchyard-translation/src/codecs/stream.rs
@@ -38,7 +38,6 @@ pub struct StreamTranslationState {
pub(crate) output_tokens_seen: u64,
pub(crate) saw_backend_usage: bool,
- pub(crate) usage_extras: BTreeMap<String, u64>,
pub(crate) stop_reason: Option<String>,
pub(crate) next_content_index: usize,
diff --git a/crates/switchyard-translation/tests/response_translation.rs b/crates/switchyard-translation/tests/response_translation.rs
index fa4022c98..6935467a9 100644
--- a/crates/switchyard-translation/tests/response_translation.rs
+++ b/crates/switchyard-translation/tests/response_translation.rs
@@ -122,6 +122,78 @@ fn responses_reasoning_usage_translates_to_openai_chat_usage_details() -> TestRe
Ok(())
}
+// Verifies OpenAI cache usage survives the Chat-to-Responses translation used by Codex.
+#[test]
+fn openai_chat_cache_usage_translates_to_responses_usage_details() -> TestResult {
+ let engine = TranslationEngine::default();
+ let body = json!({
+ "id": "chatcmpl-test",
+ "model": "gpt-cached",
+ "choices": [{
+ "index": 0,
+ "message": {"role": "assistant", "content": "Cached answer"},
+ "finish_reason": "stop"
+ }],
+ "usage": {
+ "prompt_tokens": 100,
+ "completion_tokens": 5,
+ "total_tokens": 105,
+ "prompt_tokens_details": {"cached_tokens": 80}
+ }
+ });
+
+ let output = engine
+ .translate_response(
+ WireFormat::OpenAiChat,
+ WireFormat::OpenAiResponses,
+ &body,
+ &TranslationPolicy::default(),
+ )?
+ .body;
+
+ assert_eq!(output["usage"]["input_tokens"], 100);
+ assert_eq!(
+ output["usage"]["input_tokens_details"],
+ json!({"cached_tokens": 80})
+ );
+ Ok(())
+}
+
+// Verifies OpenAI cache usage is expressed in Anthropic's standard sibling fields.
+#[test]
+fn openai_chat_cache_usage_translates_to_anthropic_usage_fields() -> TestResult {
+ let engine = TranslationEngine::default();
+ let body = json!({
+ "id": "chatcmpl-test",
+ "model": "gpt-cached",
+ "choices": [{
+ "index": 0,
+ "message": {"role": "assistant", "content": "Cached answer"},
+ "finish_reason": "stop"
+ }],
+ "usage": {
+ "prompt_tokens": 100,
+ "completion_tokens": 5,
+ "total_tokens": 105,
+ "prompt_tokens_details": {"cached_tokens": 80}
+ }
+ });
+
+ let output = engine
+ .translate_response(
+ WireFormat::OpenAiChat,
+ WireFormat::AnthropicMessages,
+ &body,
+ &TranslationPolicy::default(),
+ )?
+ .body;
+
+ assert_eq!(output["usage"]["input_tokens"], 20);
+ assert_eq!(output["usage"]["cache_read_input_tokens"], 80);
+ assert_eq!(output["usage"]["output_tokens"], 5);
+ Ok(())
+}
+
// Verifies Anthropic thinking response blocks become OpenAI reasoning_content.
#[test]
fn anthropic_thinking_response_translates_to_openai_reasoning_content() -> TestResult {
diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rs
index 3c4f5e658..1c1685ea3 100644
--- a/crates/switchyard-translation/tests/stream_translation.rs
+++ b/crates/switchyard-translation/tests/stream_translation.rs
@@ -265,6 +265,100 @@ fn openai_chat_stream_reasoning_usage_translates_to_responses_usage_details() ->
Ok(())
}
+// Verifies streamed cache usage reaches Responses clients in the standard details object.
+#[test]
+fn openai_chat_stream_cache_usage_translates_to_responses_usage_details() -> TestResult {
+ let engine = TranslationEngine::default();
+ let mut state =
+ StreamTranslationState::new(WireFormat::OpenAiChat, WireFormat::OpenAiResponses);
+ let usage = json!({
+ "id": "chatcmpl-test",
+ "object": "chat.completion.chunk",
+ "model": "gpt-cached",
+ "choices": [{
+ "index": 0,
+ "delta": {},
+ "finish_reason": "stop"
+ }],
+ "usage": {
+ "prompt_tokens": 100,
+ "completion_tokens": 5,
+ "total_tokens": 105,
+ "prompt_tokens_details": {"cached_tokens": 80}
+ }
+ });
+
+ let mut events = engine.translate_event(
+ &mut state,
+ WireFormat::OpenAiChat,
+ WireFormat::OpenAiResponses,
+ &usage,
+ )?;
+ events.extend(engine.finish_stream(&mut state, WireFormat::OpenAiResponses)?);
+
+ let Some(completed) = events
+ .iter()
+ .find(|event| event["type"] == "response.completed")
+ else {
+ return Err("expected final Responses completion event".into());
+ };
+ assert_eq!(completed["response"]["usage"]["input_tokens"], 100);
+ assert_eq!(
+ completed["response"]["usage"]["input_tokens_details"],
+ json!({"cached_tokens": 80})
+ );
+ Ok(())
+}
+
+// Verifies the streamed total-token fallback keeps cached tokens when upstream omits total_tokens.
+#[test]
+fn responses_stream_usage_without_total_keeps_cached_tokens_in_total() -> TestResult {
+ let engine = TranslationEngine::default();
+ let mut state =
+ StreamTranslationState::new(WireFormat::OpenAiResponses, WireFormat::OpenAiChat);
+ let created = json!({
+ "type": "response.created",
+ "response": {"id": "resp_1", "model": "gpt-cached"}
+ });
+ engine.translate_event(
+ &mut state,
+ WireFormat::OpenAiResponses,
+ WireFormat::OpenAiChat,
+ &created,
+ )?;
+
+ // No total_tokens field: the codec must recompute it from the aggregate input.
+ let completed = json!({
+ "type": "response.completed",
+ "response": {
+ "id": "resp_1",
+ "usage": {
+ "input_tokens": 100,
+ "output_tokens": 5,
+ "input_tokens_details": {"cached_tokens": 80}
+ }
+ }
+ });
+ let mut events = engine.translate_event(
+ &mut state,
+ WireFormat::OpenAiResponses,
+ WireFormat::OpenAiChat,
+ &completed,
+ )?;
+ events.extend(engine.finish_stream(&mut state, WireFormat::OpenAiChat)?);
+
+ let Some(usage) = events
+ .iter()
+ .find_map(|event| event.get("usage").filter(|usage| !usage.is_null()))
+ else {
+ return Err("expected a terminal OpenAI chunk carrying usage".into());
+ };
+ assert_eq!(usage["prompt_tokens"], 100);
+ assert_eq!(usage["total_tokens"], 105);
+ assert_eq!(usage["prompt_tokens_details"]["cached_tokens"], 80);
+ Ok(())
+}
+
// Verifies Responses text deltas become OpenAI Chat content chunks.
#[test]
fn responses_stream_delta_translates_to_openai_chat_chunk() -> TestResult {
diff --git a/tests/test_response_translation_engine.py b/tests/test_response_translation_engine.py
index 0bfa72d85..fd41b29ab 100644
--- a/tests/test_response_translation_engine.py
+++ b/tests/test_response_translation_engine.py
@@ -294,7 +294,7 @@ class TestStreamOpenAIToAnthropicUsage:
@pytest.mark.asyncio
async def test_cached_tokens_propagated(self):
- """cache_read_input_tokens is set when prompt_tokens_details.cached_tokens is present."""
+ """Anthropic usage splits uncached input from cache-read tokens."""
usage = SimpleNamespace(
prompt_tokens=1000,
completion_tokens=10,
@@ -306,7 +306,7 @@ class TestStreamOpenAIToAnthropicUsage:
msg_delta = _find_event(events, "message_delta")
assert msg_delta["usage"]["cache_read_input_tokens"] == 800
- assert msg_delta["usage"]["input_tokens"] == 1000
+ assert msg_delta["usage"]["input_tokens"] == 200
@pytest.mark.asyncio
async def test_heuristic_fallback_when_no_backend_usage(self):