Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(translation): preserve Responses cache-write usage (#770)

nachiketb-nvidia0c3c4a556d57
diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs
index ef3f078ed..844a31e43 100644
--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs
@@ -1933,15 +1933,30 @@ fn decode_responses_usage(value: Option<&Value>) -> Usage {
.or_else(|| value.get("prompt_tokens_details"))
.and_then(|details| details.get("cached_tokens"))
.and_then(Value::as_u64);
- let input_tokens = aggregate_input_tokens
- .map(|tokens| tokens.saturating_sub(cached_input_tokens.unwrap_or(0)));
+ let cache_creation_input_tokens = value
+ .get("input_tokens_details")
+ .and_then(|details| details.get("cache_write_tokens"))
+ .and_then(Value::as_u64)
+ .or_else(|| {
+ value.get("prompt_tokens_details").and_then(|details| {
+ details
+ .get("cache_write_tokens")
+ .and_then(Value::as_u64)
+ .or_else(|| details.get("cache_creation_tokens").and_then(Value::as_u64))
+ })
+ });
+ let input_tokens = aggregate_input_tokens.map(|tokens| {
+ tokens
+ .saturating_sub(cached_input_tokens.unwrap_or(0))
+ .saturating_sub(cache_creation_input_tokens.unwrap_or(0))
+ });
let output_tokens = value
.get("output_tokens")
.or_else(|| value.get("completion_tokens"))
.and_then(Value::as_u64);
Usage {
input_tokens,
- cache: Usage::cache_details(cached_input_tokens, None),
+ cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens),
output_tokens,
total_tokens: value
.get("total_tokens")
@@ -1979,7 +1994,10 @@ fn encode_responses_usage(usage: &Usage) -> Value {
.total_tokens
.or_else(|| Some(input_tokens + usage.output_tokens.unwrap_or(0)))
.unwrap_or(0),
- "input_tokens_details": {"cached_tokens": usage.cached_input_tokens().unwrap_or(0)},
+ "input_tokens_details": {
+ "cached_tokens": usage.cached_input_tokens().unwrap_or(0),
+ "cache_write_tokens": usage.cache_creation_input_tokens().unwrap_or(0),
+ },
"output_tokens_details": {"reasoning_tokens": usage.reasoning_tokens.unwrap_or(0)},
})
}
diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rs
index 608d0f6a8..399b0c3e2 100644
--- a/crates/switchyard-translation/src/codecs/responses/stream.rs
+++ b/crates/switchyard-translation/src/codecs/responses/stream.rs
@@ -991,12 +991,19 @@ fn responses_usage(usage: &serde_json::Map<String, Value>) -> Usage {
.get("input_tokens_details")
.and_then(|details| details.get("cached_tokens"))
.and_then(Value::as_u64);
- let input_tokens = aggregate_input_tokens
- .map(|tokens| tokens.saturating_sub(cached_input_tokens.unwrap_or(0)));
+ let cache_creation_input_tokens = usage
+ .get("input_tokens_details")
+ .and_then(|details| details.get("cache_write_tokens"))
+ .and_then(Value::as_u64);
+ let input_tokens = aggregate_input_tokens.map(|tokens| {
+ tokens
+ .saturating_sub(cached_input_tokens.unwrap_or(0))
+ .saturating_sub(cache_creation_input_tokens.unwrap_or(0))
+ });
let output_tokens = usage.get("output_tokens").and_then(Value::as_u64);
Usage {
input_tokens,
- cache: Usage::cache_details(cached_input_tokens, None),
+ cache: Usage::cache_details(cached_input_tokens, cache_creation_input_tokens),
output_tokens,
total_tokens: usage
.get("total_tokens")
@@ -1027,7 +1034,10 @@ fn responses_usage_value(usage: &Usage) -> Value {
"total_tokens": usage.total_tokens.unwrap_or_else(|| {
input_tokens + usage.output_tokens.unwrap_or(0)
}),
- "input_tokens_details": {"cached_tokens": usage.cached_input_tokens().unwrap_or(0)},
+ "input_tokens_details": {
+ "cached_tokens": usage.cached_input_tokens().unwrap_or(0),
+ "cache_write_tokens": usage.cache_creation_input_tokens().unwrap_or(0),
+ },
"output_tokens_details": {"reasoning_tokens": usage.reasoning_tokens.unwrap_or(0)},
})
}
diff --git a/crates/switchyard-translation/tests/response_translation.rs b/crates/switchyard-translation/tests/response_translation.rs
index c4ad3e83d..86276fcd7 100644
--- a/crates/switchyard-translation/tests/response_translation.rs
+++ b/crates/switchyard-translation/tests/response_translation.rs
@@ -190,6 +190,7 @@ fn responses_reasoning_usage_translates_to_openai_chat_usage_details() -> TestRe
"input_tokens": 10,
"output_tokens": 5,
"total_tokens": 15,
+ "input_tokens_details": {"cached_tokens": 4, "cache_write_tokens": 2},
"output_tokens_details": {"reasoning_tokens": 3}
}
});
@@ -209,6 +210,64 @@ fn responses_reasoning_usage_translates_to_openai_chat_usage_details() -> TestRe
output["usage"]["completion_tokens_details"],
json!({"reasoning_tokens": 3})
);
+ assert_eq!(
+ output["usage"]["prompt_tokens_details"]["cache_creation_tokens"],
+ 2
+ );
+ let decoded = engine
+ .decode_response(WireFormat::OpenAiResponses, &body, &normalized_policy())?
+ .response;
+ assert_eq!(decoded.usage.input_tokens, Some(4));
+ assert_eq!(decoded.usage.cached_input_tokens(), Some(4));
+ assert_eq!(decoded.usage.cache_creation_input_tokens(), Some(2));
+ assert_eq!(decoded.usage.total_tokens, Some(15));
+ let mut aliased = body.clone();
+ aliased["usage"] = output["usage"].clone();
+ assert_eq!(
+ engine
+ .decode_response(WireFormat::OpenAiResponses, &aliased, &normalized_policy())?
+ .response
+ .usage,
+ decoded.usage
+ );
+ let encoded = engine
+ .encode_response(WireFormat::OpenAiResponses, &decoded, &normalized_policy())?
+ .body;
+ assert_eq!(encoded["usage"], body["usage"]);
+
+ let mut state = switchyard_translation::StreamTranslationState::new(
+ WireFormat::OpenAiResponses,
+ WireFormat::OpenAiChat,
+ );
+ let events = engine.translate_event(
+ &mut state,
+ WireFormat::OpenAiResponses,
+ WireFormat::OpenAiChat,
+ &json!({"type": "response.completed", "response": body}),
+ )?;
+ assert_eq!(state.usage, decoded.usage);
+ let chat = events
+ .iter()
+ .find(|event| event.get("usage").is_some())
+ .ok_or("missing usage")?;
+ assert_eq!(chat["usage"], output["usage"]);
+ let mut state = switchyard_translation::StreamTranslationState::new(
+ WireFormat::OpenAiChat,
+ WireFormat::OpenAiResponses,
+ );
+ let mut events = engine.translate_event(
+ &mut state,
+ WireFormat::OpenAiChat,
+ WireFormat::OpenAiResponses,
+ &json!({"id": "chat_usage", "model": "model", "usage": output["usage"],
+ "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}),
+ )?;
+ events.extend(engine.finish_stream(&mut state, WireFormat::OpenAiResponses)?);
+ let completed = events
+ .iter()
+ .find(|event| event["type"] == "response.completed")
+ .ok_or("missing completion")?;
+ assert_eq!(completed["response"]["usage"], body["usage"]);
Ok(())
}
@@ -298,7 +357,7 @@ fn openai_chat_cache_usage_translates_to_responses_usage_details() -> TestResult
assert_eq!(output["usage"]["input_tokens"], 100);
assert_eq!(
output["usage"]["input_tokens_details"],
- json!({"cached_tokens": 80})
+ json!({"cached_tokens": 80, "cache_write_tokens": 0})
);
Ok(())
}
@@ -622,7 +681,7 @@ fn openai_chat_response_with_tool_call_translates_to_responses_output_item() ->
"input_tokens": 4,
"output_tokens": 3,
"total_tokens": 7,
- "input_tokens_details": {"cached_tokens": 0},
+ "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},
"output_tokens_details": {"reasoning_tokens": 0}
})
);
@@ -700,7 +759,7 @@ fn openai_chat_usage_without_breakdowns_still_emits_responses_usage_details() ->
"input_tokens": 41,
"output_tokens": 3,
"total_tokens": 44,
- "input_tokens_details": {"cached_tokens": 0},
+ "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},
"output_tokens_details": {"reasoning_tokens": 0}
})
);
@@ -739,7 +798,7 @@ fn openai_chat_cache_only_usage_still_emits_reasoning_details() -> TestResult {
assert_eq!(
output["usage"]["input_tokens_details"],
- json!({"cached_tokens": 32})
+ json!({"cached_tokens": 32, "cache_write_tokens": 0})
);
assert_eq!(
output["usage"]["output_tokens_details"],
diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rs
index c9de9b8d7..1613445fe 100644
--- a/crates/switchyard-translation/tests/stream_translation.rs
+++ b/crates/switchyard-translation/tests/stream_translation.rs
@@ -882,7 +882,7 @@ fn openai_chat_stream_cache_usage_translates_to_responses_usage_details() -> Tes
assert_eq!(completed["response"]["usage"]["input_tokens"], 100);
assert_eq!(
completed["response"]["usage"]["input_tokens_details"],
- json!({"cached_tokens": 80})
+ json!({"cached_tokens": 80, "cache_write_tokens": 0})
);
Ok(())
}
@@ -1581,7 +1581,7 @@ fn openai_chat_stream_usage_without_breakdowns_still_emits_responses_usage_detai
};
assert_eq!(
completed["response"]["usage"]["input_tokens_details"],
- json!({"cached_tokens": 0})
+ json!({"cached_tokens": 0, "cache_write_tokens": 0})
);
assert_eq!(
completed["response"]["usage"]["output_tokens_details"],