Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(observability): expose one-hour cache writes (#1189)

Soumili Nandi1faddb44ef38
diff --git a/crates/libsy-llm-client/src/observability.rs b/crates/libsy-llm-client/src/observability.rs
index fd1c13a10..9e45aef00 100644
--- a/crates/libsy-llm-client/src/observability.rs
+++ b/crates/libsy-llm-client/src/observability.rs
@@ -112,9 +112,14 @@ fn record_gen_ai_response(span: &Span, response: &AggLlmResponse) {
record_gen_ai_usage(span, &response.usage);
}
+/// Records semantic token totals and cache-usage breakdowns on a client-call span.
fn record_gen_ai_usage(span: &Span, usage: &Usage) {
let cache_read = usage.cached_input_tokens();
let cache_creation = usage.cache_creation_input_tokens();
+ let cache_creation_1h = usage
+ .cache
+ .as_ref()
+ .and_then(|cache| cache.cache_creation_1h_input_tokens);
if usage.input_tokens.is_some() || cache_read.is_some() || cache_creation.is_some() {
let input_tokens = usage
.input_tokens
@@ -127,6 +132,10 @@ fn record_gen_ai_usage(span: &Span, usage: &Usage) {
("gen_ai.usage.output_tokens", usage.output_tokens),
("gen_ai.usage.cache_read.input_tokens", cache_read),
("gen_ai.usage.cache_creation.input_tokens", cache_creation),
+ (
+ "gen_ai.usage.cache_creation_1h.input_tokens",
+ cache_creation_1h,
+ ),
(
"gen_ai.usage.reasoning.output_tokens",
usage.reasoning_tokens,
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs
index 094dde2b7..d660f8c98 100644
--- a/crates/libsy-llm-client/src/run.rs
+++ b/crates/libsy-llm-client/src/run.rs
@@ -419,6 +419,7 @@ impl CallPhase {
gen_ai.usage.output_tokens = tracing::field::Empty,
gen_ai.usage.cache_read.input_tokens = tracing::field::Empty,
gen_ai.usage.cache_creation.input_tokens = tracing::field::Empty,
+ gen_ai.usage.cache_creation_1h.input_tokens = tracing::field::Empty,
gen_ai.usage.reasoning.output_tokens = tracing::field::Empty,
outcome = tracing::field::Empty,
otel.status_code = tracing::field::Empty,
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs
index ec16a2c0f..9ca368769 100644
--- a/crates/libsy-llm-client/tests/observability.rs
+++ b/crates/libsy-llm-client/tests/observability.rs
@@ -989,16 +989,20 @@ async fn successful_run_records_metrics_spans_and_outcome_metadata() -> switchya
let total_errors_before =
u64_gauge_value(&before, "switchyard.total_errors").unwrap_or_default();
- let client = Arc::new(UsageClient {
- usage: Usage {
- input_tokens: Some(11),
- output_tokens: Some(7),
- total_tokens: Some(25),
- reasoning_tokens: Some(2),
- cache: Usage::cache_details(Some(3), Some(4)),
- provider_cost: None,
- },
- }) as Arc<dyn RoutedLlmClient>;
+ let mut usage = Usage {
+ input_tokens: Some(11),
+ output_tokens: Some(7),
+ total_tokens: Some(25),
+ reasoning_tokens: Some(2),
+ cache: Usage::cache_details(Some(3), Some(4)),
+ provider_cost: None,
+ };
+ usage
+ .cache
+ .as_mut()
+ .expect("cache details")
+ .cache_creation_1h_input_tokens = Some(2);
+ let client = Arc::new(UsageClient { usage }) as Arc<dyn RoutedLlmClient>;
let mut request = request_with_metadata("obs-session-1", "obs-corr-1");
request.llm_request.sampling.temperature = Some(0.25);
request.llm_request.sampling.top_p = Some(0.9);
@@ -1206,6 +1210,7 @@ async fn successful_run_records_metrics_spans_and_outcome_metadata() -> switchya
("gen_ai.usage.output_tokens", "7"),
("gen_ai.usage.cache_read.input_tokens", "3"),
("gen_ai.usage.cache_creation.input_tokens", "4"),
+ ("gen_ai.usage.cache_creation_1h.input_tokens", "2"),
("gen_ai.usage.reasoning.output_tokens", "2"),
] {
assert_eq!(
diff --git a/crates/switchyard-nemo-relay-plugin/src/span_capture.rs b/crates/switchyard-nemo-relay-plugin/src/span_capture.rs
index 94037b024..31bf477b8 100644
--- a/crates/switchyard-nemo-relay-plugin/src/span_capture.rs
+++ b/crates/switchyard-nemo-relay-plugin/src/span_capture.rs
@@ -158,6 +158,7 @@ impl CapturedSpan {
}
}
+/// Reconstructs the bounded response metadata forwarded to NeMo Relay.
fn response_data(fields: &Json) -> Option<Json> {
let mut output = Map::new();
for (source, target) in [
@@ -184,6 +185,12 @@ fn response_data(fields: &Json) -> Option<Json> {
usage.insert(target.into(), value.clone());
}
}
+ if let Some(value) = fields.get("gen_ai.usage.cache_creation_1h.input_tokens") {
+ usage.insert(
+ "cache_creation".into(),
+ serde_json::json!({"ephemeral_1h_input_tokens": value}),
+ );
+ }
if let Some(input) = fields
.get("gen_ai.usage.input_tokens")
.and_then(Json::as_u64)
@@ -318,6 +325,7 @@ fn relay_scope_type(name: &str) -> ScopeType {
}
}
+/// Returns whether a captured span field is safe to forward across the plugin boundary.
fn is_safe_field(name: &str) -> bool {
matches!(
name,
@@ -341,6 +349,7 @@ fn is_safe_field(name: &str) -> bool {
| "gen_ai.usage.output_tokens"
| "gen_ai.usage.cache_read.input_tokens"
| "gen_ai.usage.cache_creation.input_tokens"
+ | "gen_ai.usage.cache_creation_1h.input_tokens"
| "gen_ai.usage.reasoning.output_tokens"
| "model"
| "wire_format"
@@ -446,6 +455,26 @@ mod tests {
assert_eq!(call.children[0].fields["model"], "[REDACTED]");
}
+ #[test]
+ fn response_preserves_one_hour_cache_creation_subset() {
+ let fields = serde_json::json!({
+ "gen_ai.response.model": "target/model",
+ "gen_ai.usage.input_tokens": 180,
+ "gen_ai.usage.output_tokens": 7,
+ "gen_ai.usage.cache_read.input_tokens": 30,
+ "gen_ai.usage.cache_creation.input_tokens": 50,
+ "gen_ai.usage.cache_creation_1h.input_tokens": 20,
+ });
+
+ let response = response_data(&fields).expect("response data");
+ assert_eq!(response["usage"]["input_tokens"], 100);
+ assert_eq!(response["usage"]["cache_creation_input_tokens"], 50);
+ assert_eq!(
+ response["usage"]["cache_creation"]["ephemeral_1h_input_tokens"],
+ 20
+ );
+ }
+
#[tokio::test]
async fn captures_spans_that_close_after_the_scoped_future_returns() {
let (capture, captured) = recorder();