Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(observability): expose one-hour cache writes (#1189)
Soumili Nandi1faddb44ef38
diff --git a/crates/libsy-llm-client/src/observability.rs b/crates/libsy-llm-client/src/observability.rsindex fd1c13a10..9e45aef00 100644--- a/crates/libsy-llm-client/src/observability.rs+++ b/crates/libsy-llm-client/src/observability.rs@@ -112,9 +112,14 @@ fn record_gen_ai_response(span: &Span, response: &AggLlmResponse) {record_gen_ai_usage(span, &response.usage);}+/// Records semantic token totals and cache-usage breakdowns on a client-call span.fn record_gen_ai_usage(span: &Span, usage: &Usage) {let cache_read = usage.cached_input_tokens();let cache_creation = usage.cache_creation_input_tokens();+ let cache_creation_1h = usage+ .cache+ .as_ref()+ .and_then(|cache| cache.cache_creation_1h_input_tokens);if usage.input_tokens.is_some() || cache_read.is_some() || cache_creation.is_some() {let input_tokens = usage.input_tokens@@ -127,6 +132,10 @@ fn record_gen_ai_usage(span: &Span, usage: &Usage) {("gen_ai.usage.output_tokens", usage.output_tokens),("gen_ai.usage.cache_read.input_tokens", cache_read),("gen_ai.usage.cache_creation.input_tokens", cache_creation),+ (+ "gen_ai.usage.cache_creation_1h.input_tokens",+ cache_creation_1h,+ ),("gen_ai.usage.reasoning.output_tokens",usage.reasoning_tokens,diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rsindex 094dde2b7..d660f8c98 100644--- a/crates/libsy-llm-client/src/run.rs+++ b/crates/libsy-llm-client/src/run.rs@@ -419,6 +419,7 @@ impl CallPhase {gen_ai.usage.output_tokens = tracing::field::Empty,gen_ai.usage.cache_read.input_tokens = tracing::field::Empty,gen_ai.usage.cache_creation.input_tokens = tracing::field::Empty,+ gen_ai.usage.cache_creation_1h.input_tokens = tracing::field::Empty,gen_ai.usage.reasoning.output_tokens = tracing::field::Empty,outcome = tracing::field::Empty,otel.status_code = tracing::field::Empty,diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rsindex ec16a2c0f..9ca368769 100644--- a/crates/libsy-llm-client/tests/observability.rs+++ b/crates/libsy-llm-client/tests/observability.rs@@ -989,16 +989,20 @@ async fn successful_run_records_metrics_spans_and_outcome_metadata() -> switchyalet total_errors_before =u64_gauge_value(&before, "switchyard.total_errors").unwrap_or_default();- let client = Arc::new(UsageClient {- usage: Usage {- input_tokens: Some(11),- output_tokens: Some(7),- total_tokens: Some(25),- reasoning_tokens: Some(2),- cache: Usage::cache_details(Some(3), Some(4)),- provider_cost: None,- },- }) as Arc<dyn RoutedLlmClient>;+ let mut usage = Usage {+ input_tokens: Some(11),+ output_tokens: Some(7),+ total_tokens: Some(25),+ reasoning_tokens: Some(2),+ cache: Usage::cache_details(Some(3), Some(4)),+ provider_cost: None,+ };+ usage+ .cache+ .as_mut()+ .expect("cache details")+ .cache_creation_1h_input_tokens = Some(2);+ let client = Arc::new(UsageClient { usage }) as Arc<dyn RoutedLlmClient>;let mut request = request_with_metadata("obs-session-1", "obs-corr-1");request.llm_request.sampling.temperature = Some(0.25);request.llm_request.sampling.top_p = Some(0.9);@@ -1206,6 +1210,7 @@ async fn successful_run_records_metrics_spans_and_outcome_metadata() -> switchya("gen_ai.usage.output_tokens", "7"),("gen_ai.usage.cache_read.input_tokens", "3"),("gen_ai.usage.cache_creation.input_tokens", "4"),+ ("gen_ai.usage.cache_creation_1h.input_tokens", "2"),("gen_ai.usage.reasoning.output_tokens", "2"),] {assert_eq!(diff --git a/crates/switchyard-nemo-relay-plugin/src/span_capture.rs b/crates/switchyard-nemo-relay-plugin/src/span_capture.rsindex 94037b024..31bf477b8 100644--- a/crates/switchyard-nemo-relay-plugin/src/span_capture.rs+++ b/crates/switchyard-nemo-relay-plugin/src/span_capture.rs@@ -158,6 +158,7 @@ impl CapturedSpan {}}+/// Reconstructs the bounded response metadata forwarded to NeMo Relay.fn response_data(fields: &Json) -> Option<Json> {let mut output = Map::new();for (source, target) in [@@ -184,6 +185,12 @@ fn response_data(fields: &Json) -> Option<Json> {usage.insert(target.into(), value.clone());}}+ if let Some(value) = fields.get("gen_ai.usage.cache_creation_1h.input_tokens") {+ usage.insert(+ "cache_creation".into(),+ serde_json::json!({"ephemeral_1h_input_tokens": value}),+ );+ }if let Some(input) = fields.get("gen_ai.usage.input_tokens").and_then(Json::as_u64)@@ -318,6 +325,7 @@ fn relay_scope_type(name: &str) -> ScopeType {}}+/// Returns whether a captured span field is safe to forward across the plugin boundary.fn is_safe_field(name: &str) -> bool {matches!(name,@@ -341,6 +349,7 @@ fn is_safe_field(name: &str) -> bool {| "gen_ai.usage.output_tokens"| "gen_ai.usage.cache_read.input_tokens"| "gen_ai.usage.cache_creation.input_tokens"+ | "gen_ai.usage.cache_creation_1h.input_tokens"| "gen_ai.usage.reasoning.output_tokens"| "model"| "wire_format"@@ -446,6 +455,26 @@ mod tests {assert_eq!(call.children[0].fields["model"], "[REDACTED]");}+ #[test]+ fn response_preserves_one_hour_cache_creation_subset() {+ let fields = serde_json::json!({+ "gen_ai.response.model": "target/model",+ "gen_ai.usage.input_tokens": 180,+ "gen_ai.usage.output_tokens": 7,+ "gen_ai.usage.cache_read.input_tokens": 30,+ "gen_ai.usage.cache_creation.input_tokens": 50,+ "gen_ai.usage.cache_creation_1h.input_tokens": 20,+ });++ let response = response_data(&fields).expect("response data");+ assert_eq!(response["usage"]["input_tokens"], 100);+ assert_eq!(response["usage"]["cache_creation_input_tokens"], 50);+ assert_eq!(+ response["usage"]["cache_creation"]["ephemeral_1h_input_tokens"],+ 20+ );+ }+#[tokio::test]async fn captures_spans_that_close_after_the_scoped_future_returns() {let (capture, captured) = recorder();