Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(metrics): attribute fallback calls to each candidate (#773)
nachiketb-nvidiab5a9a7480407
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rsindex d4dcc5de7..10b098f96 100644--- a/crates/libsy-llm-client/src/run.rs+++ b/crates/libsy-llm-client/src/run.rs@@ -90,34 +90,31 @@ pub async fn run({observer(RunObservation::Outcome(metadata));}- let (result, answer_duration) = if let Some(response) = outcome.response {- (Ok(response), None)+ let result = if let Some(response) = outcome.response {+ let result = clients.remember_state_owner(&outcome.request, response);+ let served_model = result+ .as_ref()+ .ok()+ .and_then(Response::served_model)+ .unwrap_or(&selected_model_id);+ let served_model = served_model.clone();+ metrics::observe_routed_request(&algorithm_name, &served_model, None, result)} else {- let answer_started = Instant::now();let observe = |observation| {if let Some(observer) = &observer {observer(RunObservation::AnswerCall(observation));}};- let result = call_first_available(+ call_first_available(&clients,&algorithm_name,&outcome.request,&outcome.selected_model_ids,&observe,)- .await;- let answer_duration = answer_started.elapsed();- (result, Some(answer_duration))+ .await+ .and_then(|response| clients.remember_state_owner(&outcome.request, response))};- let result =- result.and_then(|response| clients.remember_state_owner(&outcome.request, response));- let result = metrics::observe_routed_request(- &algorithm_name,- &selected_model_id,- answer_duration,- result,- );if let Some(observer) = &observer {observer(RunObservation::RoutingOverhead(overhead));}@@ -321,6 +318,11 @@ async fn call_one(response});let result = observability::observe_client_call(result);+ let result = if !buffer {+ metrics::observe_routed_request(algorithm, model_id, Some(duration), result)+ } else {+ result+ };observe(LlmCallObservation {selected_model: model_id.clone(),is_success: result.is_ok(),diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rsindex 4024658e1..8150564f9 100644--- a/crates/libsy-llm-client/tests/observability.rs+++ b/crates/libsy-llm-client/tests/observability.rs@@ -764,6 +764,8 @@ async fn affinity_warns_once_when_request_has_no_usable_identity() -> switchyardasync fn affinity_keeps_the_algorithm_selection_after_client_fallback()-> switchyard_libsy::Result<()> {let _guard = serialize_test().lock().await;+ let (_, exporter, provider, _, _) = telemetry();+ let before = flushed_metrics(exporter, provider);let client = Arc::new(AffinityFallbackClient {calls: Mutex::new(Vec::new()),efficient_available: AtomicBool::new(false),@@ -795,6 +797,50 @@ async fn affinity_keeps_the_algorithm_selection_after_client_fallback()Some("affinity-fallback-strong"));+ // One client request makes two routed calls: a weak failure and a strong success.+ let after = flushed_metrics(exporter, provider);+ for (metric, model, expected) in [+ ("switchyard.errors", "affinity-fallback-weak", 1),+ ("switchyard.requests", "affinity-fallback-weak", 0),+ ("switchyard.requests", "affinity-fallback-strong", 1),+ ] {+ let attrs = [("model", model)];+ assert_eq!(+ u64_counter_value(&after, metric, &attrs).unwrap_or_default()+ - u64_counter_value(&before, metric, &attrs).unwrap_or_default(),+ expected,+ "{metric} for {model}"+ );+ }+ for (metric, expected) in [+ ("switchyard.total_requests", 2),+ ("switchyard.total_errors", 1),+ ] {+ assert_eq!(+ u64_gauge_value(&after, metric).unwrap_or_default()+ - u64_gauge_value(&before, metric).unwrap_or_default(),+ expected,+ "{metric}"+ );+ }+ for (model, outcome) in [+ ("affinity-fallback-weak", "error"),+ ("affinity-fallback-strong", "ok"),+ ] {+ assert_eq!(+ u64_counter_value(+ &after,+ "switchyard.llm_calls",+ &[+ ("algorithm", "llm_task_classifier"),+ ("selected_model", model),+ ("outcome", outcome)+ ]+ ),+ Some(1)+ );+ }+client.efficient_available.store(true, Ordering::Relaxed);let (selected, second_response) = switchyard_llm_client::run(router,