Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(metrics): attribute fallback calls to each candidate (#773)

nachiketb-nvidiab5a9a7480407
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs
index d4dcc5de7..10b098f96 100644
--- a/crates/libsy-llm-client/src/run.rs
+++ b/crates/libsy-llm-client/src/run.rs
@@ -90,34 +90,31 @@ pub async fn run(
{
observer(RunObservation::Outcome(metadata));
}
- let (result, answer_duration) = if let Some(response) = outcome.response {
- (Ok(response), None)
+ let result = if let Some(response) = outcome.response {
+ let result = clients.remember_state_owner(&outcome.request, response);
+ let served_model = result
+ .as_ref()
+ .ok()
+ .and_then(Response::served_model)
+ .unwrap_or(&selected_model_id);
+ let served_model = served_model.clone();
+ metrics::observe_routed_request(&algorithm_name, &served_model, None, result)
} else {
- let answer_started = Instant::now();
let observe = |observation| {
if let Some(observer) = &observer {
observer(RunObservation::AnswerCall(observation));
}
};
- let result = call_first_available(
+ call_first_available(
&clients,
&algorithm_name,
&outcome.request,
&outcome.selected_model_ids,
&observe,
)
- .await;
- let answer_duration = answer_started.elapsed();
- (result, Some(answer_duration))
+ .await
+ .and_then(|response| clients.remember_state_owner(&outcome.request, response))
};
- let result =
- result.and_then(|response| clients.remember_state_owner(&outcome.request, response));
- let result = metrics::observe_routed_request(
- &algorithm_name,
- &selected_model_id,
- answer_duration,
- result,
- );
if let Some(observer) = &observer {
observer(RunObservation::RoutingOverhead(overhead));
}
@@ -321,6 +318,11 @@ async fn call_one(
response
});
let result = observability::observe_client_call(result);
+ let result = if !buffer {
+ metrics::observe_routed_request(algorithm, model_id, Some(duration), result)
+ } else {
+ result
+ };
observe(LlmCallObservation {
selected_model: model_id.clone(),
is_success: result.is_ok(),
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs
index 4024658e1..8150564f9 100644
--- a/crates/libsy-llm-client/tests/observability.rs
+++ b/crates/libsy-llm-client/tests/observability.rs
@@ -764,6 +764,8 @@ async fn affinity_warns_once_when_request_has_no_usable_identity() -> switchyard
async fn affinity_keeps_the_algorithm_selection_after_client_fallback()
-> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
+ let (_, exporter, provider, _, _) = telemetry();
+ let before = flushed_metrics(exporter, provider);
let client = Arc::new(AffinityFallbackClient {
calls: Mutex::new(Vec::new()),
efficient_available: AtomicBool::new(false),
@@ -795,6 +797,50 @@ async fn affinity_keeps_the_algorithm_selection_after_client_fallback()
Some("affinity-fallback-strong")
);
+ // One client request makes two routed calls: a weak failure and a strong success.
+ let after = flushed_metrics(exporter, provider);
+ for (metric, model, expected) in [
+ ("switchyard.errors", "affinity-fallback-weak", 1),
+ ("switchyard.requests", "affinity-fallback-weak", 0),
+ ("switchyard.requests", "affinity-fallback-strong", 1),
+ ] {
+ let attrs = [("model", model)];
+ assert_eq!(
+ u64_counter_value(&after, metric, &attrs).unwrap_or_default()
+ - u64_counter_value(&before, metric, &attrs).unwrap_or_default(),
+ expected,
+ "{metric} for {model}"
+ );
+ }
+ for (metric, expected) in [
+ ("switchyard.total_requests", 2),
+ ("switchyard.total_errors", 1),
+ ] {
+ assert_eq!(
+ u64_gauge_value(&after, metric).unwrap_or_default()
+ - u64_gauge_value(&before, metric).unwrap_or_default(),
+ expected,
+ "{metric}"
+ );
+ }
+ for (model, outcome) in [
+ ("affinity-fallback-weak", "error"),
+ ("affinity-fallback-strong", "ok"),
+ ] {
+ assert_eq!(
+ u64_counter_value(
+ &after,
+ "switchyard.llm_calls",
+ &[
+ ("algorithm", "llm_task_classifier"),
+ ("selected_model", model),
+ ("outcome", outcome)
+ ]
+ ),
+ Some(1)
+ );
+ }
+
client.efficient_available.store(true, Ordering::Relaxed);
let (selected, second_response) = switchyard_llm_client::run(
router,