Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix: Populate the switchyard.router_retry_recovered metric (#474)
Graham King04adc1fe7095
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rsindex d58508ca5..bf8ec1e54 100644--- a/crates/libsy-llm-client/src/client.rs+++ b/crates/libsy-llm-client/src/client.rs@@ -261,6 +261,9 @@ impl TranslatingLlmClient {span.record("outcome", "success");span.record("status_code", response.status());span.record("will_retry", false);+ if attempt > 0 {+ metrics::record_retry_recovered();+ }return Ok(response);}Err(failure) => {diff --git a/crates/libsy-llm-client/src/metrics.rs b/crates/libsy-llm-client/src/metrics.rsindex 724e34086..bf22b421e 100644--- a/crates/libsy-llm-client/src/metrics.rs+++ b/crates/libsy-llm-client/src/metrics.rs@@ -92,6 +92,14 @@ pub(crate) fn record_upstream_attempt(status: Option<u16>) {);}+/// Records an upstream operation that succeeded after at least one retry.+pub(crate) fn record_retry_recovered() {+ global::meter("switchyard")+ .u64_counter("switchyard.router_retry_recovered")+ .build()+ .add(1, &[]);+}+/// Records the time needed to produce the routing outcome, including classifier calls,/// target resolution, request rewrites, and decision publishing.pub(crate) fn record_routing_overhead(algorithm: &str, overhead: Duration) {diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.mdindex cf107fecc..65c5083b6 100644--- a/crates/switchyard-server/README.md+++ b/crates/switchyard-server/README.md@@ -171,7 +171,7 @@ Routed-call compatibility metrics are:| `switchyard_classifier_fail_open_total` | counter | `judge_model`, `reason` | Judge failures that made a classifier route without a verdict || `switchyard_client_responses_total` | counter | `outcome` | Final LLM-route responses || `switchyard_upstream_attempts_total` | counter | `outcome`, `code` | Actual upstream HTTP attempts |-| `switchyard_router_retry_recovered_total` | counter | none | Retry recoveries (currently always zero) |+| `switchyard_router_retry_recovered_total` | counter | none | Upstream operations recovered by a retry |`switchyard_classifier_fail_open_total` counts requests that still reached a target after thejudge call failed. `judge_model` names the configured judge target, and `reason` is one of eightdiff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex fc15ba3f8..0a97f90e3 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -113,6 +113,22 @@ async fn upstream_chat(let model = body["model"].as_str().unwrap_or("unknown").to_string();let prompt = body["messages"][0]["content"].as_str().unwrap_or("");+ if prompt == "retry-once"+ && calls+ .lock()+ .await+ .iter()+ .filter(|call| call["messages"][0]["content"] == "retry-once")+ .count()+ == 1+ {+ return (+ StatusCode::SERVICE_UNAVAILABLE,+ [("retry-after", "0")],+ Json(json!({"error": {"message": "upstream is temporarily unavailable"}})),+ )+ .into_response();+ }if (model == "model/weak" && prompt == "unavailable") || prompt == "all-unavailable" {return (StatusCode::SERVICE_UNAVAILABLE,@@ -365,13 +381,21 @@ async fn upstream_count_tokens(}fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult<ServerState> {+ random_state_with_retries(base_url, routes, 0)+}++fn random_state_with_retries(+ base_url: &str,+ routes: &[(&str, &[&str])],+ max_retries: u32,+) -> TestResult<ServerState> {let backend = Backend::OpenAiChat(HttpBackendConfig {base_url: base_url.to_string(),api_key: Some("test-key".to_string()),forward_auth: false,extra_headers: BTreeMap::new(),extra_body: BTreeMap::new(),- max_retries: 0,+ max_retries,});let target_models = routes.iter()@@ -550,7 +574,12 @@ async fn stats_reset_returns_confirmation_and_clears_all_stats() -> TestResult {#[tokio::test]async fn metrics_exposes_switchyard_otel_instruments() -> TestResult {const MODEL: &str = "model/metrics-buffered";- let (_upstream, app) = test_app(&[(ROUTE_MODEL, &[MODEL])]).await?;+ let upstream = MockUpstream::start().await?;+ let app = build_switchyard_router(random_state_with_retries(+ &upstream.base_url,+ &[(ROUTE_MODEL, &[MODEL])],+ 1,+ )?);let before = send(&app, "GET", "/metrics", None).await?;assert_eq!(before.status, StatusCode::OK);@@ -588,7 +617,7 @@ async fn metrics_exposes_switchyard_otel_instruments() -> TestResult {"/v1/chat/completions",Some(json!({"model": ROUTE_MODEL,- "messages": [{"role": "user", "content": "hello"}]+ "messages": [{"role": "user", "content": "retry-once"}]})),).await?;@@ -596,6 +625,15 @@ async fn metrics_exposes_switchyard_otel_instruments() -> TestResult {let after = send(&app, "GET", "/metrics", None).await?;let metrics = after.text()?;+ assert_eq!(+ metric_delta(+ seeded,+ metrics,+ "switchyard_router_retry_recovered_total",+ &[]+ ),+ Some(1.0)+ );for expected in ["# TYPE switchyard_build_info gauge",&format!("switchyard_build_info{{version=\"{VERSION}\""),diff --git a/docs/internal/metrics_reference.md b/docs/internal/metrics_reference.mdindex 0d29bb59f..21ef27b11 100644--- a/docs/internal/metrics_reference.md+++ b/docs/internal/metrics_reference.md@@ -81,7 +81,7 @@ The `outcome` label takes exactly three values:|---|---|---|| `switchyard_client_responses_total{outcome}` | counter | HTTP responses returned to clients on the LLM-serving routes (`/v1/chat/completions`, `/v1/messages`, `/v1/responses`). The denominator for the **router-served** error rate. || `switchyard_upstream_attempts_total{outcome,code}` | counter | Individual upstream call attempts. One client request can produce N attempts via retry. The denominator for the **direct-to-endpoint** baseline error rate. The `code` label carries the raw upstream HTTP status for plotting the error-code distribution (see below). |-| `switchyard_router_retry_recovered_total` | counter | Reserved retry-recovery counter. The current server exports it as zero. |+| `switchyard_router_retry_recovered_total` | counter | Upstream operations that succeeded after at least one retry. |### The `code` label on `switchyard_upstream_attempts_total`