Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix: Populate the switchyard.router_retry_recovered metric (#474)

Graham King04adc1fe7095
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index d58508ca5..bf8ec1e54 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -261,6 +261,9 @@ impl TranslatingLlmClient {
span.record("outcome", "success");
span.record("status_code", response.status());
span.record("will_retry", false);
+ if attempt > 0 {
+ metrics::record_retry_recovered();
+ }
return Ok(response);
}
Err(failure) => {
diff --git a/crates/libsy-llm-client/src/metrics.rs b/crates/libsy-llm-client/src/metrics.rs
index 724e34086..bf22b421e 100644
--- a/crates/libsy-llm-client/src/metrics.rs
+++ b/crates/libsy-llm-client/src/metrics.rs
@@ -92,6 +92,14 @@ pub(crate) fn record_upstream_attempt(status: Option<u16>) {
);
}
+/// Records an upstream operation that succeeded after at least one retry.
+pub(crate) fn record_retry_recovered() {
+ global::meter("switchyard")
+ .u64_counter("switchyard.router_retry_recovered")
+ .build()
+ .add(1, &[]);
+}
+
/// Records the time needed to produce the routing outcome, including classifier calls,
/// target resolution, request rewrites, and decision publishing.
pub(crate) fn record_routing_overhead(algorithm: &str, overhead: Duration) {
diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md
index cf107fecc..65c5083b6 100644
--- a/crates/switchyard-server/README.md
+++ b/crates/switchyard-server/README.md
@@ -171,7 +171,7 @@ Routed-call compatibility metrics are:
| `switchyard_classifier_fail_open_total` | counter | `judge_model`, `reason` | Judge failures that made a classifier route without a verdict |
| `switchyard_client_responses_total` | counter | `outcome` | Final LLM-route responses |
| `switchyard_upstream_attempts_total` | counter | `outcome`, `code` | Actual upstream HTTP attempts |
-| `switchyard_router_retry_recovered_total` | counter | none | Retry recoveries (currently always zero) |
+| `switchyard_router_retry_recovered_total` | counter | none | Upstream operations recovered by a retry |
`switchyard_classifier_fail_open_total` counts requests that still reached a target after the
judge call failed. `judge_model` names the configured judge target, and `reason` is one of eight
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index fc15ba3f8..0a97f90e3 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -113,6 +113,22 @@ async fn upstream_chat(
let model = body["model"].as_str().unwrap_or("unknown").to_string();
let prompt = body["messages"][0]["content"].as_str().unwrap_or("");
+ if prompt == "retry-once"
+ && calls
+ .lock()
+ .await
+ .iter()
+ .filter(|call| call["messages"][0]["content"] == "retry-once")
+ .count()
+ == 1
+ {
+ return (
+ StatusCode::SERVICE_UNAVAILABLE,
+ [("retry-after", "0")],
+ Json(json!({"error": {"message": "upstream is temporarily unavailable"}})),
+ )
+ .into_response();
+ }
if (model == "model/weak" && prompt == "unavailable") || prompt == "all-unavailable" {
return (
StatusCode::SERVICE_UNAVAILABLE,
@@ -365,13 +381,21 @@ async fn upstream_count_tokens(
}
fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult<ServerState> {
+ random_state_with_retries(base_url, routes, 0)
+}
+
+fn random_state_with_retries(
+ base_url: &str,
+ routes: &[(&str, &[&str])],
+ max_retries: u32,
+) -> TestResult<ServerState> {
let backend = Backend::OpenAiChat(HttpBackendConfig {
base_url: base_url.to_string(),
api_key: Some("test-key".to_string()),
forward_auth: false,
extra_headers: BTreeMap::new(),
extra_body: BTreeMap::new(),
- max_retries: 0,
+ max_retries,
});
let target_models = routes
.iter()
@@ -550,7 +574,12 @@ async fn stats_reset_returns_confirmation_and_clears_all_stats() -> TestResult {
#[tokio::test]
async fn metrics_exposes_switchyard_otel_instruments() -> TestResult {
const MODEL: &str = "model/metrics-buffered";
- let (_upstream, app) = test_app(&[(ROUTE_MODEL, &[MODEL])]).await?;
+ let upstream = MockUpstream::start().await?;
+ let app = build_switchyard_router(random_state_with_retries(
+ &upstream.base_url,
+ &[(ROUTE_MODEL, &[MODEL])],
+ 1,
+ )?);
let before = send(&app, "GET", "/metrics", None).await?;
assert_eq!(before.status, StatusCode::OK);
@@ -588,7 +617,7 @@ async fn metrics_exposes_switchyard_otel_instruments() -> TestResult {
"/v1/chat/completions",
Some(json!({
"model": ROUTE_MODEL,
- "messages": [{"role": "user", "content": "hello"}]
+ "messages": [{"role": "user", "content": "retry-once"}]
})),
)
.await?;
@@ -596,6 +625,15 @@ async fn metrics_exposes_switchyard_otel_instruments() -> TestResult {
let after = send(&app, "GET", "/metrics", None).await?;
let metrics = after.text()?;
+ assert_eq!(
+ metric_delta(
+ seeded,
+ metrics,
+ "switchyard_router_retry_recovered_total",
+ &[]
+ ),
+ Some(1.0)
+ );
for expected in [
"# TYPE switchyard_build_info gauge",
&format!("switchyard_build_info{{version=\"{VERSION}\""),
diff --git a/docs/internal/metrics_reference.md b/docs/internal/metrics_reference.md
index 0d29bb59f..21ef27b11 100644
--- a/docs/internal/metrics_reference.md
+++ b/docs/internal/metrics_reference.md
@@ -81,7 +81,7 @@ The `outcome` label takes exactly three values:
|---|---|---|
| `switchyard_client_responses_total{outcome}` | counter | HTTP responses returned to clients on the LLM-serving routes (`/v1/chat/completions`, `/v1/messages`, `/v1/responses`). The denominator for the **router-served** error rate. |
| `switchyard_upstream_attempts_total{outcome,code}` | counter | Individual upstream call attempts. One client request can produce N attempts via retry. The denominator for the **direct-to-endpoint** baseline error rate. The `code` label carries the raw upstream HTTP status for plotting the error-code distribution (see below). |
-| `switchyard_router_retry_recovered_total` | counter | Reserved retry-recovery counter. The current server exports it as zero. |
+| `switchyard_router_retry_recovered_total` | counter | Upstream operations that succeeded after at least one retry. |
### The `code` label on `switchyard_upstream_attempts_total`