Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix: Random routing never routes to weight 0 (#700)
Graham Kingfe764190de9f
diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rsindex 7b9b86a67..8b80dc14f 100644--- a/crates/libsy/src/algorithms/rand.rs+++ b/crates/libsy/src/algorithms/rand.rs@@ -3,8 +3,8 @@//! Random routing as a stateless [`FallThrough`] composition.//!-//! [`RandomClassifier`] selects one target; [`FallThrough`] owns the common-//! processor/classifier/target-call orchestration.+//! [`RandomClassifier`] selects one target; [`FallThrough`] builds the candidate list.+//! [`Random`] removes zero-weight targets from that list.use std::sync::Arc;@@ -24,7 +24,7 @@ use switchyard_protocol::{Category, Request, Response};/// Stateless weighted classifier used by random fall-through routing.pub struct RandomClassifier {distribution: Option<WeightedIndex<f64>>,- weight_count: Option<usize>,+ weights: Option<Vec<f64>>,rng: Mutex<StdRng>,}@@ -40,8 +40,7 @@ impl RandomClassifier {/// Returns an error if explicit weights are negative or non-finite, or contain no/// positive value.pub fn new(weights: Option<Vec<f64>>, seed: Option<u64>) -> Result<Self> {- let weight_count = weights.as_ref().map(Vec::len);- let distribution = if let Some(weights) = weights {+ let distribution = if let Some(weights) = weights.as_ref() {if weights.iter().any(|weight| !weight.is_finite() || *weight < 0.0)@@ -65,7 +64,7 @@ impl RandomClassifier {};Ok(Self {distribution,- weight_count,+ weights,rng: Mutex::new(rng),})}@@ -93,7 +92,7 @@ whereif options.is_empty() {return Err(LibsyError::NoTargets);}- if let Some(weight_count) = self.weight_count+ if let Some(weight_count) = self.weights.as_ref().map(Vec::len)&& weight_count != options.len(){return Err(invalid_weights(format!(@@ -121,9 +120,10 @@ where}}-/// Random router implemented as a stateless fall-through composition.+/// Random router with fallbacks restricted to targets with positive weights.pub struct Random {inner: FallThrough<()>,+ classifier: Arc<RandomClassifier>,}impl Random {@@ -132,8 +132,8 @@ impl Random {let classifier = Arc::new(RandomClassifier::new(weights, seed)?);let inner = FallThrough::<()>::new().with_name("random")- .with_classifier(classifier);- Ok(Self { inner })+ .with_classifier(classifier.clone());+ Ok(Self { inner, classifier })}}@@ -148,7 +148,18 @@ impl Algorithm for Random {driver: Driver,request: Request,) -> Result<crate::RoutingOutcome> {- self.inner.execute(driver, request).await+ let mut outcome = self.inner.execute(driver.clone(), request).await?;+ if let Some(weights) = &self.classifier.weights {+ // FallThrough includes all runtime targets, but zero weights disable fallbacks too.+ for (model, weight) in driver.models_for(&Category::Any).iter().zip(weights) {+ if *weight == 0.0 {+ outcome+ .selected_model_ids+ .retain(|candidate| candidate != model);+ }+ }+ }+ Ok(outcome)}}diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex c8da27d8f..7097a7529 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -1012,8 +1012,7 @@ fn load_test_config(toml: &str) -> TestResult<ServerState> {Ok(load_server_state(config.path())?)}-/// A `random` route that selects `first` before any request-local fallback.-fn fallback_state(base_url: &str) -> TestResult<ServerState> {+fn weighted_random_state(base_url: &str, weights: [u32; 2]) -> TestResult<ServerState> {load_test_config(&format!(r#"schema_version = 1@@ -1037,7 +1036,8 @@ system_prompt = "strong answer prompt"id = "{ROUTE_MODEL}"type = "random"targets = ["first", "second"]-weights = [1, 0]+weights = {weights:?}+seed = 17"#,first = "model/weak",second = "model/strong",@@ -3264,12 +3264,66 @@ async fn routing_log_keeps_the_canonical_session_id_until_a_stream_drains() -> TOk(())}+#[tokio::test]+async fn random_zero_weight_target_is_not_advertised_or_called() -> TestResult {+ let upstream = MockUpstream::start().await?;+ let app = build_switchyard_router(weighted_random_state(&upstream.base_url, [1, 0])?);+ let response = send(+ &app,+ "POST",+ "/v1/decision",+ Some(json!({+ "input_format": "openai_chat",+ "request": {+ "model": ROUTE_MODEL,+ "messages": [{"role": "user", "content": "unavailable"}]+ }+ })),+ )+ .await?;+ assert_eq!(response.status, StatusCode::OK);+ let decision = response.json()?;+ assert_eq!(decision["selected"]["target"], "first");+ assert_eq!(decision["fallbacks"], json!([]));+ assert!(upstream.models().await.is_empty());++ for (path, body) in [+ (+ "/v1/chat/completions",+ json!({+ "model": ROUTE_MODEL,+ "messages": [{"role": "user", "content": "unavailable"}]+ }),+ ),+ (+ "/v1/messages",+ json!({+ "model": ROUTE_MODEL,+ "max_tokens": 16,+ "messages": [{"role": "user", "content": "unavailable"}]+ }),+ ),+ (+ "/v1/responses",+ json!({"model": ROUTE_MODEL, "input": "unavailable"}),+ ),+ ] {+ upstream.calls.lock().await.clear();+ let response = send(&app, "POST", path, Some(body)).await?;+ assert_eq!(response.status, StatusCode::SERVICE_UNAVAILABLE);+ assert_eq!(upstream.models().await, ["model/weak"]);+ }+ Ok(())+}+#[tokio::test]async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted() -> TestResult {let upstream = MockUpstream::start().await?;let temp_dir = tempfile::tempdir()?;let log_path = temp_dir.path().join("routing.jsonl");- let state = fallback_state(&upstream.base_url)?.with_routing_log(&log_path)?;+ // The fixed seed selects `first` for these requests while keeping `second` enabled.+ let state =+ weighted_random_state(&upstream.base_url, [1000, 1])?.with_routing_log(&log_path)?;let app = build_switchyard_router(state);let cases = [(