Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix: Random routing never routes to weight 0 (#700)

Graham Kingfe764190de9f
diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rs
index 7b9b86a67..8b80dc14f 100644
--- a/crates/libsy/src/algorithms/rand.rs
+++ b/crates/libsy/src/algorithms/rand.rs
@@ -3,8 +3,8 @@
//! Random routing as a stateless [`FallThrough`] composition.
//!
-//! [`RandomClassifier`] selects one target; [`FallThrough`] owns the common
-//! processor/classifier/target-call orchestration.
+//! [`RandomClassifier`] selects one target; [`FallThrough`] builds the candidate list.
+//! [`Random`] removes zero-weight targets from that list.
use std::sync::Arc;
@@ -24,7 +24,7 @@ use switchyard_protocol::{Category, Request, Response};
/// Stateless weighted classifier used by random fall-through routing.
pub struct RandomClassifier {
distribution: Option<WeightedIndex<f64>>,
- weight_count: Option<usize>,
+ weights: Option<Vec<f64>>,
rng: Mutex<StdRng>,
}
@@ -40,8 +40,7 @@ impl RandomClassifier {
/// Returns an error if explicit weights are negative or non-finite, or contain no
/// positive value.
pub fn new(weights: Option<Vec<f64>>, seed: Option<u64>) -> Result<Self> {
- let weight_count = weights.as_ref().map(Vec::len);
- let distribution = if let Some(weights) = weights {
+ let distribution = if let Some(weights) = weights.as_ref() {
if weights
.iter()
.any(|weight| !weight.is_finite() || *weight < 0.0)
@@ -65,7 +64,7 @@ impl RandomClassifier {
};
Ok(Self {
distribution,
- weight_count,
+ weights,
rng: Mutex::new(rng),
})
}
@@ -93,7 +92,7 @@ where
if options.is_empty() {
return Err(LibsyError::NoTargets);
}
- if let Some(weight_count) = self.weight_count
+ if let Some(weight_count) = self.weights.as_ref().map(Vec::len)
&& weight_count != options.len()
{
return Err(invalid_weights(format!(
@@ -121,9 +120,10 @@ where
}
}
-/// Random router implemented as a stateless fall-through composition.
+/// Random router with fallbacks restricted to targets with positive weights.
pub struct Random {
inner: FallThrough<()>,
+ classifier: Arc<RandomClassifier>,
}
impl Random {
@@ -132,8 +132,8 @@ impl Random {
let classifier = Arc::new(RandomClassifier::new(weights, seed)?);
let inner = FallThrough::<()>::new()
.with_name("random")
- .with_classifier(classifier);
- Ok(Self { inner })
+ .with_classifier(classifier.clone());
+ Ok(Self { inner, classifier })
}
}
@@ -148,7 +148,18 @@ impl Algorithm for Random {
driver: Driver,
request: Request,
) -> Result<crate::RoutingOutcome> {
- self.inner.execute(driver, request).await
+ let mut outcome = self.inner.execute(driver.clone(), request).await?;
+ if let Some(weights) = &self.classifier.weights {
+ // FallThrough includes all runtime targets, but zero weights disable fallbacks too.
+ for (model, weight) in driver.models_for(&Category::Any).iter().zip(weights) {
+ if *weight == 0.0 {
+ outcome
+ .selected_model_ids
+ .retain(|candidate| candidate != model);
+ }
+ }
+ }
+ Ok(outcome)
}
}
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index c8da27d8f..7097a7529 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -1012,8 +1012,7 @@ fn load_test_config(toml: &str) -> TestResult<ServerState> {
Ok(load_server_state(config.path())?)
}
-/// A `random` route that selects `first` before any request-local fallback.
-fn fallback_state(base_url: &str) -> TestResult<ServerState> {
+fn weighted_random_state(base_url: &str, weights: [u32; 2]) -> TestResult<ServerState> {
load_test_config(&format!(
r#"
schema_version = 1
@@ -1037,7 +1036,8 @@ system_prompt = "strong answer prompt"
id = "{ROUTE_MODEL}"
type = "random"
targets = ["first", "second"]
-weights = [1, 0]
+weights = {weights:?}
+seed = 17
"#,
first = "model/weak",
second = "model/strong",
@@ -3264,12 +3264,66 @@ async fn routing_log_keeps_the_canonical_session_id_until_a_stream_drains() -> T
Ok(())
}
+#[tokio::test]
+async fn random_zero_weight_target_is_not_advertised_or_called() -> TestResult {
+ let upstream = MockUpstream::start().await?;
+ let app = build_switchyard_router(weighted_random_state(&upstream.base_url, [1, 0])?);
+ let response = send(
+ &app,
+ "POST",
+ "/v1/decision",
+ Some(json!({
+ "input_format": "openai_chat",
+ "request": {
+ "model": ROUTE_MODEL,
+ "messages": [{"role": "user", "content": "unavailable"}]
+ }
+ })),
+ )
+ .await?;
+ assert_eq!(response.status, StatusCode::OK);
+ let decision = response.json()?;
+ assert_eq!(decision["selected"]["target"], "first");
+ assert_eq!(decision["fallbacks"], json!([]));
+ assert!(upstream.models().await.is_empty());
+
+ for (path, body) in [
+ (
+ "/v1/chat/completions",
+ json!({
+ "model": ROUTE_MODEL,
+ "messages": [{"role": "user", "content": "unavailable"}]
+ }),
+ ),
+ (
+ "/v1/messages",
+ json!({
+ "model": ROUTE_MODEL,
+ "max_tokens": 16,
+ "messages": [{"role": "user", "content": "unavailable"}]
+ }),
+ ),
+ (
+ "/v1/responses",
+ json!({"model": ROUTE_MODEL, "input": "unavailable"}),
+ ),
+ ] {
+ upstream.calls.lock().await.clear();
+ let response = send(&app, "POST", path, Some(body)).await?;
+ assert_eq!(response.status, StatusCode::SERVICE_UNAVAILABLE);
+ assert_eq!(upstream.models().await, ["model/weak"]);
+ }
+ Ok(())
+}
+
#[tokio::test]
async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted() -> TestResult {
let upstream = MockUpstream::start().await?;
let temp_dir = tempfile::tempdir()?;
let log_path = temp_dir.path().join("routing.jsonl");
- let state = fallback_state(&upstream.base_url)?.with_routing_log(&log_path)?;
+ // The fixed seed selects `first` for these requests while keeping `second` enabled.
+ let state =
+ weighted_random_state(&upstream.base_url, [1000, 1])?.with_routing_log(&log_path)?;
let app = build_switchyard_router(state);
let cases = [
(