Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(libsy): add weighted reproducible random routing (#155)

nachiketb-nvidia724a9f9bfa7f
diff --git a/crates/libsy/README.md b/crates/libsy/README.md
index f426adb89..fd5f001f2 100644
--- a/crates/libsy/README.md
+++ b/crates/libsy/README.md
@@ -300,12 +300,12 @@ instrumentation is a no-op.
## Explore
-The core crate includes uniform random routing and naive LLM classifier. Runnable
+The core crate includes weighted random routing and naive LLM classifier. Runnable
agents live in [`examples`](examples/) folder.
**Reference algorithms** — implementations to read and route with:
-- [`Random`](src/algorithms/rand.rs) — uniform random over the set
+- [`Random`](src/algorithms/rand.rs) — uniform or weighted random over the set
(one call).
- [`LlmClassifier`](src/algorithms/llm_class.rs) — classify, then route
strong/weak; fail open to strong.
@@ -326,4 +326,4 @@ agents live in [`examples`](examples/) folder.
- **`Signals` events** — `process_signals` / `Signals` exist but carry nothing yet.
- **`Context` fields** — carries the algorithm telemetry label today; correlation ids, budgets, and deadlines still to come.
-- **Config-driven construction**, **weighted random**.
+- **Config-driven construction**.
diff --git a/crates/libsy/examples/research_agent.rs b/crates/libsy/examples/research_agent.rs
index 892e162e3..6ba648515 100644
--- a/crates/libsy/examples/research_agent.rs
+++ b/crates/libsy/examples/research_agent.rs
@@ -96,8 +96,13 @@ impl ResearchAgent {
async fn main() -> Result<()> {
// Configure routing once: an LLM classifier over three named targets. Swapping
// in `Random` needs no change to the agent.
- let algo: Arc<dyn Algorithm> =
- Arc::new(LlmClassifier::new(CLASSIFIER, STRONG, WEAK, 0.5, targets()));
+ let algo: Arc<dyn Algorithm> = Arc::new(LlmClassifier::new(
+ CLASSIFIER,
+ STRONG,
+ WEAK,
+ 0.5,
+ targets(),
+ )?);
let agent = ResearchAgent { algo };
println!("{}", agent.run("what is switchyard?").await?);
diff --git a/crates/libsy/examples/research_agent_core.rs b/crates/libsy/examples/research_agent_core.rs
index 6567f5206..b653f4e6e 100644
--- a/crates/libsy/examples/research_agent_core.rs
+++ b/crates/libsy/examples/research_agent_core.rs
@@ -102,8 +102,13 @@ fn print_decision(decision: &dyn Decision) {
#[tokio::main(flavor = "current_thread")]
async fn main() -> Result<()> {
- let algo: Arc<dyn Algorithm> =
- Arc::new(LlmClassifier::new(CLASSIFIER, STRONG, WEAK, 0.5, targets()));
+ let algo: Arc<dyn Algorithm> = Arc::new(LlmClassifier::new(
+ CLASSIFIER,
+ STRONG,
+ WEAK,
+ 0.5,
+ targets(),
+ )?);
let mut agent = ResearchAgent { algo };
println!("{}", agent.run("what is switchyard?").await?);
diff --git a/crates/libsy/examples/streaming_agent.rs b/crates/libsy/examples/streaming_agent.rs
index a2b4f594c..b8140a84e 100644
--- a/crates/libsy/examples/streaming_agent.rs
+++ b/crates/libsy/examples/streaming_agent.rs
@@ -55,7 +55,7 @@ async fn main() -> Result<()> {
semantic_name: "stream/model".to_string(),
llm_client: None,
}]);
- let algo: Arc<dyn Algorithm> = Arc::new(Random::new(targets));
+ let algo: Arc<dyn Algorithm> = Arc::new(Random::new(targets, None, None)?);
let request = Request {
llm_request: text_request(Some("auto".to_string()), "tell me about switchyard"),
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index c7c6eabfc..e8d65fd56 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -14,7 +14,9 @@ use std::sync::Arc;
use async_trait::async_trait;
-use crate::{Algorithm, Context, Decision, Driver, LlmTargetSet, Request, Response, Result};
+use crate::{
+ Algorithm, Context, Decision, Driver, LibsyError, LlmTargetSet, Request, Response, Result,
+};
use switchyard_protocol::{completion_text, LlmRequest, Message, Role};
/// The system prompt tells the classifier what to do
@@ -76,20 +78,36 @@ impl LlmClassifier {
/// and weak models to route to, the score `threshold` at or above which the
/// strong model is chosen, and the `target_set` to route among. That set must
/// contain targets named `classifier_model`, `strong_model`, and `weak_model`.
+ ///
+ /// # Errors
+ ///
+ /// Returns an error when `threshold` is outside `0.0..=1.0` or when any
+ /// required target name is absent from `target_set`.
pub fn new(
classifier_model: impl Into<String>,
strong_model: impl Into<String>,
weak_model: impl Into<String>,
threshold: f64,
target_set: LlmTargetSet,
- ) -> Self {
- Self {
- classifier_model: classifier_model.into(),
- strong_model: strong_model.into(),
- weak_model: weak_model.into(),
+ ) -> Result<Self> {
+ if !threshold.is_finite() || !(0.0..=1.0).contains(&threshold) {
+ return Err(LibsyError::AlgorithmError {
+ message: "llm_classifier threshold must be between 0 and 1".to_string(),
+ });
+ }
+ let classifier_model = classifier_model.into();
+ let strong_model = strong_model.into();
+ let weak_model = weak_model.into();
+ target_set.get_target(&classifier_model)?;
+ target_set.get_target(&strong_model)?;
+ target_set.get_target(&weak_model)?;
+ Ok(Self {
+ classifier_model,
+ strong_model,
+ weak_model,
threshold,
target_set,
- }
+ })
}
}
@@ -254,7 +272,7 @@ mod tests {
}
/// Build a classifier algo whose three targets share a scoring client.
- fn algo(threshold: f64, score: &str) -> (LlmClassifier, Arc<Mutex<Vec<Request>>>) {
+ fn algo(threshold: f64, score: &str) -> Result<(LlmClassifier, Arc<Mutex<Vec<Request>>>)> {
let seen = Arc::new(Mutex::new(Vec::new()));
let client = Arc::new(ScoringClient {
classifier_model: "router/classifier".to_string(),
@@ -270,14 +288,14 @@ mod tests {
target("frontier/model"),
target("cheap/model"),
]);
- let algo = LlmClassifier {
- classifier_model: "router/classifier".to_string(),
- strong_model: "frontier/model".to_string(),
- weak_model: "cheap/model".to_string(),
+ let algo = LlmClassifier::new(
+ "router/classifier",
+ "frontier/model",
+ "cheap/model",
threshold,
target_set,
- };
- (algo, seen)
+ )?;
+ Ok((algo, seen))
}
fn request(prompt: &str) -> Request {
@@ -307,7 +325,7 @@ mod tests {
#[tokio::test]
async fn score_at_or_above_threshold_routes_strong() -> Result<()> {
- let (algo, _) = algo(0.5, "0.9");
+ let (algo, _) = algo(0.5, "0.9")?;
let (trace, response) = orch(algo)
.run(Context::default(), request("solve this proof"))
.await?;
@@ -330,7 +348,7 @@ mod tests {
#[tokio::test]
async fn score_below_threshold_routes_weak() -> Result<()> {
- let (algo, _) = algo(0.5, "0.2");
+ let (algo, _) = algo(0.5, "0.2")?;
let (trace, response) = orch(algo)
.run(Context::default(), request("say hello"))
.await?;
@@ -350,7 +368,7 @@ mod tests {
#[tokio::test]
async fn score_exactly_at_threshold_routes_strong() -> Result<()> {
- let (algo, _) = algo(0.5, "0.5");
+ let (algo, _) = algo(0.5, "0.5")?;
let (_, response) = orch(algo)
.run(Context::default(), request("borderline"))
.await?;
@@ -367,7 +385,7 @@ mod tests {
#[tokio::test]
async fn unparseable_score_defaults_to_strong() -> Result<()> {
- let (algo, _) = algo(0.5, "not-a-number");
+ let (algo, _) = algo(0.5, "not-a-number")?;
let (trace, response) = orch(algo).run(Context::default(), request("hi")).await?;
assert_eq!(
response
@@ -382,4 +400,13 @@ mod tests {
assert_eq!(routed.score, None);
Ok(())
}
+
+ #[test]
+ fn rejects_invalid_threshold() {
+ let error = algo(1.5, "0.5")
+ .err()
+ .map(|error| error.to_string())
+ .unwrap_or_default();
+ assert!(error.contains("threshold must be between 0 and 1"));
+ }
}
diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rs
index 8399b5b86..88de59e9d 100644
--- a/crates/libsy/src/algorithms/rand.rs
+++ b/crates/libsy/src/algorithms/rand.rs
@@ -3,18 +3,20 @@
//! Random router built on the [`Algorithm`] interfaces.
//!
-//! Selects one target from the set uniformly at random and calls it. This is the
-//! simplest possible routing algorithm and the reference for the single-call
-//! shape: one `driver.call_llm_target` inside `create_run_task`. Weighted selection
-//! can be layered on later; the set defines the candidates.
+//! Selects one target from the set at random and calls it. Selection is uniform
+//! by default and can use relative weights and a reproducible seed.
-use std::sync::Arc;
+use std::collections::BTreeSet;
+use std::sync::{Arc, Mutex};
use async_trait::async_trait;
-use rand::seq::SliceRandom;
+use rand::distributions::{Distribution, WeightedIndex};
+use rand::rngs::StdRng;
+use rand::SeedableRng;
use crate::{
- Algorithm, Context, Decision, Driver, LibsyError, LlmTargetSet, Request, Response, Result,
+ Algorithm, Context, Decision, Driver, LibsyError, LlmTarget, LlmTargetSet, Request, Response,
+ Result,
};
/// Decision produced by [`Random`]: which target was chosen and why.
@@ -39,18 +41,91 @@ impl Decision for RandomDecision {
}
}
-/// Uniform random router over a target set.
+/// Random router over a target set.
pub struct Random {
target_set: LlmTargetSet,
+ distribution: WeightedIndex<f64>,
+ rng: Mutex<StdRng>,
}
impl Random {
/// Creates a router over `target_set`.
///
- /// Wrap it in an [`Arc`] and drive it with [`Algorithm::run`] or
- /// [`Algorithm::run_stream`].
- pub fn new(target_set: LlmTargetSet) -> Self {
- Self { target_set }
+ /// Missing weights default to one per target. Explicit weights are relative,
+ /// follow target order, and need not sum to one. Zero disables a target.
+ /// Missing `seed` uses entropy-backed randomness.
+ ///
+ /// # Errors
+ ///
+ /// Returns an error when targets are empty or duplicated, or when explicit
+ /// weights have the wrong length, are negative or non-finite, or contain no
+ /// positive value.
+ pub fn new(
+ target_set: LlmTargetSet,
+ weights: Option<Vec<f64>>,
+ seed: Option<u64>,
+ ) -> Result<Self> {
+ let target_count = target_set.targets().len();
+ if target_count == 0 {
+ return Err(LibsyError::NoTargets);
+ }
+ let unique_targets = target_set
+ .targets()
+ .iter()
+ .map(|target| target.semantic_name.as_str())
+ .collect::<BTreeSet<_>>();
+ if unique_targets.len() != target_count {
+ return Err(LibsyError::AlgorithmError {
+ message: "random targets must be unique".to_string(),
+ });
+ }
+
+ let weights = weights.unwrap_or_else(|| vec![1.0; target_count]);
+ if weights.len() != target_count {
+ return Err(invalid_weights(format!(
+ "expected {target_count} weights, got {}",
+ weights.len()
+ )));
+ }
+ if weights
+ .iter()
+ .any(|weight| !weight.is_finite() || *weight < 0.0)
+ {
+ return Err(invalid_weights(
+ "weights must be finite and nonnegative".to_string(),
+ ));
+ }
+ if !weights.iter().any(|weight| *weight > 0.0) {
+ return Err(invalid_weights(
+ "at least one weight must be positive".to_string(),
+ ));
+ }
+ let distribution =
+ WeightedIndex::new(weights).map_err(|error| invalid_weights(error.to_string()))?;
+ let rng = match seed {
+ Some(seed) => StdRng::seed_from_u64(seed),
+ None => StdRng::from_entropy(),
+ };
+ Ok(Self {
+ target_set,
+ distribution,
+ rng: Mutex::new(rng),
+ })
+ }
+
+ fn select_target(&self) -> Result<LlmTarget> {
+ let targets = self.target_set.targets();
+ let mut rng = self.rng.lock().map_err(|_| LibsyError::AlgorithmError {
+ message: "random number generator lock was poisoned".to_string(),
+ })?;
+ let index = self.distribution.sample(&mut *rng);
+ Ok(targets[index].clone())
+ }
+}
+
+fn invalid_weights(message: String) -> LibsyError {
+ LibsyError::AlgorithmError {
+ message: format!("invalid random weights: {message}"),
}
}
@@ -66,15 +141,7 @@ impl Algorithm for Random {
driver: Driver,
request: Request,
) -> Result<Response> {
- // Scope the non-Send ThreadRng before the await so the future remains Send.
- let target = {
- let mut rng = rand::thread_rng();
- self.target_set
- .targets()
- .choose(&mut rng)
- .ok_or(LibsyError::NoTargets)?
- .clone()
- };
+ let target = self.select_target()?;
let selected = target.semantic_name.clone();
let decision: Arc<dyn Decision> = Arc::new(RandomDecision {
@@ -125,7 +192,7 @@ mod tests {
}
/// Builds a random router whose targets all share an echo client.
- fn algorithm(names: &[&str]) -> Random {
+ fn algorithm(names: &[&str], weights: Option<Vec<f64>>, seed: Option<u64>) -> Result<Random> {
let targets = names
.iter()
.map(|name| LlmTarget {
@@ -133,16 +200,31 @@ mod tests {
llm_client: Some(Arc::new(EchoClient)),
})
.collect();
- Random::new(LlmTargetSet::new(targets))
+ Random::new(LlmTargetSet::new(targets), weights, seed)
}
- fn shared_algorithm(names: &[&str]) -> Arc<dyn Algorithm> {
- Arc::new(algorithm(names))
+ fn shared_algorithm(names: &[&str]) -> Result<Arc<dyn Algorithm>> {
+ Ok(Arc::new(algorithm(names, None, None)?))
+ }
+
+ async fn selected_models(algorithm: Arc<dyn Algorithm>, count: usize) -> Result<Vec<String>> {
+ let mut selected = Vec::with_capacity(count);
+ for _ in 0..count {
+ let (_, response) = algorithm.clone().run(Context::default(), request()).await?;
+ selected.push(
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ );
+ }
+ Ok(selected)
}
#[tokio::test]
async fn single_target_is_always_selected_and_called() -> Result<()> {
- let algorithm = shared_algorithm(&["only/model"]);
+ let algorithm = shared_algorithm(&["only/model"])?;
let (trace, response) = algorithm.run(Context::default(), request()).await?;
assert_eq!(
@@ -161,7 +243,7 @@ mod tests {
#[tokio::test]
async fn selected_target_is_in_the_set_and_matches_the_trace() -> Result<()> {
let names = ["a/model", "b/model", "c/model"];
- let algorithm = shared_algorithm(&names);
+ let algorithm = shared_algorithm(&names)?;
for _ in 0..50 {
let (trace, response) = algorithm.clone().run(Context::default(), request()).await?;
@@ -181,7 +263,7 @@ mod tests {
#[tokio::test]
async fn selection_covers_all_targets_over_many_runs() -> Result<()> {
- let algorithm = shared_algorithm(&["a/model", "b/model"]);
+ let algorithm = shared_algorithm(&["a/model", "b/model"])?;
let mut seen = HashSet::new();
for _ in 0..100 {
@@ -205,15 +287,66 @@ mod tests {
}
#[tokio::test]
- async fn empty_target_set_errors() {
- let algorithm = shared_algorithm(&[]);
- let error = algorithm.run(Context::default(), request()).await.err();
+ async fn weighted_seeded_selection_is_reproducible() -> Result<()> {
+ let first: Arc<dyn Algorithm> = Arc::new(algorithm(
+ &["a/model", "b/model"],
+ Some(vec![1.0, 3.0]),
+ Some(42),
+ )?);
+ let second: Arc<dyn Algorithm> = Arc::new(algorithm(
+ &["a/model", "b/model"],
+ Some(vec![1.0, 3.0]),
+ Some(42),
+ )?);
+
+ let first_selections = selected_models(first, 1_000).await?;
+ let second_selections = selected_models(second, 1_000).await?;
+ assert_eq!(first_selections, second_selections);
+
+ let second_count = first_selections
+ .iter()
+ .filter(|model| model.as_str() == "b/model")
+ .count();
+ assert!(
+ (700..=800).contains(&second_count),
+ "expected a roughly 25/75 split, selected b/model {second_count} times"
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn rejects_invalid_weights() {
+ let cases = [
+ (vec![1.0], "expected 2 weights"),
+ (vec![1.0, -1.0], "finite and nonnegative"),
+ (vec![0.0, 0.0], "at least one weight must be positive"),
+ (vec![1.0, f64::INFINITY], "finite and nonnegative"),
+ ];
+
+ for (weights, expected) in cases {
+ let error = algorithm(&["a/model", "b/model"], Some(weights), None)
+ .err()
+ .map(|error| error.to_string())
+ .unwrap_or_default();
+ assert!(error.contains(expected), "unexpected error: {error}");
+ }
+ }
+
+ #[test]
+ fn rejects_invalid_targets() {
+ let error = algorithm(&[], None, None).err();
assert!(matches!(error, Some(LibsyError::NoTargets)));
+
+ let error = algorithm(&["same/model", "same/model"], None, None)
+ .err()
+ .map(|error| error.to_string())
+ .unwrap_or_default();
+ assert!(error.contains("random targets must be unique"));
}
#[tokio::test]
async fn process_signals_is_a_noop() -> Result<()> {
- Arc::new(algorithm(&["only/model"]))
+ Arc::new(algorithm(&["only/model"], None, None)?)
.process_signals(Signals {})
.await?;
Ok(())
@@ -221,7 +354,7 @@ mod tests {
#[tokio::test]
async fn decision_is_inspectable_and_downcasts() -> Result<()> {
- let algorithm = shared_algorithm(&["only/model"]);
+ let algorithm = shared_algorithm(&["only/model"])?;
let (trace, _) = algorithm.run(Context::default(), request()).await?;
let decision = &trace[0];
diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs
index 947436e11..8270ff8b8 100644
--- a/crates/libsy/src/lib.rs
+++ b/crates/libsy/src/lib.rs
@@ -72,7 +72,7 @@
//!
//! Concrete algorithms live in [`algorithms`]:
//!
-//! [`algorithms::Random`] provides uniform random routing.
+//! [`algorithms::Random`] provides uniform or weighted random routing.
//!
//! [`algorithms::LlmClassifier`] classifies with one model, then routes to a strong/weak
//! model depending on the classifier's choice.
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs
index 68984c9eb..934d3b859 100644
--- a/crates/switchyard-py/src/libsy_bindings.rs
+++ b/crates/switchyard-py/src/libsy_bindings.rs
@@ -11,8 +11,8 @@ use pyo3::prelude::*;
use serde_json::{json, Value};
use switchyard_libsy::algorithms::{Noop, Random};
use switchyard_libsy::{
- AggLlmResponse, Algorithm, Context, Decision, LlmClientError, LlmResponse, LlmTarget,
- LlmTargetSet, Metadata, Request, Response, RoutedLlmClient,
+ AggLlmResponse, Algorithm, Context, Decision, LibsyError as RustLibsyError, LlmClientError,
+ LlmResponse, LlmTarget, LlmTargetSet, Metadata, Request, Response, RoutedLlmClient,
};
use crate::errors::py_libsy_error;
@@ -160,19 +160,27 @@ fn noop_algorithm() -> PyAlgorithm {
PyAlgorithm::new(Arc::new(Noop {}))
}
-/// Construct uniform random routing over targets with Python clients.
+/// Construct random routing over targets with optional relative weights and seed.
#[pyfunction(name = "random")]
-fn random_algorithm(py: Python<'_>, targets: Vec<Py<PyLlmTarget>>) -> PyResult<PyAlgorithm> {
- if targets.is_empty() {
- return Err(PyValueError::new_err("random requires at least one target"));
- }
+#[pyo3(signature = (targets, *, weights=None, seed=None))]
+fn random_algorithm(
+ py: Python<'_>,
+ targets: Vec<Py<PyLlmTarget>>,
+ weights: Option<Vec<f64>>,
+ seed: Option<u64>,
+) -> PyResult<PyAlgorithm> {
let targets = targets
.iter()
.map(|target| Ok(target.bind(py).try_borrow()?.clone_core(py)))
.collect::<PyResult<Vec<_>>>()?;
- Ok(PyAlgorithm::new(Arc::new(Random::new(LlmTargetSet::new(
- targets,
- )))))
+ let algorithm =
+ Random::new(LlmTargetSet::new(targets), weights, seed).map_err(|error| match error {
+ RustLibsyError::NoTargets => {
+ PyValueError::new_err("random requires at least one target")
+ }
+ other => PyValueError::new_err(other.to_string()),
+ })?;
+ Ok(PyAlgorithm::new(Arc::new(algorithm)))
}
fn other_python_error(error: PyErr) -> LlmClientError {
diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md
index 56838e013..de5f16f41 100644
--- a/crates/switchyard-server/README.md
+++ b/crates/switchyard-server/README.md
@@ -25,6 +25,8 @@ llm_client = "example"
id = "switchyard/general"
type = "random"
targets = ["model_a", "model_b"]
+weights = [1, 3]
+seed = 42
[routes.classified]
id = "switchyard/classified"
@@ -49,4 +51,7 @@ Each target references an entry under `llm_clients`. All configured clients use
`api_key_env` value names an environment variable; the TOML never contains the secret itself. If
omitted, the client sends no authentication.
+Random-route `weights` are relative, follow target order, and do not need to sum to one. Omit them
+for equal weighting. The optional `seed` reproduces the selection sequence for the same call order.
+
See [CONFIGURATION.md](CONFIGURATION.md) to add an LLM client, target, or algorithm.
diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs
index ded8d4695..cd8f62b98 100644
--- a/crates/switchyard-server/src/config.rs
+++ b/crates/switchyard-server/src/config.rs
@@ -3,7 +3,7 @@
//! Typed TOML configuration and explicit construction for the Rust server.
-use std::collections::{BTreeMap, BTreeSet};
+use std::collections::BTreeMap;
use std::fs;
use std::path::Path;
use std::sync::Arc;
@@ -168,6 +168,8 @@ enum RouteConfig {
Random {
id: String,
targets: Vec<String>,
+ weights: Option<Vec<f64>>,
+ seed: Option<u64>,
},
LlmClassifier {
id: String,
@@ -234,23 +236,17 @@ fn build_algorithm(
) -> ServerResult<Arc<dyn Algorithm>> {
match config {
RouteConfig::Noop { .. } => Ok(Arc::new(Noop {})),
- RouteConfig::Random { targets: names, .. } => {
- if names.is_empty() {
- return Err(ServerError::new(format!(
- "random route {route_name} requires at least one target"
- )));
- }
- let unique = names.iter().collect::<BTreeSet<_>>();
- if unique.len() != names.len() {
- return Err(ServerError::new(format!(
- "random route {route_name} contains duplicate targets"
- )));
- }
- Ok(Arc::new(Random::new(resolve_targets(
- route_name,
- names.iter().map(String::as_str),
- targets,
- )?)))
+ RouteConfig::Random {
+ targets: names,
+ weights,
+ seed,
+ ..
+ } => {
+ let target_set =
+ resolve_targets(route_name, names.iter().map(String::as_str), targets)?;
+ let algorithm = Random::new(target_set, weights.clone(), *seed)
+ .map_err(|error| ServerError::new(format!("random route {route_name}: {error}")))?;
+ Ok(Arc::new(algorithm))
}
RouteConfig::LlmClassifier {
classifier_target,
@@ -259,23 +255,22 @@ fn build_algorithm(
threshold,
..
} => {
- if !threshold.is_finite() || !(0.0..=1.0).contains(threshold) {
- return Err(ServerError::new(format!(
- "llm_classifier route {route_name} threshold must be between 0 and 1"
- )));
- }
let classifier = resolve_target(route_name, classifier_target, targets)?;
let strong = resolve_target(route_name, strong_target, targets)?;
let weak = resolve_target(route_name, weak_target, targets)?;
let target_set =
LlmTargetSet::new(vec![classifier.clone(), strong.clone(), weak.clone()]);
- Ok(Arc::new(LlmClassifier::new(
+ let algorithm = LlmClassifier::new(
classifier.semantic_name,
strong.semantic_name,
weak.semantic_name,
*threshold,
target_set,
- )))
+ )
+ .map_err(|error| {
+ ServerError::new(format!("llm_classifier route {route_name}: {error}"))
+ })?;
+ Ok(Arc::new(algorithm))
}
}
}
@@ -412,7 +407,21 @@ threshold = 0.5
"targets = [\"strong\", \"weak\"]",
"targets = [\"strong\", \"strong\"]",
),
- "duplicate targets",
+ "random targets must be unique",
+ ),
+ (
+ VALID_CONFIG.replace(
+ "targets = [\"strong\", \"weak\"]",
+ "targets = [\"strong\", \"weak\"]\nweights = [1]",
+ ),
+ "expected 2 weights, got 1",
+ ),
+ (
+ VALID_CONFIG.replace(
+ "targets = [\"strong\", \"weak\"]",
+ "targets = [\"strong\", \"weak\"]\nweights = [0, 0]",
+ ),
+ "at least one weight must be positive",
),
(
VALID_CONFIG.replace("threshold = 0.5", "threshold = 1.5"),
@@ -436,6 +445,16 @@ threshold = 0.5
}
}
+ #[test]
+ fn accepts_relative_weights_and_seed() -> ServerResult<()> {
+ let weighted = VALID_CONFIG.replace(
+ "targets = [\"strong\", \"weak\"]",
+ "targets = [\"strong\", \"weak\"]\nweights = [1, 3]\nseed = 42",
+ );
+ server_state_from_toml(&weighted)?;
+ Ok(())
+ }
+
#[test]
fn api_key_environment_reference_is_validated() {
let missing = VALID_CONFIG.replacen(
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 7cb1a9938..91334dd24 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -134,19 +134,22 @@ fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult<Server
.map(|model| ModelConfig::new(model, backend.clone(), None))
.collect::<Vec<_>>();
let client: Arc<dyn RoutedLlmClient> = Arc::new(TranslatingLlmClient::new(&model_configs)?);
- let entries = routes.iter().map(|(route_model, targets)| {
- let target_set = LlmTargetSet::new(
- targets
- .iter()
- .map(|model| LlmTarget {
- semantic_name: (*model).to_string(),
- llm_client: Some(Arc::clone(&client)),
- })
- .collect(),
- );
- let algorithm: Arc<dyn Algorithm> = Arc::new(Random::new(target_set));
- ((*route_model).to_string(), algorithm)
- });
+ let entries = routes
+ .iter()
+ .map(|(route_model, targets)| {
+ let target_set = LlmTargetSet::new(
+ targets
+ .iter()
+ .map(|model| LlmTarget {
+ semantic_name: (*model).to_string(),
+ llm_client: Some(Arc::clone(&client)),
+ })
+ .collect(),
+ );
+ let algorithm: Arc<dyn Algorithm> = Arc::new(Random::new(target_set, None, None)?);
+ Ok(((*route_model).to_string(), algorithm))
+ })
+ .collect::<TestResult<Vec<_>>>()?;
Ok(ServerState::new(entries)?)
}
diff --git a/examples/libsy.py b/examples/libsy.py
index d0e5c1958..5da1d2530 100644
--- a/examples/libsy.py
+++ b/examples/libsy.py
@@ -39,7 +39,9 @@ async def main() -> None:
[
LlmTarget("fast", EchoClient("fast")),
LlmTarget("quality", EchoClient("quality")),
- ]
+ ],
+ weights=[1, 3],
+ seed=42,
)
random_decisions, random_response = await random.run(request)
print("Random:", random_decisions, random_response)
diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py
index b2014a33a..307b82216 100644
--- a/tests/test_libsy_minimal_bindings.py
+++ b/tests/test_libsy_minimal_bindings.py
@@ -60,6 +60,36 @@ async def test_random_runs_with_a_python_client() -> None:
assert response["outputs"][0]["content"] == [{"type": "text", "text": "fast"}]
+async def test_random_weights_and_seed_are_reproducible() -> None:
+ def algorithm():
+ return algorithms.random(
+ [
+ LlmTarget("fast", EchoClient("fast")),
+ LlmTarget("capable", EchoClient("capable")),
+ ],
+ weights=[1, 3],
+ seed=42,
+ )
+
+ first_router = algorithm()
+ second_router = algorithm()
+ first = [(await first_router.run(request_body()))[1]["model"] for _ in range(100)]
+ second = [(await second_router.run(request_body()))[1]["model"] for _ in range(100)]
+
+ assert first == second
+ assert 65 <= second.count("capable") <= 85
+
+
+def test_random_rejects_invalid_weights() -> None:
+ targets = [
+ LlmTarget("fast", EchoClient("fast")),
+ LlmTarget("capable", EchoClient("capable")),
+ ]
+
+ with pytest.raises(ValueError, match="expected 2 weights, got 1"):
+ algorithms.random(targets, weights=[1])
+
+
async def test_noop_needs_no_client() -> None:
decisions, response = await algorithms.noop().run(request_body())