Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

chore(libsy): move random routing algorithm into core (#81)

Ayush Agarwal320aa00d6e8d
diff --git a/Cargo.lock b/Cargo.lock
index aa3ce13f0..266129c7d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -855,6 +855,7 @@ version = "0.1.0"
dependencies = [
"async-trait",
"futures",
+ "rand 0.8.6",
"serde_json",
"switchyard-protocol",
"tokio",
@@ -868,7 +869,6 @@ dependencies = [
"async-trait",
"futures",
"libsy",
- "rand 0.8.6",
"switchyard-protocol",
"tokio",
"tokio-stream",
diff --git a/crates/libsy-examples/Cargo.toml b/crates/libsy-examples/Cargo.toml
index 3e8de0653..1c83d1f6e 100644
--- a/crates/libsy-examples/Cargo.toml
+++ b/crates/libsy-examples/Cargo.toml
@@ -16,7 +16,6 @@ rust-version.workspace = true
libsy = { path = "../libsy" }
switchyard-protocol = { path = "../libsy-protocol" }
async-trait = "0.1"
-rand = "0.8"
futures = "0.3"
tokio = { version = "1", features = ["full"] }
tokio-stream = "0.1"
diff --git a/crates/libsy-examples/examples/research_agent.rs b/crates/libsy-examples/examples/research_agent.rs
index a21f3ceae..221a37b6b 100644
--- a/crates/libsy-examples/examples/research_agent.rs
+++ b/crates/libsy-examples/examples/research_agent.rs
@@ -86,7 +86,7 @@ impl ResearchAgent {
#[tokio::main(flavor = "current_thread")]
async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
// Configure routing once: an LLM classifier over three named targets. Swapping
- // in `RandomOrchAlgo` needs no change to the agent.
+ // in `RandomAlgo` needs no change to the agent.
let algo: Arc<dyn Algorithm> = Arc::new(LlmClassifierOrchAlgo::new(
CLASSIFIER,
STRONG,
diff --git a/crates/libsy-examples/examples/streaming_agent.rs b/crates/libsy-examples/examples/streaming_agent.rs
index aa14f9e57..f871a4e07 100644
--- a/crates/libsy-examples/examples/streaming_agent.rs
+++ b/crates/libsy-examples/examples/streaming_agent.rs
@@ -19,9 +19,8 @@ use std::sync::Arc;
use futures::StreamExt;
use libsy::{
Algorithm, Context, LlmResponse, LlmResponseChunk, LlmResponseStream, LlmTarget, LlmTargetSet,
- Request, Response, Step,
+ RandomAlgo, Request, Response, Step,
};
-use libsy_examples::rand::RandomOrchAlgo;
use switchyard_protocol::{completion_text, text_request};
/// The "real" model call the agent makes to fulfill an offloaded promise: a response
@@ -56,7 +55,7 @@ async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
semantic_name: "stream/model".to_string(),
llm_client: None,
}]);
- let algo: Arc<dyn Algorithm> = Arc::new(RandomOrchAlgo::new(targets));
+ let algo: Arc<dyn Algorithm> = Arc::new(RandomAlgo::new(targets));
let request = Request {
llm_request: text_request(Some("auto".to_string()), "tell me about switchyard"),
diff --git a/crates/libsy-examples/src/lib.rs b/crates/libsy-examples/src/lib.rs
index 91874debf..802964834 100644
--- a/crates/libsy-examples/src/lib.rs
+++ b/crates/libsy-examples/src/lib.rs
@@ -4,9 +4,9 @@
//! Reference algorithms for [`libsy`], kept out of the core crate but compiled and
//! tested here so they stay current. Each is a worked example of the
//! [`Algorithm`](libsy::Algorithm) trait; the `examples/` directory has runnable agents
-//! that drive them.
+//! that drive them. The core crate provides [`libsy::RandomAlgo`] for uniform random
+//! routing.
//!
-//! - [`rand::RandomOrchAlgo`] — uniform random over the target set (one call).
//! - [`llm_class::LlmClassifierOrchAlgo`] — classify with one model, then route to a
//! strong/weak model (multi-step).
//! - [`ensemble::EnsembleOrchAlgo`] — fan out to several models, judge, and commit
@@ -14,4 +14,3 @@
pub mod ensemble;
pub mod llm_class;
-pub mod rand;
diff --git a/crates/libsy/Cargo.toml b/crates/libsy/Cargo.toml
index 5054ac1cf..df8303835 100644
--- a/crates/libsy/Cargo.toml
+++ b/crates/libsy/Cargo.toml
@@ -15,6 +15,7 @@ rust-version.workspace = true
async-trait = "0.1"
serde_json = "1"
futures = "0.3"
+rand = "0.8"
switchyard-protocol = { path = "../libsy-protocol" }
tokio = { version = "1", features = ["full"] }
tokio-stream = "0.1"
diff --git a/crates/libsy/README.md b/crates/libsy/README.md
index 99a8771ee..423ab070b 100644
--- a/crates/libsy/README.md
+++ b/crates/libsy/README.md
@@ -251,13 +251,14 @@ impl Algorithm for LlmClassifierOrchAlgo {
## Explore
-Reference algorithms and runnable agents live in the sibling
-[`libsy-examples`](../libsy-examples) crate (kept out of `libsy` itself, but compiled and
-tested — `cargo test -p libsy-examples`).
+The core crate includes uniform random routing. More reference algorithms and runnable
+agents live in the sibling [`libsy-examples`](../libsy-examples) crate (compiled and tested
+with `cargo test -p libsy-examples`).
**Reference algorithms** — implementations to read and route with:
-- [`RandomOrchAlgo`](../libsy-examples/src/rand.rs) — uniform random over the set (one call).
+- [`RandomAlgo`](src/algorithms/rand.rs) — uniform random over the set
+ (one call).
- [`LlmClassifierOrchAlgo`](../libsy-examples/src/llm_class.rs) — classify, then route
strong/weak; fail open to strong.
- [`EnsembleOrchAlgo`](../libsy-examples/src/ensemble.rs) — stateful: fan out to
diff --git a/crates/libsy/src/algorithms.rs b/crates/libsy/src/algorithms.rs
index e08c8fd2d..6f88a702d 100644
--- a/crates/libsy/src/algorithms.rs
+++ b/crates/libsy/src/algorithms.rs
@@ -2,3 +2,4 @@
// SPDX-License-Identifier: Apache-2.0
pub mod noop;
+pub mod rand;
diff --git a/crates/libsy-examples/src/rand.rs b/crates/libsy/src/algorithms/rand.rs
similarity index 66%
rename from crates/libsy-examples/src/rand.rs
rename to crates/libsy/src/algorithms/rand.rs
index faf95cf1b..b80a84a21 100644
--- a/crates/libsy-examples/src/rand.rs
+++ b/crates/libsy/src/algorithms/rand.rs
@@ -5,8 +5,8 @@
//!
//! Selects one target from the set uniformly at random and calls it. This is the
//! simplest possible routing algorithm and the reference for the single-call
-//! shape: one `driver.call_llm_target` inside `create_run_task`. (Weighted selection could
-//! be layered on later; the set defines the candidates.)
+//! shape: one `driver.call_llm_target` inside `create_run_task`. Weighted selection
+//! can be layered on later; the set defines the candidates.
use std::error::Error;
use std::sync::Arc;
@@ -14,9 +14,9 @@ use std::sync::Arc;
use async_trait::async_trait;
use rand::seq::SliceRandom;
-use libsy::{Algorithm, Context, Decision, Driver, LlmTargetSet, Request, Response};
+use crate::{Algorithm, Context, Decision, Driver, LlmTargetSet, Request, Response};
-/// Decision produced by [`RandomOrchAlgo`]: which target was chosen and why.
+/// Decision produced by [`RandomAlgo`]: which target was chosen and why.
pub struct RandomDecision {
/// The randomly selected target/model.
pub selected_model: String,
@@ -28,40 +28,40 @@ impl Decision for RandomDecision {
fn selected_model(&self) -> &str {
&self.selected_model
}
+
fn reasoning(&self) -> Option<&str> {
Some(&self.reasoning)
}
+
fn as_any(&self) -> &dyn std::any::Any {
self
}
}
/// Uniform random router over a target set.
-pub struct RandomOrchAlgo {
+pub struct RandomAlgo {
target_set: LlmTargetSet,
}
-impl RandomOrchAlgo {
- /// Create a router over `target_set`. Wrap it in an
- /// [`Arc`](std::sync::Arc) and drive it with
- /// [`run`](libsy::Algorithm::run) or
- /// [`run_stream`](libsy::Algorithm::run_stream).
+impl RandomAlgo {
+ /// Creates a router over `target_set`.
+ ///
+ /// Wrap it in an [`Arc`] and drive it with [`Algorithm::run`] or
+ /// [`Algorithm::run_stream`].
pub fn new(target_set: LlmTargetSet) -> Self {
Self { target_set }
}
}
#[async_trait]
-impl Algorithm for RandomOrchAlgo {
+impl Algorithm for RandomAlgo {
async fn create_run_task(
self: Arc<Self>,
ctx: Context,
driver: Driver,
request: Request,
) -> Result<Response, Box<dyn Error + Send + Sync>> {
- // Select a target uniformly at random. Scope the RNG so the non-Send
- // `ThreadRng` is dropped before the await below, keeping the returned
- // future `Send` (required by the `Algorithm` bound).
+ // Scope the non-Send ThreadRng before the await so the future remains Send.
let target = {
let mut rng = rand::thread_rng();
self.target_set
@@ -71,16 +71,13 @@ impl Algorithm for RandomOrchAlgo {
.clone()
};
- // Route by target semantic name; the caller's client (or offload host) maps
- // it to the provider model id when it serves or offloads the call.
let selected = target.semantic_name.clone();
let decision: Arc<dyn Decision> = Arc::new(RandomDecision {
reasoning: format!("random routing selected target '{selected}'"),
selected_model: selected,
});
- // Publish the decision to the stream, then offload the call.
- driver.info(ctx.clone(), decision.clone()).await?;
+ driver.info(ctx.clone(), Arc::clone(&decision)).await?;
driver
.call_llm_target(ctx, &target, request, decision)
.await
@@ -89,13 +86,14 @@ impl Algorithm for RandomOrchAlgo {
#[cfg(test)]
mod tests {
- use super::*;
- use libsy::{LlmClient, LlmResponse, LlmTarget, Response, RoutedRequest, Signals};
use std::collections::HashSet;
+
use switchyard_protocol::{completion_text, text_request, text_response};
- /// Echoes back the target name it was called with, so a test can tell which
- /// target the algo selected.
+ use super::*;
+ use crate::{LlmClient, LlmResponse, LlmTarget, RoutedRequest, Signals};
+
+ /// Echoes the selected target so tests can inspect which target was called.
struct EchoClient;
#[async_trait]
@@ -122,27 +120,28 @@ mod tests {
}
}
- /// Build a random-routing algorithm over `names`; every target echoes its name.
- fn orch(names: &[&str]) -> Arc<dyn Algorithm> {
- Arc::new(algo(names))
- }
-
- fn algo(names: &[&str]) -> RandomOrchAlgo {
- let targets: Vec<LlmTarget> = names
+ /// Builds a random router whose targets all share an echo client.
+ fn algorithm(names: &[&str]) -> RandomAlgo {
+ let targets = names
.iter()
.map(|name| LlmTarget {
- semantic_name: name.to_string(),
+ semantic_name: (*name).to_string(),
llm_client: Some(Arc::new(EchoClient)),
})
.collect();
- RandomOrchAlgo::new(LlmTargetSet::new(targets))
+ RandomAlgo::new(LlmTargetSet::new(targets))
+ }
+
+ fn shared_algorithm(names: &[&str]) -> Arc<dyn Algorithm> {
+ Arc::new(algorithm(names))
}
#[tokio::test]
async fn single_target_is_always_selected_and_called(
) -> Result<(), Box<dyn Error + Send + Sync>> {
- let orch = orch(&["only/model"]);
- let (trace, response) = orch.clone().run(Context::default(), request()).await?;
+ let algorithm = shared_algorithm(&["only/model"]);
+ let (trace, response) = algorithm.run(Context::default(), request()).await?;
+
assert_eq!(
response
.llm_response
@@ -160,9 +159,10 @@ mod tests {
async fn selected_target_is_in_the_set_and_matches_the_trace(
) -> Result<(), Box<dyn Error + Send + Sync>> {
let names = ["a/model", "b/model", "c/model"];
- let orch = orch(&names);
+ let algorithm = shared_algorithm(&names);
+
for _ in 0..50 {
- let (trace, response) = orch.clone().run(Context::default(), request()).await?;
+ let (trace, response) = algorithm.clone().run(Context::default(), request()).await?;
let selected = response
.llm_response
.as_agg()
@@ -172,7 +172,6 @@ mod tests {
names.contains(&selected.as_str()),
"selected {selected} not in target set"
);
- // The trace records the same target that was actually called.
assert_eq!(trace[0].selected_model(), selected.as_str());
}
Ok(())
@@ -181,10 +180,11 @@ mod tests {
#[tokio::test]
async fn selection_covers_all_targets_over_many_runs(
) -> Result<(), Box<dyn Error + Send + Sync>> {
- let orch = orch(&["a/model", "b/model"]);
+ let algorithm = shared_algorithm(&["a/model", "b/model"]);
let mut seen = HashSet::new();
+
for _ in 0..100 {
- let (_, response) = orch.clone().run(Context::default(), request()).await?;
+ let (_, response) = algorithm.clone().run(Context::default(), request()).await?;
seen.insert(
response
.llm_response
@@ -193,7 +193,8 @@ mod tests {
.unwrap_or_default(),
);
}
- // 100 uniform draws over two targets: both should appear (miss ~ 2^-99).
+
+ // Missing either target after 100 uniform draws has probability about 2^-99.
assert_eq!(
seen.len(),
2,
@@ -204,33 +205,29 @@ mod tests {
#[tokio::test]
async fn empty_target_set_errors() {
- let orch = orch(&[]);
- assert!(orch
- .clone()
- .run(Context::default(), request())
- .await
- .is_err());
+ let algorithm = shared_algorithm(&[]);
+ assert!(algorithm.run(Context::default(), request()).await.is_err());
}
#[tokio::test]
async fn process_signals_is_a_noop() -> Result<(), Box<dyn Error + Send + Sync>> {
- let algo = algo(&["only/model"]);
- Arc::new(algo).process_signals(Signals {}).await?;
+ Arc::new(algorithm(&["only/model"]))
+ .process_signals(Signals {})
+ .await?;
Ok(())
}
#[tokio::test]
async fn decision_is_inspectable_and_downcasts() -> Result<(), Box<dyn Error + Send + Sync>> {
- let orch = orch(&["only/model"]);
- let (trace, _) = orch.clone().run(Context::default(), request()).await?;
+ let algorithm = shared_algorithm(&["only/model"]);
+ let (trace, _) = algorithm.run(Context::default(), request()).await?;
let decision = &trace[0];
- // Uniform, algo-agnostic access via the trait — no concrete type needed.
+
assert_eq!(decision.selected_model(), "only/model");
assert!(decision
.reasoning()
.unwrap_or_default()
.contains("only/model"));
- // Escape hatch: downcast to the concrete decision when the algo is known.
let concrete = decision
.as_any()
.downcast_ref::<RandomDecision>()
diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs
index 26259c832..d4b81ee32 100644
--- a/crates/libsy/src/lib.rs
+++ b/crates/libsy/src/lib.rs
@@ -52,13 +52,15 @@
//! responsible for its own thread-safety — stateless (like the reference routers) or
//! interior mutability over just its own state.
//!
-//! ## Reference algorithms
+//! ## Algorithms
//!
-//! Worked implementations — a random router, an LLM classifier, and a stateful
-//! ensemble — plus runnable agents live in the `libsy-examples` crate.
+//! [`RandomAlgo`] provides uniform random routing in the core crate. Worked
+//! implementations of an LLM classifier and a stateful ensemble, plus runnable
+//! agents, live in the `libsy-examples` crate.
mod algorithms;
pub use algorithms::noop::{NoopAlgo, NoopDecision};
+pub use algorithms::rand::{RandomAlgo, RandomDecision};
mod driver;