Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
chore(libsy): move random routing algorithm into core (#81)
Ayush Agarwal320aa00d6e8d
diff --git a/Cargo.lock b/Cargo.lockindex aa3ce13f0..266129c7d 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -855,6 +855,7 @@ version = "0.1.0"dependencies = ["async-trait","futures",+ "rand 0.8.6","serde_json","switchyard-protocol","tokio",@@ -868,7 +869,6 @@ dependencies = ["async-trait","futures","libsy",- "rand 0.8.6","switchyard-protocol","tokio","tokio-stream",diff --git a/crates/libsy-examples/Cargo.toml b/crates/libsy-examples/Cargo.tomlindex 3e8de0653..1c83d1f6e 100644--- a/crates/libsy-examples/Cargo.toml+++ b/crates/libsy-examples/Cargo.toml@@ -16,7 +16,6 @@ rust-version.workspace = truelibsy = { path = "../libsy" }switchyard-protocol = { path = "../libsy-protocol" }async-trait = "0.1"-rand = "0.8"futures = "0.3"tokio = { version = "1", features = ["full"] }tokio-stream = "0.1"diff --git a/crates/libsy-examples/examples/research_agent.rs b/crates/libsy-examples/examples/research_agent.rsindex a21f3ceae..221a37b6b 100644--- a/crates/libsy-examples/examples/research_agent.rs+++ b/crates/libsy-examples/examples/research_agent.rs@@ -86,7 +86,7 @@ impl ResearchAgent {#[tokio::main(flavor = "current_thread")]async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {// Configure routing once: an LLM classifier over three named targets. Swapping- // in `RandomOrchAlgo` needs no change to the agent.+ // in `RandomAlgo` needs no change to the agent.let algo: Arc<dyn Algorithm> = Arc::new(LlmClassifierOrchAlgo::new(CLASSIFIER,STRONG,diff --git a/crates/libsy-examples/examples/streaming_agent.rs b/crates/libsy-examples/examples/streaming_agent.rsindex aa14f9e57..f871a4e07 100644--- a/crates/libsy-examples/examples/streaming_agent.rs+++ b/crates/libsy-examples/examples/streaming_agent.rs@@ -19,9 +19,8 @@ use std::sync::Arc;use futures::StreamExt;use libsy::{Algorithm, Context, LlmResponse, LlmResponseChunk, LlmResponseStream, LlmTarget, LlmTargetSet,- Request, Response, Step,+ RandomAlgo, Request, Response, Step,};-use libsy_examples::rand::RandomOrchAlgo;use switchyard_protocol::{completion_text, text_request};/// The "real" model call the agent makes to fulfill an offloaded promise: a response@@ -56,7 +55,7 @@ async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {semantic_name: "stream/model".to_string(),llm_client: None,}]);- let algo: Arc<dyn Algorithm> = Arc::new(RandomOrchAlgo::new(targets));+ let algo: Arc<dyn Algorithm> = Arc::new(RandomAlgo::new(targets));let request = Request {llm_request: text_request(Some("auto".to_string()), "tell me about switchyard"),diff --git a/crates/libsy-examples/src/lib.rs b/crates/libsy-examples/src/lib.rsindex 91874debf..802964834 100644--- a/crates/libsy-examples/src/lib.rs+++ b/crates/libsy-examples/src/lib.rs@@ -4,9 +4,9 @@//! Reference algorithms for [`libsy`], kept out of the core crate but compiled and//! tested here so they stay current. Each is a worked example of the//! [`Algorithm`](libsy::Algorithm) trait; the `examples/` directory has runnable agents-//! that drive them.+//! that drive them. The core crate provides [`libsy::RandomAlgo`] for uniform random+//! routing.//!-//! - [`rand::RandomOrchAlgo`] — uniform random over the target set (one call).//! - [`llm_class::LlmClassifierOrchAlgo`] — classify with one model, then route to a//! strong/weak model (multi-step).//! - [`ensemble::EnsembleOrchAlgo`] — fan out to several models, judge, and commit@@ -14,4 +14,3 @@pub mod ensemble;pub mod llm_class;-pub mod rand;diff --git a/crates/libsy/Cargo.toml b/crates/libsy/Cargo.tomlindex 5054ac1cf..df8303835 100644--- a/crates/libsy/Cargo.toml+++ b/crates/libsy/Cargo.toml@@ -15,6 +15,7 @@ rust-version.workspace = trueasync-trait = "0.1"serde_json = "1"futures = "0.3"+rand = "0.8"switchyard-protocol = { path = "../libsy-protocol" }tokio = { version = "1", features = ["full"] }tokio-stream = "0.1"diff --git a/crates/libsy/README.md b/crates/libsy/README.mdindex 99a8771ee..423ab070b 100644--- a/crates/libsy/README.md+++ b/crates/libsy/README.md@@ -251,13 +251,14 @@ impl Algorithm for LlmClassifierOrchAlgo {## Explore-Reference algorithms and runnable agents live in the sibling-[`libsy-examples`](../libsy-examples) crate (kept out of `libsy` itself, but compiled and-tested — `cargo test -p libsy-examples`).+The core crate includes uniform random routing. More reference algorithms and runnable+agents live in the sibling [`libsy-examples`](../libsy-examples) crate (compiled and tested+with `cargo test -p libsy-examples`).**Reference algorithms** — implementations to read and route with:-- [`RandomOrchAlgo`](../libsy-examples/src/rand.rs) — uniform random over the set (one call).+- [`RandomAlgo`](src/algorithms/rand.rs) — uniform random over the set+ (one call).- [`LlmClassifierOrchAlgo`](../libsy-examples/src/llm_class.rs) — classify, then routestrong/weak; fail open to strong.- [`EnsembleOrchAlgo`](../libsy-examples/src/ensemble.rs) — stateful: fan out todiff --git a/crates/libsy/src/algorithms.rs b/crates/libsy/src/algorithms.rsindex e08c8fd2d..6f88a702d 100644--- a/crates/libsy/src/algorithms.rs+++ b/crates/libsy/src/algorithms.rs@@ -2,3 +2,4 @@// SPDX-License-Identifier: Apache-2.0pub mod noop;+pub mod rand;diff --git a/crates/libsy-examples/src/rand.rs b/crates/libsy/src/algorithms/rand.rssimilarity index 66%rename from crates/libsy-examples/src/rand.rsrename to crates/libsy/src/algorithms/rand.rsindex faf95cf1b..b80a84a21 100644--- a/crates/libsy-examples/src/rand.rs+++ b/crates/libsy/src/algorithms/rand.rs@@ -5,8 +5,8 @@//!//! Selects one target from the set uniformly at random and calls it. This is the//! simplest possible routing algorithm and the reference for the single-call-//! shape: one `driver.call_llm_target` inside `create_run_task`. (Weighted selection could-//! be layered on later; the set defines the candidates.)+//! shape: one `driver.call_llm_target` inside `create_run_task`. Weighted selection+//! can be layered on later; the set defines the candidates.use std::error::Error;use std::sync::Arc;@@ -14,9 +14,9 @@ use std::sync::Arc;use async_trait::async_trait;use rand::seq::SliceRandom;-use libsy::{Algorithm, Context, Decision, Driver, LlmTargetSet, Request, Response};+use crate::{Algorithm, Context, Decision, Driver, LlmTargetSet, Request, Response};-/// Decision produced by [`RandomOrchAlgo`]: which target was chosen and why.+/// Decision produced by [`RandomAlgo`]: which target was chosen and why.pub struct RandomDecision {/// The randomly selected target/model.pub selected_model: String,@@ -28,40 +28,40 @@ impl Decision for RandomDecision {fn selected_model(&self) -> &str {&self.selected_model}+fn reasoning(&self) -> Option<&str> {Some(&self.reasoning)}+fn as_any(&self) -> &dyn std::any::Any {self}}/// Uniform random router over a target set.-pub struct RandomOrchAlgo {+pub struct RandomAlgo {target_set: LlmTargetSet,}-impl RandomOrchAlgo {- /// Create a router over `target_set`. Wrap it in an- /// [`Arc`](std::sync::Arc) and drive it with- /// [`run`](libsy::Algorithm::run) or- /// [`run_stream`](libsy::Algorithm::run_stream).+impl RandomAlgo {+ /// Creates a router over `target_set`.+ ///+ /// Wrap it in an [`Arc`] and drive it with [`Algorithm::run`] or+ /// [`Algorithm::run_stream`].pub fn new(target_set: LlmTargetSet) -> Self {Self { target_set }}}#[async_trait]-impl Algorithm for RandomOrchAlgo {+impl Algorithm for RandomAlgo {async fn create_run_task(self: Arc<Self>,ctx: Context,driver: Driver,request: Request,) -> Result<Response, Box<dyn Error + Send + Sync>> {- // Select a target uniformly at random. Scope the RNG so the non-Send- // `ThreadRng` is dropped before the await below, keeping the returned- // future `Send` (required by the `Algorithm` bound).+ // Scope the non-Send ThreadRng before the await so the future remains Send.let target = {let mut rng = rand::thread_rng();self.target_set@@ -71,16 +71,13 @@ impl Algorithm for RandomOrchAlgo {.clone()};- // Route by target semantic name; the caller's client (or offload host) maps- // it to the provider model id when it serves or offloads the call.let selected = target.semantic_name.clone();let decision: Arc<dyn Decision> = Arc::new(RandomDecision {reasoning: format!("random routing selected target '{selected}'"),selected_model: selected,});- // Publish the decision to the stream, then offload the call.- driver.info(ctx.clone(), decision.clone()).await?;+ driver.info(ctx.clone(), Arc::clone(&decision)).await?;driver.call_llm_target(ctx, &target, request, decision).await@@ -89,13 +86,14 @@ impl Algorithm for RandomOrchAlgo {#[cfg(test)]mod tests {- use super::*;- use libsy::{LlmClient, LlmResponse, LlmTarget, Response, RoutedRequest, Signals};use std::collections::HashSet;+use switchyard_protocol::{completion_text, text_request, text_response};- /// Echoes back the target name it was called with, so a test can tell which- /// target the algo selected.+ use super::*;+ use crate::{LlmClient, LlmResponse, LlmTarget, RoutedRequest, Signals};++ /// Echoes the selected target so tests can inspect which target was called.struct EchoClient;#[async_trait]@@ -122,27 +120,28 @@ mod tests {}}- /// Build a random-routing algorithm over `names`; every target echoes its name.- fn orch(names: &[&str]) -> Arc<dyn Algorithm> {- Arc::new(algo(names))- }-- fn algo(names: &[&str]) -> RandomOrchAlgo {- let targets: Vec<LlmTarget> = names+ /// Builds a random router whose targets all share an echo client.+ fn algorithm(names: &[&str]) -> RandomAlgo {+ let targets = names.iter().map(|name| LlmTarget {- semantic_name: name.to_string(),+ semantic_name: (*name).to_string(),llm_client: Some(Arc::new(EchoClient)),}).collect();- RandomOrchAlgo::new(LlmTargetSet::new(targets))+ RandomAlgo::new(LlmTargetSet::new(targets))+ }++ fn shared_algorithm(names: &[&str]) -> Arc<dyn Algorithm> {+ Arc::new(algorithm(names))}#[tokio::test]async fn single_target_is_always_selected_and_called() -> Result<(), Box<dyn Error + Send + Sync>> {- let orch = orch(&["only/model"]);- let (trace, response) = orch.clone().run(Context::default(), request()).await?;+ let algorithm = shared_algorithm(&["only/model"]);+ let (trace, response) = algorithm.run(Context::default(), request()).await?;+assert_eq!(response.llm_response@@ -160,9 +159,10 @@ mod tests {async fn selected_target_is_in_the_set_and_matches_the_trace() -> Result<(), Box<dyn Error + Send + Sync>> {let names = ["a/model", "b/model", "c/model"];- let orch = orch(&names);+ let algorithm = shared_algorithm(&names);+for _ in 0..50 {- let (trace, response) = orch.clone().run(Context::default(), request()).await?;+ let (trace, response) = algorithm.clone().run(Context::default(), request()).await?;let selected = response.llm_response.as_agg()@@ -172,7 +172,6 @@ mod tests {names.contains(&selected.as_str()),"selected {selected} not in target set");- // The trace records the same target that was actually called.assert_eq!(trace[0].selected_model(), selected.as_str());}Ok(())@@ -181,10 +180,11 @@ mod tests {#[tokio::test]async fn selection_covers_all_targets_over_many_runs() -> Result<(), Box<dyn Error + Send + Sync>> {- let orch = orch(&["a/model", "b/model"]);+ let algorithm = shared_algorithm(&["a/model", "b/model"]);let mut seen = HashSet::new();+for _ in 0..100 {- let (_, response) = orch.clone().run(Context::default(), request()).await?;+ let (_, response) = algorithm.clone().run(Context::default(), request()).await?;seen.insert(response.llm_response@@ -193,7 +193,8 @@ mod tests {.unwrap_or_default(),);}- // 100 uniform draws over two targets: both should appear (miss ~ 2^-99).++ // Missing either target after 100 uniform draws has probability about 2^-99.assert_eq!(seen.len(),2,@@ -204,33 +205,29 @@ mod tests {#[tokio::test]async fn empty_target_set_errors() {- let orch = orch(&[]);- assert!(orch- .clone()- .run(Context::default(), request())- .await- .is_err());+ let algorithm = shared_algorithm(&[]);+ assert!(algorithm.run(Context::default(), request()).await.is_err());}#[tokio::test]async fn process_signals_is_a_noop() -> Result<(), Box<dyn Error + Send + Sync>> {- let algo = algo(&["only/model"]);- Arc::new(algo).process_signals(Signals {}).await?;+ Arc::new(algorithm(&["only/model"]))+ .process_signals(Signals {})+ .await?;Ok(())}#[tokio::test]async fn decision_is_inspectable_and_downcasts() -> Result<(), Box<dyn Error + Send + Sync>> {- let orch = orch(&["only/model"]);- let (trace, _) = orch.clone().run(Context::default(), request()).await?;+ let algorithm = shared_algorithm(&["only/model"]);+ let (trace, _) = algorithm.run(Context::default(), request()).await?;let decision = &trace[0];- // Uniform, algo-agnostic access via the trait — no concrete type needed.+assert_eq!(decision.selected_model(), "only/model");assert!(decision.reasoning().unwrap_or_default().contains("only/model"));- // Escape hatch: downcast to the concrete decision when the algo is known.let concrete = decision.as_any().downcast_ref::<RandomDecision>()diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rsindex 26259c832..d4b81ee32 100644--- a/crates/libsy/src/lib.rs+++ b/crates/libsy/src/lib.rs@@ -52,13 +52,15 @@//! responsible for its own thread-safety — stateless (like the reference routers) or//! interior mutability over just its own state.//!-//! ## Reference algorithms+//! ## Algorithms//!-//! Worked implementations — a random router, an LLM classifier, and a stateful-//! ensemble — plus runnable agents live in the `libsy-examples` crate.+//! [`RandomAlgo`] provides uniform random routing in the core crate. Worked+//! implementations of an LLM classifier and a stateful ensemble, plus runnable+//! agents, live in the `libsy-examples` crate.mod algorithms;pub use algorithms::noop::{NoopAlgo, NoopDecision};+pub use algorithms::rand::{RandomAlgo, RandomDecision};mod driver;